diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 61e0e64aa8..3a6d2acf8a 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -108,8 +108,6 @@ add_library( settings_setting.h slot_vector.h socket_types.h - sparse_large_vector.cpp - sparse_large_vector.h spin_lock.h stb.cpp stb.h @@ -139,6 +137,8 @@ add_library( uuid.cpp uuid.h vector_math.h + virtual_buffer.cpp + virtual_buffer.h zstd_compression.cpp zstd_compression.h fs/ryujinx_compat.h fs/ryujinx_compat.cpp diff --git a/src/common/fiber.cpp b/src/common/fiber.cpp index 0e7f28c5b9..69eca732eb 100644 --- a/src/common/fiber.cpp +++ b/src/common/fiber.cpp @@ -9,6 +9,7 @@ #include "common/assert.h" #include "common/fiber.h" +#include "common/virtual_buffer.h" #include diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 75a9da80e9..dd0780a283 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -178,14 +178,6 @@ public: Release(); } - void* Allocate(size_t size) { - auto* ptr = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - if (ptr == nullptr) { - LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, error {}", size, GetLastError()); - } - return ptr; - } - void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) { std::unique_lock lock{placeholder_mutex}; if (!IsNiechePlaceholder(virtual_offset, length)) { @@ -406,10 +398,6 @@ private: // For managarm: see https://github.com/managarm/managarm/issues/1370 #else // ^^^ Windows ^^^ vvv POSIX vvv -#ifndef MAP_NOCORE -#define MAP_NOCORE 0 -#endif - #ifdef ARCHITECTURE_arm64 static void* ChooseVirtualBase(size_t virtual_size) { @@ -434,7 +422,7 @@ static void* ChooseVirtualBase(size_t virtual_size) { // Note: we may be able to take advantage of MAP_FIXED_NOREPLACE here. void* map_pointer = mmap(reinterpret_cast(hint_address), virtual_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_NOCORE, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); // If we successfully mapped, we're done. if (reinterpret_cast(map_pointer) == hint_address) { @@ -454,11 +442,11 @@ static void* ChooseVirtualBase(size_t virtual_size) { static void* ChooseVirtualBase(size_t virtual_size) { #if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__OpenBSD__) || defined(__sun__) || defined(__HAIKU__) || defined(__managarm__) || defined(__AIX__) - void* virtual_base = mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_ALIGNED_SUPER | MAP_NOCORE, -1, 0); + void* virtual_base = mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_ALIGNED_SUPER, -1, 0); if (virtual_base != MAP_FAILED) return virtual_base; #endif - return mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_NOCORE, -1, 0); + return mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); } #endif @@ -552,13 +540,13 @@ public: } if (use_anon) { LOG_WARNING(Common_Memory, "Using private mappings instead of shared ones"); - backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_NOCORE, -1, 0)); + backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); if (fd > 0) { fd = -1; close(fd); } } else { - backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_NOCORE, fd, 0)); + backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0)); } if (backing_base == MAP_FAILED) { LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno)); @@ -582,14 +570,6 @@ public: Release(); } - void* Allocate(size_t size) { - auto* ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (ptr == MAP_FAILED) { - LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, {}", size, strerror(errno)); - } - return ptr; - } - void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) { // Intersect the range with our address space. AdjustMap(&virtual_offset, &length); @@ -710,10 +690,12 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) { #if defined(__OPENORBIS__) || defined(__managarm__) LOG_WARNING(HW_Memory, "Platform doesn't support fastmem"); - backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + fallback_buffer.emplace(backing_size); + backing_base = fallback_buffer->data(); virtual_base = nullptr; #else // Try to allocate a fastmem arena. + // The implementation will fail with std::bad_alloc on errors. impl = std::make_unique(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize); if (impl->Init()) { backing_base = impl->backing_base; @@ -724,26 +706,16 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) virtual_base_offset = virtual_base - impl->virtual_base; } } else { - LOG_WARNING(HW_Memory, "Platform can support fastmem, but can't create it"); - fallback_buffer = true; - backing_base = static_cast(impl->Allocate(backing_size)); - virtual_base = nullptr; impl.reset(); + LOG_WARNING(HW_Memory, "Platform can support fastmem, but can't create it"); + fallback_buffer.emplace(backing_size); + backing_base = fallback_buffer->data(); + virtual_base = nullptr; } #endif } -HostMemory::~HostMemory() { -#ifdef _WIN32 - if (fallback_buffer) { - VirtualFree(backing_base, backing_size, MEM_RELEASE); - } -#else - if (fallback_buffer) { - munmap(backing_base, backing_size); - } -#endif -} +HostMemory::~HostMemory() = default; HostMemory::HostMemory(HostMemory&&) noexcept = default; diff --git a/src/common/host_memory.h b/src/common/host_memory.h index a7fd0532b7..24f4670732 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -10,6 +10,7 @@ #include #include "common/common_funcs.h" #include "common/common_types.h" +#include "common/virtual_buffer.h" namespace Common { @@ -85,7 +86,7 @@ private: u8* virtual_base{}; size_t virtual_base_offset{}; // Windows requires it for kernels whom lack proper support for some functions! - bool fallback_buffer{false}; + std::optional> fallback_buffer; }; } // namespace Common diff --git a/src/common/page_table.cpp b/src/common/page_table.cpp index 73278cfc59..d34ba89993 100644 --- a/src/common/page_table.cpp +++ b/src/common/page_table.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project @@ -13,11 +13,39 @@ PageTable::PageTable() = default; PageTable::~PageTable() noexcept = default; -void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_bits) { - auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_bits); - entries.ResizeAndClear(num_page_table_entries); +bool PageTable::BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context, + Common::ProcessAddress address) const { + out_context->next_offset = GetInteger(address); + out_context->next_page = address / page_size; + + return this->ContinueTraversal(out_entry, out_context); +} + +bool PageTable::ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const { + // Setup invalid defaults. + out_entry->phys_addr = 0; + out_entry->block_size = page_size; + // Validate that we can read the actual entry. + if (auto const page = context->next_page; page < entries.size()) { + // Validate that the entry is mapped. + if (auto const paddr = entries[page].addr; paddr != 0) { + // Populate the results. + out_entry->phys_addr = paddr + context->next_offset; + context->next_page += 1; + context->next_offset += page_size; + return true; + } + } + context->next_page += 1; + context->next_offset += page_size; + return false; +} + +void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits) { + auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_size_in_bits); + entries.resize(num_page_table_entries); current_address_space_width_in_bits = address_space_width_in_bits; - current_page_bits = page_bits; + page_size = 1ULL << page_size_in_bits; } } // namespace Common diff --git a/src/common/page_table.h b/src/common/page_table.h index 1588eca616..0dfd152331 100644 --- a/src/common/page_table.h +++ b/src/common/page_table.h @@ -9,22 +9,22 @@ #include #include "common/common_types.h" -#include "common/sparse_large_vector.h" #include "common/typed_address.h" +#include "common/virtual_buffer.h" namespace Common { enum class PageType : u8 { /// Page is unmapped and should cause an access error. - Unmapped = 0b00, + Unmapped, /// Page is mapped to regular memory. This is the only type you can get pointers to. - Memory = 0b01, + Memory, /// Page is mapped to regular memory, but inaccessible from CPU fastmem and must use /// the callbacks. - DebugMemory = 0b10, + DebugMemory, /// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and /// invalidation - RasterizerCachedMemory = 0b11, + RasterizerCachedMemory, }; /** @@ -42,86 +42,57 @@ struct PageTable { u64 next_offset{}; }; - /// Masks out bits reserved for attribute tagging. - static constexpr u64 ATTRIBUTE_MASK = ((1ULL << 44) - 1) << 12; - - /// Specifies sign bit for page table entries. - static constexpr u64 SIGN_BIT = 45 + 12; // 44 bits of data + page offset + /// Number of bits reserved for attribute tagging. + /// This can be at most the guaranteed alignment of the pointers in the page table. + static constexpr int ATTRIBUTE_BITS = 2; /** - * Atomic tuple of host pointer, page type, and block id. - * This uses the lower bits of a given pointer to store the attributes. + * Pair of host pointer and page type attribute. + * This uses the lower bits of a given pointer to store the attribute tag. * Writing and reading the pointer attribute pair is guaranteed to be atomic for the same method * call. In other words, they are guaranteed to be synchronized at all times. */ - class PageEntryData { + class PageInfo { public: - struct Data { - Data(bool marked_, PageType type_, u16 block_, u64 page_) - : marked(static_cast(marked_) & 0b1) - , type(static_cast(type_) & ((1ULL << 2) - 1)) - , block(static_cast(block_) & ((1ULL << 9) - 1)) - , page((page_ >> 12) & ((1ULL << 45) - 1)) - , block2((static_cast(block_) >> 9) & ((1ULL << 7) - 1)) {} - u64 marked : 1; - u64 type : 2; - u64 block : 9; - u64 page : 45; // 44 bits of actual data (64 - page offset (12) - reserved (8)) + a sign bit - u64 block2 : 7; - }; - - [[nodiscard]] Data Raw() const noexcept { - return std::bit_cast(data_raw.load(std::memory_order_relaxed)); - } - /// Returns the page pointer - [[nodiscard]] uintptr_t Pointer(bool ignored_marked = false) const noexcept { - return ExtractPointer(std::bit_cast(data_raw.load(std::memory_order_relaxed)), ignored_marked); + [[nodiscard]] uintptr_t Pointer() const noexcept { + return ExtractPointer(raw.load(std::memory_order_relaxed)); } /// Returns the page type attribute [[nodiscard]] PageType Type() const noexcept { - return static_cast(std::bit_cast(data_raw.load(std::memory_order_relaxed)).type); - } - - /// Returns the block identifier. - [[nodiscard]] u16 Block() const noexcept { - return ExtractBlock(std::bit_cast(data_raw.load(std::memory_order_relaxed))); + return ExtractType(raw.load(std::memory_order_relaxed)); } /// Returns the page pointer and attribute pair, extracted from the same atomic read - [[nodiscard]] std::tuple PointerTypeBlock(bool ignore_marked = false) const noexcept { - const auto non_atomic_raw = std::bit_cast(data_raw.load(std::memory_order_relaxed)); - return {ExtractPointer(non_atomic_raw, ignore_marked), static_cast(non_atomic_raw.type), ExtractBlock(non_atomic_raw)}; + [[nodiscard]] std::pair PointerType() const noexcept { + const uintptr_t non_atomic_raw = raw.load(std::memory_order_relaxed); + return {ExtractPointer(non_atomic_raw), ExtractType(non_atomic_raw)}; } - /// Write page info atomically - constexpr void Store(bool marked, PageType type, u16 block, uintptr_t pointer) noexcept { - data_raw.store(std::bit_cast(Data{marked, type, block, pointer})); + /// Returns the raw representation of the page information. + /// Use ExtractPointer and ExtractType to unpack the value. + [[nodiscard]] uintptr_t Raw() const noexcept { + return raw.load(std::memory_order_relaxed); } - constexpr void MarkRasterizerCached() noexcept { - data_raw.fetch_or(0b111); - } - - constexpr void MarkDebug(u64 ptr, u16 block) noexcept { - Store(true, PageType::DebugMemory, block, ptr); + /// Write a page pointer and type pair atomically + void Store(uintptr_t pointer, PageType type) noexcept { + raw.store(pointer | uintptr_t(type)); } /// Unpack a pointer from a page info raw representation - [[nodiscard]] static uintptr_t ExtractPointer(Data raw, bool ignore_marked = false) noexcept { - return raw.marked && !ignore_marked ? 0 - // shift raw.page's fake sign bit to the actual sign bit, then sign extend - : ((s64)(raw.page << (64 - 44))) >> (64 - 44 - 12); + [[nodiscard]] static uintptr_t ExtractPointer(uintptr_t raw) noexcept { + return raw & (~uintptr_t{0} << ATTRIBUTE_BITS); } - [[nodiscard]] static u16 ExtractBlock(Data raw) noexcept { - return static_cast(raw.block | (raw.block2 << 9)); + /// Unpack a page type from a page info raw representation + [[nodiscard]] static PageType ExtractType(uintptr_t raw) noexcept { + return static_cast(raw & ((uintptr_t{1} << ATTRIBUTE_BITS) - 1)); } private: - std::atomic data_raw; - static_assert(sizeof(Data) == sizeof(std::atomic)); + std::atomic raw; }; PageTable(); @@ -129,8 +100,13 @@ struct PageTable { PageTable(const PageTable&) = delete; PageTable& operator=(const PageTable&) = delete; - PageTable(PageTable&&) noexcept = delete; - PageTable& operator=(PageTable&&) noexcept = delete; + + PageTable(PageTable&&) noexcept = default; + PageTable& operator=(PageTable&&) noexcept = default; + + bool BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context, + Common::ProcessAddress address) const; + bool ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const; /** * Resizes the page table to be able to accommodate enough pages within @@ -145,14 +121,30 @@ struct PageTable { return current_address_space_width_in_bits; } + bool GetPhysicalAddress(Common::PhysicalAddress* out_phys_addr, + Common::ProcessAddress virt_addr) const { + if (virt_addr > (1ULL << this->GetAddressSpaceBits())) { + return false; + } + + *out_phys_addr = entries[virt_addr / page_size].addr + GetInteger(virt_addr); + return true; + } + /// Vector of memory pointers backing each page. An entry can only be non-null if the /// corresponding attribute element is of type `Memory`. - SparseLargeVector entries; - static_assert(sizeof(PageEntryData) == 8); + struct PageEntryData { + PageInfo ptr; + u64 block; + u64 addr; + u64 padding; + }; + VirtualBuffer entries; + static_assert(sizeof(PageEntryData) == 32); u8* fastmem_arena{}; std::size_t current_address_space_width_in_bits{}; - std::size_t current_page_bits{}; + std::size_t page_size{}; }; } // namespace Common diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp deleted file mode 100644 index ce6455a235..0000000000 --- a/src/common/sparse_large_vector.cpp +++ /dev/null @@ -1,143 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -/* virtual_buffer.cpp */ -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#ifdef _WIN32 -#include -#include -#else -#include -#endif - -#include "common/alignment.h" -#include "common/assert.h" -#include "common/sparse_large_vector.h" - -namespace Common { - -#ifdef _WIN32 -static std::vector> vector_regions {}; - -// Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error -static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { - DWORD code = info->ExceptionRecord->ExceptionCode; - u64 exception_addr = reinterpret_cast(info->ExceptionRecord->ExceptionAddress); - - if (code != EXCEPTION_ACCESS_VIOLATION) { - // Not our problem - return EXCEPTION_CONTINUE_SEARCH; - } - - u64 addr = 0, addr2 = 0; - - for (auto region: vector_regions) { - auto addr_shifted = exception_addr >> HostPageBits; - if (region.first <= addr_shifted && addr_shifted <= region.second) { - addr = addr_shifted; - } - - // Page-boundary accesses - if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ <= region.second) { - addr2 = addr_; - } - - if (addr != 0 || addr2 != 0) { - break; - } - } - - if (addr == 0 && addr2 == 0) { - // Not our problem - return EXCEPTION_CONTINUE_SEARCH; - } - - LOG_ERROR(HW_Memory, "Accessing an unallocated region of a SparseLargeVector at {:#x}; this shouldn't happen and is likely a Dynarmic error!", exception_addr); - - // Commit this region - if (addr != 0) { - if (!CommitVectorPage(addr << HostPageBits, false)) { - return EXCEPTION_CONTINUE_SEARCH; - } - } - // Commit next region if needed - if (addr2 != 0) { - if (!CommitVectorPage(addr2 << HostPageBits, false)) { - return EXCEPTION_CONTINUE_SEARCH; - } - } - - return EXCEPTION_CONTINUE_EXECUTION; -} - -bool CommitVectorPage(uintptr_t addr, bool write) noexcept { - MEMORY_BASIC_INFORMATION info {}; - auto res = VirtualQuery(reinterpret_cast(addr), &info, sizeof(info)); - if (res == 0) { - LOG_CRITICAL(HW_Memory, "Failed to query large buffer region at {:#x} with error {}, will try committing anyway", addr, GetLastError()); - } else if (info.State != MEM_RESERVE) { - LOG_ERROR(HW_Memory, "Tried to commit an unreserved large buffer region at {:#x} that is not mapped or is already committed (state {:#x})", addr, info.State); - return false; - } - - auto perm = write ? PAGE_READWRITE : PAGE_READONLY; - void* res2 = VirtualAlloc(reinterpret_cast(addr), HostPageSize, MEM_COMMIT, perm); - if (res2 == nullptr) { - LOG_ERROR(HW_Memory, "Failed to commit large buffer region at {:#x}, error {}", addr, GetLastError()); - return false; - } - - return true; -} -#endif - -#ifndef MAP_NOCORE -#define MAP_NOCORE 0 -#endif - -void* AllocateMemoryPages(std::size_t size) noexcept { - if (auto page = HostPageSize; size % page != 0) { - LOG_WARNING(HW_Memory, "Allocating unaligned large vector with size {:#x}; aligning to {} page size", size, page); - size = AlignUp(size, page); - } - -#ifdef _WIN32 - // We will never use this memory entirely so instead of committing it up front let's just reserve it and commit each page individually - void* base = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE); - - if (base != nullptr) { - vector_regions.emplace_back(reinterpret_cast(base), reinterpret_cast(base) + size); - - static std::once_flag flag; - std::call_once(flag, []() { AddVectoredExceptionHandler(1, FakePageFaultHandler); }); - } else { - // Try committing everything instead?? - LOG_WARNING(HW_Memory, "Failed to reserve large vector region with error {}, trying to commit instead..", GetLastError()); - base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE); - } - ASSERT_MSG(base, "Failed to reserve {:#x} sized region with error {}", size, GetLastError()); -#else - void* base = mmap(nullptr, size, PROT_READ, MAP_ANON | MAP_PRIVATE | MAP_NOCORE, -1, 0); - if (base == MAP_FAILED) - base = nullptr; - ASSERT_MSG(base, "Failed to allocate {:#x} sized region with error {}", size, strerror(errno)); -#endif - return base; -} - -void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { - if (auto page = HostPageSize; size % page != 0) { - size = AlignUp(size, page); - } - if (!base) - return; -#ifdef _WIN32 - ASSERT(VirtualFree(base, 0, MEM_RELEASE)); -#else - ASSERT(munmap(base, size) == 0); -#endif -} - -} // namespace Common diff --git a/src/common/sparse_large_vector.h b/src/common/sparse_large_vector.h deleted file mode 100644 index 827944fb08..0000000000 --- a/src/common/sparse_large_vector.h +++ /dev/null @@ -1,194 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -/* virtual_buffer.h */ -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#pragma once - -#include -#include -#include -#include - -#ifndef _WIN32 -#include -#include -#endif - -#include "common/alignment.h" -#include "common/assert.h" - -namespace Common { - -#ifdef _WIN32 -constexpr u64 HostPageSize = 0x1000; -constexpr u64 HostPageBits = 12; -constexpr u64 HostPageMask = ~(HostPageSize - 1); -bool CommitVectorPage(uintptr_t addr, bool write) noexcept; -#else -const u64 HostPageSize = sysconf(_SC_PAGESIZE); -const u64 HostPageBits = std::countr_zero(HostPageSize); -const u64 HostPageMask = ~(HostPageSize - 1); -#endif - -void* AllocateMemoryPages(std::size_t size) noexcept; -void FreeMemoryPages(void* base, std::size_t size) noexcept; - -/// A large page-aligned buffer that has optimized memory usage for zero-writes. -template -requires std::is_trivially_copyable_v -class SparseLargeVector final { -public: - constexpr SparseLargeVector() = default; - - explicit SparseLargeVector(std::size_t count) noexcept - : alloc_size{count * sizeof(T)} - { - base_ptr = static_cast(AllocateMemoryPages(alloc_size)); - - // each item in vector holds information for 64 pages - auto denom = HostPageSize * 64; - committed_pages = std::vector>((alloc_size + denom - 1) / denom); - } - - ~SparseLargeVector() noexcept { - FreeMemoryPages(base_ptr, alloc_size); - } - - SparseLargeVector(const SparseLargeVector&) = delete; - SparseLargeVector& operator=(const SparseLargeVector&) = delete; - SparseLargeVector(SparseLargeVector&& other) = delete; - SparseLargeVector& operator=(SparseLargeVector&& other) = delete; - - void ResizeAndClear(std::size_t count) noexcept { - if (auto const new_size = count * sizeof(T); new_size != alloc_size) { - FreeMemoryPages(base_ptr, alloc_size); - alloc_size = new_size; - base_ptr = static_cast(AllocateMemoryPages(alloc_size)); - - auto denom = HostPageSize * 64; - committed_pages = std::vector>((alloc_size + denom - 1) / denom); - } - } - - /// Returns a reference to the value of the requested index and allocates memory if needed. - T& GetAndFault(std::size_t index) noexcept { - if (index > alloc_size / sizeof(T)) { - UNREACHABLE_MSG("Out of bounds RW access on SparseLargeVector @ {}", index); - } - - if (!IsCommittedPage(index)) { - CommitPage(index); - } - return base_ptr[index]; - } - - /// Returns a reference to the value of the requested index if initialized, or will otherwise return a zero-initialized object. - const T& GetOrDefault(std::size_t index) const { -#ifdef _WIN32 - if (!IsCommittedPage(index)) { - return *reinterpret_cast(&default_val); - } -#endif - // On non-Windows, OS page table should optimize this by pointing to a zero page if unallocated. - return base_ptr[index]; - } - - void Set(std::size_t index, const T& value) noexcept { - if (index > alloc_size / sizeof(T)) { - LOG_CRITICAL(Common_Memory, "Out of bounds write on SparseLargeVector @ {}", index); - return; - } - if (!IsCommittedPage(index)) - CommitPage(index); - base_ptr[index] = value; - } - - void ZeroRegion(std::size_t start, std::size_t end_) noexcept { - u64 base = reinterpret_cast(&base_ptr[start]); - const u64 end = reinterpret_cast(&base_ptr[end_]); - - const u64 end_page = AlignUp(base, HostPageSize); - const u64 first_size = (std::min)(end_page, end) - base; - - if (IsCommittedPage(start / sizeof(T))) { - std::memset(reinterpret_cast(base), 0, first_size); - } - - if (end <= end_page) - return; - - base = end_page; - - for (u64 page = base; page < end; page += HostPageSize) { - if (!IsCommittedPage((page - reinterpret_cast(base_ptr)) / sizeof(T))) { - continue; - } - - std::memset(reinterpret_cast(page), 0, (std::min)( HostPageSize, end - page)); - } - } - - constexpr void CommitRegion(size_t index, size_t end_) { - const u64 base = static_cast(index) * sizeof(T); - const u64 end = static_cast(end_) * sizeof(T); - - for (u64 page = AlignDown(base, HostPageSize); page < end; page += HostPageSize) { - if (!IsCommittedPage(page / sizeof(T))) { - CommitPage(page / sizeof(T)); - } - } - } - - constexpr T& GetUnchecked(size_t index) { - return base_ptr[index]; - } - - [[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { - return GetOrDefault(index); - } - - [[nodiscard]] constexpr const T* data() const noexcept { - return base_ptr; - } - - [[nodiscard]] constexpr std::size_t size() const noexcept { - return alloc_size / sizeof(T); - } - -private: - [[nodiscard]] constexpr bool IsCommittedPage(std::size_t index) const noexcept { - if (index > alloc_size / sizeof(T)) { - LOG_CRITICAL(Common_Memory, "Out of bounds access on large vector @ {}", index); - return false; - } - - auto page = (index * sizeof(T)) >> HostPageBits; - auto val = committed_pages[page >> 6].load(std::memory_order_acquire); - return (val >> (page & 63)) & 1; - } - - constexpr void CommitPage(std::size_t index) noexcept { - auto page_index = (index * sizeof(T)) >> HostPageBits; - auto page = reinterpret_cast(base_ptr + index) & HostPageMask; -#if defined(_WIN32) - CommitVectorPage(page, true); -#else - mprotect(reinterpret_cast(page), HostPageSize, PROT_READ | PROT_WRITE); -#endif - - committed_pages[page_index >> 6].fetch_or(1ULL << (page_index & 63), std::memory_order_release); - } - - std::size_t alloc_size{}; - T* base_ptr{}; - - std::vector> committed_pages{}; -#ifdef _WIN32 - const std::array default_val{}; -#endif -}; - -} // namespace Common diff --git a/src/common/virtual_buffer.cpp b/src/common/virtual_buffer.cpp new file mode 100644 index 0000000000..8a4265e3e6 --- /dev/null +++ b/src/common/virtual_buffer.cpp @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#ifdef _WIN32 +#include +#else +#include +#endif + +#include "common/assert.h" +#include "common/virtual_buffer.h" + +namespace Common { + +void* AllocateMemoryPages(std::size_t size) noexcept { +#ifdef _WIN32 + void* base = VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + if (base == nullptr) { + // Probably failing to reserve is less likely than failing to commit + base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE); + } +#else + void* base = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); + if (base == MAP_FAILED) + base = nullptr; +#endif + ASSERT(base); + return base; +} + +void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { + if (!base) + return; +#ifdef _WIN32 + ASSERT(VirtualFree(base, 0, MEM_RELEASE)); +#else + ASSERT(munmap(base, size) == 0); +#endif +} + +} // namespace Common diff --git a/src/common/virtual_buffer.h b/src/common/virtual_buffer.h new file mode 100644 index 0000000000..d6386e2a4d --- /dev/null +++ b/src/common/virtual_buffer.h @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include + +namespace Common { + +void* AllocateMemoryPages(std::size_t size) noexcept; +void FreeMemoryPages(void* base, std::size_t size) noexcept; + +template +class VirtualBuffer final { +public: + // TODO: Uncomment this and change Common::PageTable::PageInfo to be trivially constructible + // using std::atomic_ref once libc++ has support for it + // static_assert( + // std::is_trivially_constructible_v, + // "T must be trivially constructible, as non-trivial constructors will not be executed " + // "with the current allocator"); + + constexpr VirtualBuffer() = default; + explicit VirtualBuffer(std::size_t count) noexcept + : alloc_size{count * sizeof(T)} + { + base_ptr = reinterpret_cast(AllocateMemoryPages(alloc_size)); + } + + ~VirtualBuffer() noexcept { + FreeMemoryPages(base_ptr, alloc_size); + } + + VirtualBuffer(const VirtualBuffer&) = delete; + VirtualBuffer& operator=(const VirtualBuffer&) = delete; + + VirtualBuffer(VirtualBuffer&& other) noexcept + : alloc_size{std::exchange(other.alloc_size, 0)} + , base_ptr{std::exchange(other.base_ptr, nullptr)} + {} + + VirtualBuffer& operator=(VirtualBuffer&& other) noexcept { + alloc_size = std::exchange(other.alloc_size, 0); + base_ptr = std::exchange(other.base_ptr, nullptr); + return *this; + } + + void resize(std::size_t count) noexcept { + if (auto const new_size = count * sizeof(T); new_size != alloc_size) { + FreeMemoryPages(base_ptr, alloc_size); + alloc_size = new_size; + base_ptr = reinterpret_cast(AllocateMemoryPages(alloc_size)); + } + } + + [[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { + return base_ptr[index]; + } + + [[nodiscard]] constexpr T& operator[](std::size_t index) noexcept { + return base_ptr[index]; + } + + [[nodiscard]] constexpr T* data() noexcept { + return base_ptr; + } + + [[nodiscard]] constexpr const T* data() const noexcept { + return base_ptr; + } + + [[nodiscard]] constexpr std::size_t size() const noexcept { + return alloc_size / sizeof(T); + } + +private: + std::size_t alloc_size{}; + T* base_ptr{}; +}; + +} // namespace Common diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.cpp b/src/core/arm/dynarmic/arm_dynarmic_32.cpp index 80e2b2dca9..97284efa4c 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_32.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_32.cpp @@ -172,12 +172,12 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) { if (page_table) { constexpr size_t PageBits = 12; constexpr size_t NumPageTableEntries = 1 << (32 - PageBits); + constexpr size_t PageLog2Stride = 5; + static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData)); - // Dynarmic will not write to the page table, const_cast is safe here - config.page_table = reinterpret_cast*>( - const_cast(page_table->entries.data())); - config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK; - config.page_table_marked_bit = 0; + config.page_table = reinterpret_cast*>(page_table->entries.data()); + config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS; + config.page_table_log2_stride = PageLog2Stride; config.absolute_offset_page_table = true; config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; config.only_detect_misalignment_via_page_table_on_page_boundary = true; @@ -188,13 +188,6 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) { config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt; config.recompile_on_exclusive_fastmem_failure = true; - - if (reinterpret_cast(m_system.DeviceMemory().buffer.BackingBasePointer() + - Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) { - // Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries, - // we have to manually sign extend when our actual pointer is negative. - config.page_table_sign_extension = Common::PageTable::SIGN_BIT; - } } // Multi-process state @@ -427,7 +420,6 @@ void ArmDynarmic32::SignalInterrupt(Kernel::KThread* thread) { } void ArmDynarmic32::ClearInstructionCache() { - m_cb->last_code_addr = u64(-1); m_jit->ClearCache(); } diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index 9662bf1712..0b7f6874a7 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -211,12 +211,13 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s // Memory if (page_table) { - // Dynarmic will not write to the page table, const_cast is safe here - config.page_table = reinterpret_cast( - const_cast(page_table->entries.data())); + constexpr size_t PageLog2Stride = 5; + static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData)); + + config.page_table = reinterpret_cast(page_table->entries.data()); config.page_table_address_space_bits = std::uint32_t(address_space_bits); - config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK; - config.page_table_marked_bit = 0; + config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS; + config.page_table_log2_stride = PageLog2Stride; config.silently_mirror_page_table = false; config.absolute_offset_page_table = true; config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; @@ -230,13 +231,6 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt; config.recompile_on_exclusive_fastmem_failure = true; - - if (reinterpret_cast(m_system.DeviceMemory().buffer.BackingBasePointer() + - Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) { - // Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries, - // we have to manually sign extend when our actual pointer is negative. - config.page_table_sign_extension = Common::PageTable::SIGN_BIT; - } } // Multi-process state @@ -453,7 +447,6 @@ void ArmDynarmic64::SignalInterrupt(Kernel::KThread* thread) { } void ArmDynarmic64::ClearInstructionCache() { - m_cb->last_code_addr = u64(-1); m_jit->ClearCache(); } diff --git a/src/core/device_memory.h b/src/core/device_memory.h index b5103e23bc..11bf0e3268 100644 --- a/src/core/device_memory.h +++ b/src/core/device_memory.h @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -29,11 +26,8 @@ public: template Common::PhysicalAddress GetPhysicalAddr(const T* ptr) const { - return GetPhysicalAddr(reinterpret_cast(ptr)); - } - - Common::PhysicalAddress GetPhysicalAddr(uintptr_t ptr) const { - return (ptr - reinterpret_cast(buffer.BackingBasePointer())) + + return (reinterpret_cast(ptr) - + reinterpret_cast(buffer.BackingBasePointer())) + DramMemoryMap::Base; } diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index b4b3b46088..3d97fdcc5c 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -18,7 +18,7 @@ #include "common/common_types.h" #include "common/range_mutex.h" #include "common/scratch_buffer.h" -#include "common/sparse_large_vector.h" +#include "common/virtual_buffer.h" namespace Core { @@ -178,8 +178,8 @@ private: u32 continuity_tracker; u32 compressed_physical_ptr; }; - Common::SparseLargeVector compressed_device_addr; - Common::SparseLargeVector tracked_entries; + Common::VirtualBuffer compressed_device_addr; + Common::VirtualBuffer tracked_entries; // Process memory interfaces @@ -200,8 +200,8 @@ private: return std::make_pair(asid, address); } - constexpr void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) { - tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit); + void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) { + tracked_entries[page_index].cpu_backing_address = address | (asid.id << asid_start_bit); } std::array t_slot{}; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 97c2706abb..9866f3a4f7 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -177,6 +177,17 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo { impl = std::make_unique>(); cached_pages = std::make_unique(); + + const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS; + for (size_t i = 0; i < total_virtual; i++) { + tracked_entries[i].compressed_physical_ptr = 0; + tracked_entries[i].continuity_tracker = 1; + tracked_entries[i].cpu_backing_address = 0; + } + const size_t total_phys = 1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS); + for (size_t i = 0; i < total_phys; i++) { + compressed_device_addr[i] = 0; + } } template @@ -209,28 +220,26 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; std::scoped_lock lk(mapping_guard); - - tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); for (size_t i = 0; i < num_pages; i++) { const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE; auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress)); if (ptr == nullptr) [[unlikely]] { - tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = 0; + tracked_entries[start_page_d + i].compressed_physical_ptr = 0; continue; } auto phys_addr = static_cast(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; - tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = phys_addr; + tracked_entries[start_page_d + i].compressed_physical_ptr = phys_addr; InsertCPUBacking(start_page_d + i, new_vaddress, asid); const u32 base_dev = compressed_device_addr[phys_addr - 1U]; const u32 new_dev = static_cast(start_page_d + i); if (base_dev == 0) [[likely]] { - compressed_device_addr.GetAndFault(phys_addr - 1U) = new_dev; + compressed_device_addr[phys_addr - 1U] = new_dev; continue; } u32 start_id = base_dev & MULTI_MASK; if ((base_dev >> MULTI_FLAG_BITS) == 0) { start_id = impl->multi_dev_address.Register(base_dev); - compressed_device_addr.GetAndFault(phys_addr - 1U) = MULTI_FLAG | start_id; + compressed_device_addr[phys_addr - 1U] = MULTI_FLAG | start_id; } impl->multi_dev_address.Register(new_dev, start_id); } @@ -246,26 +255,24 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; device_inter->InvalidateRegion(address, size); std::scoped_lock lk(mapping_guard); - - tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); // should already be committed, but just in case for (size_t i = 0; i < num_pages; i++) { - auto& entry = tracked_entries.GetUnchecked(start_page_d + i); - auto phys_addr = entry.compressed_physical_ptr; - entry.compressed_physical_ptr = 0; - entry.cpu_backing_address = 0; + auto phys_addr = tracked_entries[start_page_d + i].compressed_physical_ptr; + tracked_entries[start_page_d + i].compressed_physical_ptr = 0; + tracked_entries[start_page_d + i].cpu_backing_address = 0; if (phys_addr != 0) [[likely]] { - u32& base_dev = compressed_device_addr.GetAndFault(phys_addr - 1U); + const u32 base_dev = compressed_device_addr[phys_addr - 1U]; if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] { - base_dev = 0; + compressed_device_addr[phys_addr - 1] = 0; continue; } const auto [more_entries, new_start] = impl->multi_dev_address.Unregister( static_cast(start_page_d + i), base_dev & MULTI_MASK); if (!more_entries) { - base_dev = impl->multi_dev_address.ReleaseEntry(new_start); + compressed_device_addr[phys_addr - 1] = + impl->multi_dev_address.ReleaseEntry(new_start); continue; } - base_dev = new_start | MULTI_FLAG; + compressed_device_addr[phys_addr - 1] = new_start | MULTI_FLAG; } } t_slot = {}; @@ -278,8 +285,6 @@ void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtu size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; uintptr_t last_ptr = 0; size_t page_count = 1; - - tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); for (size_t i = num_pages; i > 0; i--) { size_t index = i - 1; const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE; @@ -291,14 +296,14 @@ void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtu page_count = 1; } last_ptr = new_ptr; - tracked_entries.GetUnchecked(start_page_d + index).continuity_tracker = static_cast(page_count) - 1; + tracked_entries[start_page_d + index].continuity_tracker = static_cast(page_count); } } template u8* DeviceMemoryManager::GetSpan(const DAddr src_addr, const std::size_t size) { size_t page_index = src_addr >> page_bits; size_t subbits = src_addr & page_mask; - if ((static_cast(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) { + if ((static_cast(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) { return GetPointer(src_addr); } return nullptr; @@ -308,7 +313,7 @@ template const u8* DeviceMemoryManager::GetSpan(const DAddr src_addr, const std::size_t size) const { size_t page_index = src_addr >> page_bits; size_t subbits = src_addr & page_mask; - if ((static_cast(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) { + if ((static_cast(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) { return GetPointer(src_addr); } return nullptr; @@ -378,7 +383,7 @@ void DeviceMemoryManager::WalkBlock(DAddr addr, std::size_t size, auto o std::size_t page_index = addr >> Memory::YUZU_PAGEBITS; std::size_t page_offset = addr & Memory::YUZU_PAGEMASK; while (remaining_size) { - const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker+1); + const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker); const std::size_t copy_amount = (std::min)((next_pages << Memory::YUZU_PAGEBITS) - page_offset, remaining_size); const auto current_vaddr = u64((page_index << Memory::YUZU_PAGEBITS) + page_offset); SCOPE_EXIT{ diff --git a/src/core/hle/kernel/k_page_table_base.cpp b/src/core/hle/kernel/k_page_table_base.cpp index 2a0a7d49d5..a727355a9f 100644 --- a/src/core/hle/kernel/k_page_table_base.cpp +++ b/src/core/hle/kernel/k_page_table_base.cpp @@ -635,36 +635,6 @@ Result KPageTableBase::CheckMemoryState(const KMemoryInfo& info, KMemoryState st R_SUCCEED(); } -bool KPageTableBase::BeginTraversal(const Common::PageTable &impl, TraversalEntry *out_entry, TraversalContext *out_context, - Common::ProcessAddress address) const { - out_context->next_offset = GetInteger(address); - out_context->next_page = GetInteger(address) >> PageBits; - - return ContinueTraversal(impl, out_entry, out_context); -} - -bool KPageTableBase::ContinueTraversal(const Common::PageTable &impl, TraversalEntry *out_entry, - TraversalContext *context) const { - // Setup invalid defaults. - out_entry->phys_addr = 0; - out_entry->block_size = PageSize; - // Validate that we can read the actual entry. - if (auto const page = context->next_page; page < impl.entries.size()) { - // Validate that the entry is mapped. - if (auto const paddr = impl.entries[page].Pointer(true); paddr != 0) { - // Populate the results and return true - out_entry->phys_addr = GetInteger(m_system.DeviceMemory().GetPhysicalAddr(paddr + context->next_offset)); - context->next_page += 1; - context->next_offset += PageSize; - return true; - } - } - context->next_page += 1; - context->next_offset += PageSize; - // Otherwise return false - return false; -} - Result KPageTableBase::CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, @@ -970,7 +940,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a size_t tot_size = 0; next_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), region_start); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), region_start); next_entry.block_size = (next_entry.block_size - (GetInteger(region_start) & (next_entry.block_size - 1))); @@ -1006,7 +976,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a break; } - next_valid = ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + next_valid = impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); } // Check the last entry. @@ -1784,7 +1754,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_ // Begin traversal. TraversalContext context; TraversalEntry next_entry; - R_UNLESS(BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr), + R_UNLESS(impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr), ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -1794,7 +1764,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_ // Iterate, adding to group as we go. while (tot_size < size) { - R_UNLESS(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)), + R_UNLESS(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)), ResultInvalidCurrentMemory); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -1858,7 +1828,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr // Begin traversal. TraversalContext context; TraversalEntry next_entry; - if (!BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr)) { + if (!impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr)) { return false; } @@ -1869,7 +1839,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr // Iterate, comparing expected to actual. while (tot_size < size) { - if (!ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context))) { + if (!impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context))) { return false; } @@ -1926,7 +1896,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState( // Begin a traversal. TraversalContext context; TraversalEntry cur_entry = {.phys_addr = 0, .block_size = 0}; - R_UNLESS(BeginTraversal(impl, std::addressof(cur_entry), std::addressof(context), address), + R_UNLESS(impl.BeginTraversal(std::addressof(cur_entry), std::addressof(context), address), ResultInvalidCurrentMemory); // Traverse until we have enough size or we aren't contiguous any more. @@ -1935,7 +1905,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState( for (contig_size = cur_entry.block_size - (GetInteger(phys_address) & (cur_entry.block_size - 1)); contig_size < size; contig_size += cur_entry.block_size) { - if (!ContinueTraversal(impl, std::addressof(cur_entry), std::addressof(context))) { + if (!impl.ContinueTraversal(std::addressof(cur_entry), std::addressof(context))) { break; } if (cur_entry.phys_addr != phys_address + contig_size) { @@ -2364,7 +2334,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out, TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(m_impl, std::addressof(next_entry), std::addressof(context), virt_addr); + m_impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Set tracking variables. @@ -2375,7 +2345,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out, while (true) { // Continue the traversal. traverse_valid = - ContinueTraversal(m_impl, std::addressof(next_entry), std::addressof(context)); + m_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); if (!traverse_valid) { break; } @@ -2597,7 +2567,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre TraversalContext context; TraversalEntry next_entry; ASSERT( - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address)); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address)); // Check that the physical region matches. R_UNLESS(next_entry.phys_addr == phys_addr, ResultInvalidMemoryRegion); @@ -2607,7 +2577,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre next_entry.block_size - (GetInteger(phys_addr) & (next_entry.block_size - 1)); checked_size < size; checked_size += next_entry.block_size) { // Continue the traversal. - ASSERT(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context))); + ASSERT(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context))); // Check that the physical region matches. R_UNLESS(next_entry.phys_addr == phys_addr + checked_size, ResultInvalidMemoryRegion); @@ -3059,7 +3029,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_ TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), address); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), address); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -3071,7 +3041,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_ while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3159,7 +3129,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_address); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_address); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -3197,7 +3167,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3255,7 +3225,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -3297,7 +3267,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3758,7 +3728,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser( TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -3798,7 +3768,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser( while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3852,7 +3822,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel( TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -3875,7 +3845,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel( while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3932,7 +3902,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear( TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -3971,7 +3941,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear( while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -4027,7 +3997,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -4050,7 +4020,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -4119,10 +4089,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap( bool traverse_valid; // Begin traversal. - traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry), + traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry), std::addressof(src_context), src_addr); ASSERT(traverse_valid); - traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry), + traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry), std::addressof(dst_context), dst_addr); ASSERT(traverse_valid); @@ -4157,7 +4127,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap( if (ofs + cur_copy_size != size) { if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) { // Continue the src traversal. - traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry), + traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry), std::addressof(src_context)); ASSERT(traverse_valid); @@ -4168,7 +4138,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap( if (cur_dst_addr + cur_min_size == dst_next_entry.phys_addr + dst_next_entry.block_size) { // Continue the dst traversal. - traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry), + traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry), std::addressof(dst_context)); ASSERT(traverse_valid); @@ -4253,10 +4223,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination( bool traverse_valid; // Begin traversal. - traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry), + traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry), std::addressof(src_context), src_addr); ASSERT(traverse_valid); - traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry), + traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry), std::addressof(dst_context), dst_addr); ASSERT(traverse_valid); @@ -4291,7 +4261,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination( if (ofs + cur_copy_size != size) { if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) { // Continue the src traversal. - traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry), + traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry), std::addressof(src_context)); ASSERT(traverse_valid); @@ -4302,7 +4272,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination( if (cur_dst_addr + cur_min_size == dst_next_entry.phys_addr + dst_next_entry.block_size) { // Continue the dst traversal. - traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry), + traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry), std::addressof(dst_context)); ASSERT(traverse_valid); @@ -4577,7 +4547,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, // Begin traversal. TraversalContext context; TraversalEntry next_entry; - bool traverse_valid = BeginTraversal(src_impl, std::addressof(next_entry), + bool traverse_valid = src_impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), aligned_src_start); ASSERT(traverse_valid); @@ -4627,7 +4597,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, // If the block's size was one page, we may need to continue traversal. if (cur_block_size == 0 && aligned_src_size > PageSize) { traverse_valid = - ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context)); + src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); cur_block_addr = next_entry.phys_addr; @@ -4640,7 +4610,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, while (aligned_src_start + tot_block_size < mapping_src_end) { // Continue the traversal. traverse_valid = - ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context)); + src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); // Process the block. @@ -4683,7 +4653,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, if (mapped_block_end + cur_block_size < aligned_src_end && cur_block_size == last_block_size) { traverse_valid = - ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context)); + src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); cur_block_addr = next_entry.phys_addr; @@ -5631,7 +5601,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si ContiguousRangeInfo(KPageTableBase& pt, KProcessAddress address, size_t size) : m_pt(pt), m_remaining_size(size) { // Begin a traversal. - ASSERT(m_pt.BeginTraversal(m_pt.GetImpl(), std::addressof(m_entry), + ASSERT(m_pt.GetImpl().BeginTraversal(std::addressof(m_entry), std::addressof(m_context), address)); // Setup tracking fields. @@ -5662,7 +5632,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si void DetermineContiguousBlockExtents() { // Continue traversing until we're not contiguous, or we have enough. while (m_cur_size < m_remaining_size) { - ASSERT(m_pt.ContinueTraversal(m_pt.GetImpl(), std::addressof(m_entry), + ASSERT(m_pt.GetImpl().ContinueTraversal(std::addressof(m_entry), std::addressof(m_context))); // If we're not contiguous, we're done. diff --git a/src/core/hle/kernel/k_page_table_base.h b/src/core/hle/kernel/k_page_table_base.h index 0c222910fc..1d4fa85f04 100644 --- a/src/core/hle/kernel/k_page_table_base.h +++ b/src/core/hle/kernel/k_page_table_base.h @@ -370,10 +370,6 @@ private: size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const; - bool BeginTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* out_context, - Common::ProcessAddress address) const; - bool ContinueTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* context) const; - Result CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, @@ -478,14 +474,7 @@ private: // Validate pre-conditions. ASSERT(this->IsLockedByCurrentThread()); - if (virt_addr > (1ULL << m_address_space_width)) { - return false; - } - - *out = m_system.DeviceMemory().GetPhysicalAddr( - this->GetImpl().entries[GetInteger(virt_addr) >> PageBits].Pointer(true) + GetInteger(virt_addr)); - - return true; + return this->GetImpl().GetPhysicalAddress(out, virt_addr); } public: diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 837016de1d..a5e6b8592c 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -101,10 +101,8 @@ struct Memory::Impl { } u64 protect_bytes = 0, protect_begin = 0; - - current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr + size) >> YUZU_PAGEBITS); for (u64 addr = vaddr; addr < vaddr + size; addr += YUZU_PAGESIZE) { - const Common::PageType page_type = current_page_table->entries.GetUnchecked(addr >> YUZU_PAGEBITS).Type(); + const Common::PageType page_type = current_page_table->entries[addr >> YUZU_PAGEBITS].ptr.Type(); switch (page_type) { case Common::PageType::RasterizerCachedMemory: if (protect_bytes > 0) { @@ -125,14 +123,16 @@ struct Memory::Impl { } [[nodiscard]] u8* GetPointerFromRasterizerCachedMemory(u64 vaddr) const { - if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr) - return reinterpret_cast(paddr) + vaddr; + Common::PhysicalAddress const paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr; + if (paddr) + return system.DeviceMemory().GetPointer(paddr + vaddr); return {}; } [[nodiscard]] u8* GetPointerFromDebugMemory(u64 vaddr) const { - if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr) - return reinterpret_cast(paddr) + vaddr; + const Common::PhysicalAddress paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr; + if (paddr != 0) + return system.DeviceMemory().GetPointer(paddr + vaddr); return {}; } @@ -243,12 +243,10 @@ struct Memory::Impl { std::size_t page_index = addr >> YUZU_PAGEBITS; std::size_t page_offset = addr & YUZU_PAGEMASK; bool user_accessible = true; - - current_page_table->entries.CommitRegion(page_index, page_index + (size >> YUZU_PAGEBITS) + 1); while (remaining_size != 0) { const std::size_t copy_amount = (std::min)(std::size_t(YUZU_PAGESIZE) - page_offset, remaining_size); const auto current_vaddr = u64((page_index << YUZU_PAGEBITS) + page_offset); - const auto [pointer, type, _] = current_page_table->entries.GetUnchecked(page_index).PointerTypeBlock(); + const auto [pointer, type] = current_page_table->entries[page_index].ptr.PointerType(); switch (type) { case Common::PageType::Unmapped: { user_accessible = false; @@ -299,10 +297,10 @@ struct Memory::Impl { } [[nodiscard]] inline const u8* GetSpan(const VAddr addr, const std::size_t size) const noexcept { - return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr; + return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr; } [[nodiscard]] inline u8* GetSpan(const VAddr addr, const std::size_t size) noexcept { - return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr; + return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr; } bool WriteBlockImpl(const Common::ProcessAddress addr, const void* buffer, const std::size_t size, bool unsafe) { @@ -406,14 +404,11 @@ struct Memory::Impl { // The region is at a granularity of CPU pages. const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1; - - current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages); for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) { - auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS); - const auto [pointer, type, block] = entry.PointerTypeBlock(true); + const Common::PageType page_type = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type(); if (debug) { // Switch page type to debug if now debug - switch (type) { + switch (page_type) { case Common::PageType::Unmapped: ASSERT(false && "Attempted to mark unmapped pages as debug"); break; @@ -422,14 +417,14 @@ struct Memory::Impl { // Page is already marked. break; case Common::PageType::Memory: - entry.MarkDebug(pointer, block); + current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::DebugMemory); break; default: UNREACHABLE(); } } else { // Switch page type to non-debug if now non-debug - switch (type) { + switch (page_type) { case Common::PageType::Unmapped: ASSERT(false && "Attempted to mark unmapped pages as non-debug"); break; @@ -438,7 +433,8 @@ struct Memory::Impl { // Don't mess with already non-debug or rasterizer memory. break; case Common::PageType::DebugMemory: { - entry.Store(false, Common::PageType::Memory, block, pointer); + u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK); + current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); break; } default: @@ -470,10 +466,8 @@ struct Memory::Impl { // is different). This assumes the specified GPU address region is contiguous as well. const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1; - current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages); for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) { - auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS); - const Common::PageType page_type = entry.Type(); + const Common::PageType page_type= current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type(); if (cached) { // Switch page type to cached if now cached switch (page_type) { @@ -483,7 +477,7 @@ struct Memory::Impl { break; case Common::PageType::DebugMemory: case Common::PageType::Memory: - entry.MarkRasterizerCached(); + current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::RasterizerCachedMemory); break; case Common::PageType::RasterizerCachedMemory: // There can be more than one GPU region mapped per CPU region, so it's common @@ -505,13 +499,13 @@ struct Memory::Impl { // that this area is already unmarked as cached. break; case Common::PageType::RasterizerCachedMemory: { - if (auto [ptr, _, block] = entry.PointerTypeBlock(true); ptr == 0) { + if (u8* const pointer = GetPointerFromRasterizerCachedMemory(vaddr & ~YUZU_PAGEMASK); pointer == nullptr) { // It's possible that this function has been called while updating the // pagetable after unmapping a VMA. In that case the underlying VMA will no // longer exist, and we should just leave the pagetable entry blank. - entry.Store(false, Common::PageType::Unmapped, block, 0); + current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::Unmapped); } else { - entry.Store(false, Common::PageType::Memory, block, ptr); + current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); } break; } @@ -545,18 +539,22 @@ struct Memory::Impl { ASSERT_MSG(type != Common::PageType::Memory, "Mapping memory page without a pointer @ {:016x}", base * YUZU_PAGESIZE); - page_table.entries.ZeroRegion(base, end); - } else { - auto current_block = block_count.fetch_add(1, std::memory_order_relaxed); - ASSERT(current_block != 65535); - - page_table.entries.CommitRegion(base, end); while (base != end) { - auto host_ptr = reinterpret_cast(system.DeviceMemory().GetPointer(target)) - (base << YUZU_PAGEBITS);; - auto& entry = page_table.entries.GetUnchecked(base); + page_table.entries[base].ptr.Store(0, type); + page_table.entries[base].addr = 0; + page_table.entries[base].block = 0; + base += 1; + } + } else { + auto orig_base = base; + while (base != end) { + auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer(target)) - (base << YUZU_PAGEBITS); + auto backing = GetInteger(target) - (base << YUZU_PAGEBITS); + page_table.entries[base].ptr.Store(host_ptr, type); + page_table.entries[base].addr = backing; + page_table.entries[base].block = orig_base << YUZU_PAGEBITS; - entry.Store(false, type, current_block, host_ptr); - ASSERT_MSG(page_table.entries[base].Pointer(), + ASSERT_MSG(page_table.entries[base].ptr.Pointer(), "memory mapping base yield a nullptr within the table"); base += 1; @@ -571,11 +569,11 @@ struct Memory::Impl { vaddr &= 0xffffffffffffULL; if (AddressSpaceContains(*current_page_table, vaddr, 1)) [[likely]] { // Avoid adding any extra logic to this fast-path block - const auto raw = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Raw(); - if (auto pointer = Common::PageTable::PageEntryData::ExtractPointer(raw); pointer) [[likely]] { + const uintptr_t raw_pointer = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Raw(); + if (const uintptr_t pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) [[likely]] { return reinterpret_cast(pointer + vaddr); } else { - switch (static_cast(raw.type)) { + switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) { case Common::PageType::Memory: ASSERT_MSG(false, "Mapped memory page without a pointer @ {:#016x}", vaddr); return nullptr; @@ -775,7 +773,6 @@ struct Memory::Impl { #else Common::HostMemory* host_buffer{}; #endif - std::atomic block_count = 0; }; Memory::Memory(Core::System& system_) : system{system_} { @@ -814,7 +811,7 @@ bool Memory::IsValidVirtualAddress(const Common::ProcessAddress vaddr) const { if (page >= page_table.entries.size()) { return false; } - const auto [pointer, type, _] = page_table.entries[page].PointerTypeBlock(); + const auto [pointer, type] = page_table.entries[page].ptr.PointerType(); return pointer != 0 || type == Common::PageType::RasterizerCachedMemory || type == Common::PageType::DebugMemory; } diff --git a/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp b/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp index 8b6ec2506e..da51220e9c 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp @@ -371,10 +371,8 @@ EmitConfig A32AddressSpace::GetEmitConfig() { .page_table_pointer = std::bit_cast(conf.page_table), .page_table_address_space_bits = 32, - .page_table_pointer_mask = conf.page_table_pointer_mask, + .page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits, .page_table_log2_stride = conf.page_table_log2_stride, - .page_table_marked_bit = conf.page_table_marked_bit, - .page_table_sign_extension = conf.page_table_sign_extension, .silently_mirror_page_table = true, .absolute_offset_page_table = conf.absolute_offset_page_table, .detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table, diff --git a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp index 2dbb7d2e1d..2c71ffe282 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp @@ -545,10 +545,8 @@ EmitConfig A64AddressSpace::GetEmitConfig() { .page_table_pointer = std::bit_cast(conf.page_table), .page_table_address_space_bits = conf.page_table_address_space_bits, - .page_table_pointer_mask = conf.page_table_pointer_mask, + .page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits, .page_table_log2_stride = conf.page_table_log2_stride, - .page_table_marked_bit = conf.page_table_marked_bit, - .page_table_sign_extension = conf.page_table_sign_extension, .silently_mirror_page_table = conf.silently_mirror_page_table, .absolute_offset_page_table = conf.absolute_offset_page_table, .detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table, diff --git a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h index 693f71ea39..f2c89bbfc0 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h +++ b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h @@ -128,10 +128,8 @@ struct EmitConfig { // Page table u64 page_table_pointer; std::size_t page_table_address_space_bits; - u64 page_table_pointer_mask; + int page_table_pointer_mask_bits; std::size_t page_table_log2_stride; - std::optional page_table_marked_bit; - std::optional page_table_sign_extension; bool silently_mirror_page_table; bool absolute_offset_page_table; u8 detect_misaligned_access_via_page_table; diff --git a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp index 170fcf9613..143244d60b 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp @@ -273,18 +273,9 @@ std::pair InlinePageTableEmitVAddrLookup(oaknut::Cod // load x0 = *<(u8*)pagetable + index> code.LDR(Xscratch0, Xpagetable, Xscratch0); - if (ctx.conf.page_table_marked_bit) { - code.TST(Xscratch0, 1ULL << *ctx.conf.page_table_marked_bit); - code.B(NE, *fallback); - } - - if (ctx.conf.page_table_pointer_mask != 0) { - code.AND(Xscratch0, Xscratch0, ctx.conf.page_table_pointer_mask); - } - - // TODO: combine this with page_table_pointer_mask - if (ctx.conf.page_table_sign_extension) { - code.SBFM(Xscratch0, Xscratch0, 0, *ctx.conf.page_table_sign_extension); + if (ctx.conf.page_table_pointer_mask_bits != 0) { + const u64 mask = u64(~u64(0)) << ctx.conf.page_table_pointer_mask_bits; + code.AND(Xscratch0, Xscratch0, mask); } code.CBZ(Xscratch0, *fallback); diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h index 9686e4fc04..3ac078f1d7 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h +++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h @@ -9,7 +9,6 @@ #pragma once #include -#include #include "dynarmic/backend/x64/xbyak.h" #include "dynarmic/backend/x64/a32_emit_x64.h" @@ -79,40 +78,27 @@ Xbyak::RegExp EmitVAddrLookup(BlockOfCode& code, EmitContext& ctx, size_t bitsiz template<> [[maybe_unused]] Xbyak::RegExp EmitVAddrLookup(BlockOfCode& code, A32EmitContext& ctx, size_t bitsize, Xbyak::Label& abort, Xbyak::Reg64 vaddr) { const Xbyak::Reg64 page = ctx.reg_alloc.ScratchGpr(code); - const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table && ctx.conf.page_table_pointer_mask == 0 ? page : ctx.reg_alloc.ScratchGpr(code); + const Xbyak::Reg32 tmp = ctx.conf.absolute_offset_page_table ? page.cvt32() : ctx.reg_alloc.ScratchGpr(code).cvt32(); - EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp); + EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp.cvt64()); - code.mov(tmp, vaddr); + // TODO: This code assumes vaddr has been zext from 32-bits to 64-bits. + + code.mov(tmp, vaddr.cvt32()); code.shr(tmp, int(page_table_const_bits)); code.shl(tmp, int(ctx.conf.page_table_log2_stride)); code.mov(page, qword[r14 + tmp.cvt64()]); - - // check for marked bit, use as unmapped if marked - if (ctx.conf.page_table_marked_bit) { - code.bt(page, *ctx.conf.page_table_marked_bit); - code.jc(abort, code.T_NEAR); - } - // mask away attributes - if (ctx.conf.page_table_pointer_mask == 0) { + if (ctx.conf.page_table_pointer_mask_bits == 0) { code.test(page, page); - } else if (std::in_range(ctx.conf.page_table_pointer_mask)) { - code.and_(page, ctx.conf.page_table_pointer_mask); } else { - code.mov(tmp, ctx.conf.page_table_pointer_mask); - code.and_(page, tmp); + code.and_(page, ~u32(0) << ctx.conf.page_table_pointer_mask_bits); } - if (ctx.conf.page_table_sign_extension) { - code.shl(page, *ctx.conf.page_table_sign_extension); - code.sar(page, *ctx.conf.page_table_sign_extension); - } - code.jz(abort, code.T_NEAR); if (ctx.conf.absolute_offset_page_table) { return page + vaddr; } - code.mov(tmp, vaddr); - code.and_(tmp, u32(page_table_const_mask)); + code.mov(tmp, vaddr.cvt32()); + code.and_(tmp, static_cast(page_table_const_mask)); return page + tmp.cvt64(); } @@ -122,7 +108,7 @@ template<> const size_t unused_top_bits = 64 - ctx.conf.page_table_address_space_bits; const Xbyak::Reg64 page = ctx.reg_alloc.ScratchGpr(code); - const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table && ctx.conf.page_table_pointer_mask == 0 ? page : ctx.reg_alloc.ScratchGpr(code); + const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table ? page : ctx.reg_alloc.ScratchGpr(code); EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp); @@ -157,26 +143,11 @@ template<> code.shl(tmp, int(ctx.conf.page_table_log2_stride)); code.mov(page, qword[r14 + tmp]); - - // check for marked bit, use as unmapped if marked - if (ctx.conf.page_table_marked_bit) { - code.bt(page, *ctx.conf.page_table_marked_bit); - code.jc(abort, code.T_NEAR); - } - // mask away attributes - if (ctx.conf.page_table_pointer_mask == 0) { + if (ctx.conf.page_table_pointer_mask_bits == 0) { code.test(page, page); - } else if (std::in_range(ctx.conf.page_table_pointer_mask)) { - code.and_(page, ctx.conf.page_table_pointer_mask); } else { - code.mov(tmp, ctx.conf.page_table_pointer_mask); - code.and_(page, tmp); + code.and_(page, ~u32(0) << ctx.conf.page_table_pointer_mask_bits); } - if (ctx.conf.page_table_sign_extension) { - code.shl(page, *ctx.conf.page_table_sign_extension); - code.sar(page, *ctx.conf.page_table_sign_extension); - } - code.jz(abort, code.T_NEAR); if (ctx.conf.absolute_offset_page_table) { return page + vaddr; diff --git a/src/dynarmic/src/dynarmic/interface/A32/config.h b/src/dynarmic/src/dynarmic/interface/A32/config.h index b93b116164..5a97fb69f3 100644 --- a/src/dynarmic/src/dynarmic/interface/A32/config.h +++ b/src/dynarmic/src/dynarmic/interface/A32/config.h @@ -159,23 +159,14 @@ struct UserConfig { /// Maximum size is limited by the maximum length of a x86_64 / arm64 jump. std::uint32_t code_cache_size = 128 * 1024 * 1024; // bytes - /// Applies a bit mask to the bits in host pointers from the page table. + /// Masks out the first N bits in host pointers from the page table. /// The intention behind this is to allow users of Dynarmic to pack attributes in the /// same integer and update the pointer attribute pair atomically. - /// If the configured value is ~(0b111ULL), all pointers will be forcefully aligned to 8 bytes. - std::uint64_t page_table_pointer_mask = 0; + /// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes. + std::int32_t page_table_pointer_mask_bits = 0; - /// Log2 of the size per page entry, value should be either 3 or 4 - std::uint32_t page_table_log2_stride = 3; - - /// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table. - /// If the bit is set to 1, Dynarmic will treat it as unmapped. - /// This bit should be included as part of `page_table_pointer_mask_bits`. - std::optional page_table_marked_bit = std::nullopt; - - /// If this value is set, Dynarmic will sign extend the page table pointer by this bit. - /// Useful for compacting bits into the page table and should be used as part of `page_table_pointer_mask`. - std::optional page_table_sign_extension = std::nullopt; + // Log2 of the size per page entry, value should be either 3 or 4 + std::size_t page_table_log2_stride = 3; /// Select the architecture version to use. /// There are minor behavioural differences between versions. diff --git a/src/dynarmic/src/dynarmic/interface/A64/config.h b/src/dynarmic/src/dynarmic/interface/A64/config.h index 818aed73e7..83c1593fd8 100644 --- a/src/dynarmic/src/dynarmic/interface/A64/config.h +++ b/src/dynarmic/src/dynarmic/interface/A64/config.h @@ -173,23 +173,14 @@ struct UserConfig { /// This is only used if page_table is not nullptr. std::uint32_t page_table_address_space_bits = 36; - /// Applies a bit mask to the bits in host pointers from the page table. + /// Masks out the first N bits in host pointers from the page table. /// The intention behind this is to allow users of Dynarmic to pack attributes in the /// same integer and update the pointer attribute pair atomically. - /// If the configured value is ~(0b111ULL), all pointers will be forcefully aligned to 8 bytes. - std::uint64_t page_table_pointer_mask = 0; + /// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes. + std::int32_t page_table_pointer_mask_bits = 0; - /// Log2 of the size per page entry, value should be either 3 or 4 - std::uint32_t page_table_log2_stride = 3; - - /// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table. - /// If the bit is set to 1, Dynarmic will treat it as unmapped. - /// This bit should be included as part of `page_table_pointer_mask`. - std::optional page_table_marked_bit = std::nullopt; - - /// If this value is set, Dynarmic will sign extend the page table pointer by this bit. - /// Useful for compacting bits into the page table and should be used as part of `page_table_pointer_mask`. - std::optional page_table_sign_extension = std::nullopt; + // Log2 of the size per page entry, value should be either 3 or 4 + std::size_t page_table_log2_stride = 3; /// Counter-timer frequency register. The value of the register is not interpreted by /// dynarmic. diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index d5838a387d..b9b136f6b8 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -46,8 +46,8 @@ MemoryManager::MemoryManager(Core::System& system_, MaxwellDeviceMemoryManager& page_table_mask = page_table_size - 1; big_page_table_mask = big_page_table_size - 1; - big_page_table_dev.ResizeAndClear(big_page_table_size); big_entries.resize(big_page_table_size / 32, 0); + big_page_table_dev.resize(big_page_table_size); big_page_continuous.resize(big_page_table_size / continuous_bits, 0); entries.resize(page_table_size / 32, 0); } @@ -143,7 +143,7 @@ GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr const DAddr current_dev_addr = dev_addr + offset; const auto index = PageEntryIndex(current_gpu_addr, true); const u32 sub_value = static_cast(current_dev_addr >> cpu_page_bits); - big_page_table_dev.Set(index, sub_value); + big_page_table_dev[index] = sub_value; const bool is_continuous = ([&] { uintptr_t base_ptr{ reinterpret_cast(memory.GetPointer(current_dev_addr))}; diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index 08978567d2..f9fddd177a 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -17,7 +17,7 @@ #include "common/multi_level_page_table.h" #include "common/range_map.h" #include "common/scratch_buffer.h" -#include "common/sparse_large_vector.h" +#include "common/virtual_buffer.h" #include "video_core/invalidation_accumulator.h" #include "video_core/cache_types.h" #include "video_core/host1x/gpu_device_memory_manager.h" @@ -214,7 +214,7 @@ private: Common::MultiLevelPageTable page_table; Common::RangeMap kind_map; - Common::SparseLargeVector big_page_table_dev; + Common::VirtualBuffer big_page_table_dev; std::vector big_page_continuous; boost::container::small_vector, 32> page_stash{};