debuggerx/eden
1
0
Fork 0
mirror of https://git.eden-emu.dev/eden-emu/eden.git synced 2026-09-10 22:16:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

[common/core/dynarmic] Optimize page table allocations (#4219)

Browse Source 
Reduces the page entries from 32 bytes to 8 and rewrites `VirtualBuffer` to be more efficient in memory usage and specifically for large zero regions.

The page table will now only reserve 1GiB instead of 4GiB and of this memory it should only use at most ~8MiB.

This PR has the side effect of using Eden on Windows on low memory systems much more plausible since it would previously require ~10GiB of committable memory at front (despite using ~5-6 at most, inadvertently stalling other processes) where as now it should only require around the amount that it'll actually use.

Co-authored-by: Lizzie <lizzie@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4219
Reviewed-by: Maufeat <sahyno1996@gmail.com>
Reviewed-by: lizzie <lizzie@eden-emu.dev>
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
This commit is contained in:
Exverge
2026-09-09 21:35:49 +02:00
committed by crueter
parent c5f6f1ca5e
commit 5f142c7926
27 changed files with 730 additions and 393 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/arm/dynarmic/arm_dynarmic_32.cpp
+13 -5
View File
@@ -172,12 +172,12 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) {
if (page_table) {
constexpr size_t PageBits = 12;
constexpr size_t NumPageTableEntries = 1 << (32 - PageBits);
constexpr size_t PageLog2Stride = 5;
static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData));
config.page_table = reinterpret_cast<std::array<std::uint8_t*, NumPageTableEntries>*>(page_table->entries.data());
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
config.page_table_log2_stride = PageLog2Stride;
// Dynarmic will not write to the page table, const_cast is safe here
config.page_table = reinterpret_cast<std::array<std::uint8_t*, NumPageTableEntries>*>(
const_cast<Common::PageTable::PageEntryData*>(page_table->entries.data()));
config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK;
config.page_table_marked_bit = 0;
config.absolute_offset_page_table = true;
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
config.only_detect_misalignment_via_page_table_on_page_boundary = true;
@@ -188,6 +188,13 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) {
config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt;
config.recompile_on_exclusive_fastmem_failure = true;
if (reinterpret_cast<u64>(m_system.DeviceMemory().buffer.BackingBasePointer() +
Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) {
// Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries,
// we have to manually sign extend when our actual pointer is negative.
config.page_table_sign_extension = Common::PageTable::SIGN_BIT;
}
}
// Multi-process state
@@ -420,6 +427,7 @@ void ArmDynarmic32::SignalInterrupt(Kernel::KThread* thread) {
}
void ArmDynarmic32::ClearInstructionCache() {
m_cb->last_code_addr = u64(-1);
m_jit->ClearCache();
}
src/core/arm/dynarmic/arm_dynarmic_64.cpp
+13 -6
View File
@@ -211,13 +211,12 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s
// Memory
if (page_table) {
constexpr size_t PageLog2Stride = 5;
static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData));
config.page_table = reinterpret_cast<void**>(page_table->entries.data());
// Dynarmic will not write to the page table, const_cast is safe here
config.page_table = reinterpret_cast<void**>(
const_cast<Common::PageTable::PageEntryData*>(page_table->entries.data()));
config.page_table_address_space_bits = std::uint32_t(address_space_bits);
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
config.page_table_log2_stride = PageLog2Stride;
config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK;
config.page_table_marked_bit = 0;
config.silently_mirror_page_table = false;
config.absolute_offset_page_table = true;
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
@@ -231,6 +230,13 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s
config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt;
config.recompile_on_exclusive_fastmem_failure = true;
if (reinterpret_cast<u64>(m_system.DeviceMemory().buffer.BackingBasePointer() +
Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) {
// Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries,
// we have to manually sign extend when our actual pointer is negative.
config.page_table_sign_extension = Common::PageTable::SIGN_BIT;
}
}
// Multi-process state
@@ -447,6 +453,7 @@ void ArmDynarmic64::SignalInterrupt(Kernel::KThread* thread) {
}
void ArmDynarmic64::ClearInstructionCache() {
m_cb->last_code_addr = u64(-1);
m_jit->ClearCache();
}
src/core/device_memory.h
+8 -2
View File
@@ -1,3 +1,6 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -26,8 +29,11 @@ public:
template <typename T>
Common::PhysicalAddress GetPhysicalAddr(const T* ptr) const {
return (reinterpret_cast<uintptr_t>(ptr) -
reinterpret_cast<uintptr_t>(buffer.BackingBasePointer())) +
return GetPhysicalAddr(reinterpret_cast<uintptr_t>(ptr));
}
Common::PhysicalAddress GetPhysicalAddr(uintptr_t ptr) const {
return (ptr - reinterpret_cast<uintptr_t>(buffer.BackingBasePointer())) +
DramMemoryMap::Base;
}
src/core/device_memory_manager.h
+5 -5
View File
@@ -18,7 +18,7 @@
#include "common/common_types.h"
#include "common/range_mutex.h"
#include "common/scratch_buffer.h"
#include "common/virtual_buffer.h"
#include "common/sparse_large_vector.h"
namespace Core {
@@ -178,8 +178,8 @@ private:
u32 continuity_tracker;
u32 compressed_physical_ptr;
};
Common::VirtualBuffer<u32> compressed_device_addr;
Common::VirtualBuffer<TrackedEntry> tracked_entries;
Common::SparseLargeVector<u32> compressed_device_addr;
Common::SparseLargeVector<TrackedEntry> tracked_entries;
// Process memory interfaces
@@ -200,8 +200,8 @@ private:
return std::make_pair(asid, address);
}
void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) {
tracked_entries[page_index].cpu_backing_address = address | (asid.id << asid_start_bit);
constexpr void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) {
tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit);
}
std::array<TranslationEntry, 4> t_slot{};
src/core/device_memory_manager.inc
+22 -27
View File
@@ -177,17 +177,6 @@ DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memo
{
impl = std::make_unique<DeviceMemoryManagerAllocator<Traits>>();
cached_pages = std::make_unique<CachedPages>();
const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS;
for (size_t i = 0; i < total_virtual; i++) {
tracked_entries[i].compressed_physical_ptr = 0;
tracked_entries[i].continuity_tracker = 1;
tracked_entries[i].cpu_backing_address = 0;
}
const size_t total_phys = 1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS);
for (size_t i = 0; i < total_phys; i++) {
compressed_device_addr[i] = 0;
}
}
template <typename Traits>
@@ -220,26 +209,28 @@ void DeviceMemoryManager<Traits>::Map(DAddr address, VAddr virtual_address, size
size_t start_page_d = address >> Memory::YUZU_PAGEBITS;
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
std::scoped_lock lk(mapping_guard);
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages);
for (size_t i = 0; i < num_pages; i++) {
const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE;
auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress));
if (ptr == nullptr) [[unlikely]] {
tracked_entries[start_page_d + i].compressed_physical_ptr = 0;
tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = 0;
continue;
}
auto phys_addr = static_cast<u32>(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U;
tracked_entries[start_page_d + i].compressed_physical_ptr = phys_addr;
tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = phys_addr;
InsertCPUBacking(start_page_d + i, new_vaddress, asid);
const u32 base_dev = compressed_device_addr[phys_addr - 1U];
const u32 new_dev = static_cast<u32>(start_page_d + i);
if (base_dev == 0) [[likely]] {
compressed_device_addr[phys_addr - 1U] = new_dev;
compressed_device_addr.GetAndFault(phys_addr - 1U) = new_dev;
continue;
}
u32 start_id = base_dev & MULTI_MASK;
if ((base_dev >> MULTI_FLAG_BITS) == 0) {
start_id = impl->multi_dev_address.Register(base_dev);
compressed_device_addr[phys_addr - 1U] = MULTI_FLAG | start_id;
compressed_device_addr.GetAndFault(phys_addr - 1U) = MULTI_FLAG | start_id;
}
impl->multi_dev_address.Register(new_dev, start_id);
}
@@ -255,24 +246,26 @@ void DeviceMemoryManager<Traits>::Unmap(DAddr address, size_t size) {
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
device_inter->InvalidateRegion(address, size);
std::scoped_lock lk(mapping_guard);
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); // should already be committed, but just in case
for (size_t i = 0; i < num_pages; i++) {
auto phys_addr = tracked_entries[start_page_d + i].compressed_physical_ptr;
tracked_entries[start_page_d + i].compressed_physical_ptr = 0;
tracked_entries[start_page_d + i].cpu_backing_address = 0;
auto& entry = tracked_entries.GetUnchecked(start_page_d + i);
auto phys_addr = entry.compressed_physical_ptr;
entry.compressed_physical_ptr = 0;
entry.cpu_backing_address = 0;
if (phys_addr != 0) [[likely]] {
const u32 base_dev = compressed_device_addr[phys_addr - 1U];
u32& base_dev = compressed_device_addr.GetAndFault(phys_addr - 1U);
if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] {
compressed_device_addr[phys_addr - 1] = 0;
base_dev = 0;
continue;
}
const auto [more_entries, new_start] = impl->multi_dev_address.Unregister(
static_cast<u32>(start_page_d + i), base_dev & MULTI_MASK);
if (!more_entries) {
compressed_device_addr[phys_addr - 1] =
impl->multi_dev_address.ReleaseEntry(new_start);
base_dev = impl->multi_dev_address.ReleaseEntry(new_start);
continue;
}
compressed_device_addr[phys_addr - 1] = new_start | MULTI_FLAG;
base_dev = new_start | MULTI_FLAG;
}
}
t_slot = {};
@@ -285,6 +278,8 @@ void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtu
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
uintptr_t last_ptr = 0;
size_t page_count = 1;
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages);
for (size_t i = num_pages; i > 0; i--) {
size_t index = i - 1;
const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE;
@@ -296,14 +291,14 @@ void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtu
page_count = 1;
}
last_ptr = new_ptr;
tracked_entries[start_page_d + index].continuity_tracker = static_cast<u32>(page_count);
tracked_entries.GetUnchecked(start_page_d + index).continuity_tracker = static_cast<u32>(page_count) - 1;
}
}
template <typename Traits>
u8* DeviceMemoryManager<Traits>::GetSpan(const DAddr src_addr, const std::size_t size) {
size_t page_index = src_addr >> page_bits;
size_t subbits = src_addr & page_mask;
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) {
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) {
return GetPointer<u8>(src_addr);
}
return nullptr;
@@ -313,7 +308,7 @@ template <typename Traits>
const u8* DeviceMemoryManager<Traits>::GetSpan(const DAddr src_addr, const std::size_t size) const {
size_t page_index = src_addr >> page_bits;
size_t subbits = src_addr & page_mask;
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) {
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) {
return GetPointer<u8>(src_addr);
}
return nullptr;
@@ -383,7 +378,7 @@ void DeviceMemoryManager<Traits>::WalkBlock(DAddr addr, std::size_t size, auto o
std::size_t page_index = addr >> Memory::YUZU_PAGEBITS;
std::size_t page_offset = addr & Memory::YUZU_PAGEMASK;
while (remaining_size) {
const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker);
const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker+1);
const std::size_t copy_amount = (std::min)((next_pages << Memory::YUZU_PAGEBITS) - page_offset, remaining_size);
const auto current_vaddr = u64((page_index << Memory::YUZU_PAGEBITS) + page_offset);
SCOPE_EXIT{
src/core/hle/kernel/k_page_table_base.cpp
+70 -40
View File
@@ -635,6 +635,36 @@ Result KPageTableBase::CheckMemoryState(const KMemoryInfo& info, KMemoryState st
R_SUCCEED();
}
bool KPageTableBase::BeginTraversal(const Common::PageTable &impl, TraversalEntry *out_entry, TraversalContext *out_context,
Common::ProcessAddress address) const {
out_context->next_offset = GetInteger(address);
out_context->next_page = GetInteger(address) >> PageBits;
return ContinueTraversal(impl, out_entry, out_context);
}
bool KPageTableBase::ContinueTraversal(const Common::PageTable &impl, TraversalEntry *out_entry,
TraversalContext *context) const {
// Setup invalid defaults.
out_entry->phys_addr = 0;
out_entry->block_size = PageSize;
// Validate that we can read the actual entry.
if (auto const page = context->next_page; page < impl.entries.size()) {
// Validate that the entry is mapped.
if (auto const paddr = impl.entries[page].Pointer(true); paddr != 0) {
// Populate the results and return true
out_entry->phys_addr = GetInteger(m_system.DeviceMemory().GetPhysicalAddr(paddr + context->next_offset));
context->next_page += 1;
context->next_offset += PageSize;
return true;
}
}
context->next_page += 1;
context->next_offset += PageSize;
// Otherwise return false
return false;
}
Result KPageTableBase::CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr,
size_t size, KMemoryState state_mask,
KMemoryState state, KMemoryPermission perm_mask,
@@ -940,7 +970,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a
size_t tot_size = 0;
next_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), region_start);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), region_start);
next_entry.block_size =
(next_entry.block_size - (GetInteger(region_start) & (next_entry.block_size - 1)));
@@ -976,7 +1006,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a
break;
}
next_valid = impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
next_valid = ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
}
// Check the last entry.
@@ -1754,7 +1784,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
R_UNLESS(impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr),
R_UNLESS(BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr),
ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -1764,7 +1794,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_
// Iterate, adding to group as we go.
while (tot_size < size) {
R_UNLESS(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)),
R_UNLESS(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)),
ResultInvalidCurrentMemory);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -1828,7 +1858,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
if (!impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr)) {
if (!BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr)) {
return false;
}
@@ -1839,7 +1869,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr
// Iterate, comparing expected to actual.
while (tot_size < size) {
if (!impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context))) {
if (!ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context))) {
return false;
}
@@ -1896,7 +1926,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState(
// Begin a traversal.
TraversalContext context;
TraversalEntry cur_entry = {.phys_addr = 0, .block_size = 0};
R_UNLESS(impl.BeginTraversal(std::addressof(cur_entry), std::addressof(context), address),
R_UNLESS(BeginTraversal(impl, std::addressof(cur_entry), std::addressof(context), address),
ResultInvalidCurrentMemory);
// Traverse until we have enough size or we aren't contiguous any more.
@@ -1905,7 +1935,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState(
for (contig_size =
cur_entry.block_size - (GetInteger(phys_address) & (cur_entry.block_size - 1));
contig_size < size; contig_size += cur_entry.block_size) {
if (!impl.ContinueTraversal(std::addressof(cur_entry), std::addressof(context))) {
if (!ContinueTraversal(impl, std::addressof(cur_entry), std::addressof(context))) {
break;
}
if (cur_entry.phys_addr != phys_address + contig_size) {
@@ -2334,7 +2364,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out,
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
m_impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr);
BeginTraversal(m_impl, std::addressof(next_entry), std::addressof(context), virt_addr);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Set tracking variables.
@@ -2345,7 +2375,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out,
while (true) {
// Continue the traversal.
traverse_valid =
m_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(m_impl, std::addressof(next_entry), std::addressof(context));
if (!traverse_valid) {
break;
}
@@ -2567,7 +2597,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre
TraversalContext context;
TraversalEntry next_entry;
ASSERT(
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address));
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address));
// Check that the physical region matches.
R_UNLESS(next_entry.phys_addr == phys_addr, ResultInvalidMemoryRegion);
@@ -2577,7 +2607,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre
next_entry.block_size - (GetInteger(phys_addr) & (next_entry.block_size - 1));
checked_size < size; checked_size += next_entry.block_size) {
// Continue the traversal.
ASSERT(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)));
ASSERT(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)));
// Check that the physical region matches.
R_UNLESS(next_entry.phys_addr == phys_addr + checked_size, ResultInvalidMemoryRegion);
@@ -3029,7 +3059,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), address);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -3041,7 +3071,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3129,7 +3159,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_address);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -3167,7 +3197,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3225,7 +3255,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -3267,7 +3297,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3728,7 +3758,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -3768,7 +3798,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3822,7 +3852,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -3845,7 +3875,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3902,7 +3932,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -3941,7 +3971,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3997,7 +4027,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -4020,7 +4050,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -4089,10 +4119,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
bool traverse_valid;
// Begin traversal.
traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry),
traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry),
std::addressof(src_context), src_addr);
ASSERT(traverse_valid);
traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry),
traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry),
std::addressof(dst_context), dst_addr);
ASSERT(traverse_valid);
@@ -4127,7 +4157,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
if (ofs + cur_copy_size != size) {
if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) {
// Continue the src traversal.
traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry),
traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry),
std::addressof(src_context));
ASSERT(traverse_valid);
@@ -4138,7 +4168,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
if (cur_dst_addr + cur_min_size ==
dst_next_entry.phys_addr + dst_next_entry.block_size) {
// Continue the dst traversal.
traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry),
traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry),
std::addressof(dst_context));
ASSERT(traverse_valid);
@@ -4223,10 +4253,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
bool traverse_valid;
// Begin traversal.
traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry),
traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry),
std::addressof(src_context), src_addr);
ASSERT(traverse_valid);
traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry),
traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry),
std::addressof(dst_context), dst_addr);
ASSERT(traverse_valid);
@@ -4261,7 +4291,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
if (ofs + cur_copy_size != size) {
if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) {
// Continue the src traversal.
traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry),
traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry),
std::addressof(src_context));
ASSERT(traverse_valid);
@@ -4272,7 +4302,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
if (cur_dst_addr + cur_min_size ==
dst_next_entry.phys_addr + dst_next_entry.block_size) {
// Continue the dst traversal.
traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry),
traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry),
std::addressof(dst_context));
ASSERT(traverse_valid);
@@ -4547,7 +4577,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid = src_impl.BeginTraversal(std::addressof(next_entry),
bool traverse_valid = BeginTraversal(src_impl, std::addressof(next_entry),
std::addressof(context), aligned_src_start);
ASSERT(traverse_valid);
@@ -4597,7 +4627,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
// If the block's size was one page, we may need to continue traversal.
if (cur_block_size == 0 && aligned_src_size > PageSize) {
traverse_valid =
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
cur_block_addr = next_entry.phys_addr;
@@ -4610,7 +4640,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
while (aligned_src_start + tot_block_size < mapping_src_end) {
// Continue the traversal.
traverse_valid =
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
// Process the block.
@@ -4653,7 +4683,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
if (mapped_block_end + cur_block_size < aligned_src_end &&
cur_block_size == last_block_size) {
traverse_valid =
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
cur_block_addr = next_entry.phys_addr;
@@ -5601,7 +5631,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si
ContiguousRangeInfo(KPageTableBase& pt, KProcessAddress address, size_t size)
: m_pt(pt), m_remaining_size(size) {
// Begin a traversal.
ASSERT(m_pt.GetImpl().BeginTraversal(std::addressof(m_entry),
ASSERT(m_pt.BeginTraversal(m_pt.GetImpl(), std::addressof(m_entry),
std::addressof(m_context), address));
// Setup tracking fields.
@@ -5632,7 +5662,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si
void DetermineContiguousBlockExtents() {
// Continue traversing until we're not contiguous, or we have enough.
while (m_cur_size < m_remaining_size) {
ASSERT(m_pt.GetImpl().ContinueTraversal(std::addressof(m_entry),
ASSERT(m_pt.ContinueTraversal(m_pt.GetImpl(), std::addressof(m_entry),
std::addressof(m_context)));
// If we're not contiguous, we're done.
src/core/hle/kernel/k_page_table_base.h
+12 -1
View File
@@ -370,6 +370,10 @@ private:
size_t num_pages, size_t alignment, size_t offset,
size_t guard_pages) const;
bool BeginTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* out_context,
Common::ProcessAddress address) const;
bool ContinueTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* context) const;
Result CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size,
KMemoryState state_mask, KMemoryState state,
KMemoryPermission perm_mask, KMemoryPermission perm,
@@ -474,7 +478,14 @@ private:
// Validate pre-conditions.
ASSERT(this->IsLockedByCurrentThread());
return this->GetImpl().GetPhysicalAddress(out, virt_addr);
if (virt_addr > (1ULL << m_address_space_width)) {
return false;
}
*out = m_system.DeviceMemory().GetPhysicalAddr(
this->GetImpl().entries[GetInteger(virt_addr) >> PageBits].Pointer(true) + GetInteger(virt_addr));
return true;
}
public:
src/core/memory.cpp
+42 -39
View File
@@ -101,8 +101,10 @@ struct Memory::Impl {
}
u64 protect_bytes = 0, protect_begin = 0;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr + size) >> YUZU_PAGEBITS);
for (u64 addr = vaddr; addr < vaddr + size; addr += YUZU_PAGESIZE) {
const Common::PageType page_type = current_page_table->entries[addr >> YUZU_PAGEBITS].ptr.Type();
const Common::PageType page_type = current_page_table->entries.GetUnchecked(addr >> YUZU_PAGEBITS).Type();
switch (page_type) {
case Common::PageType::RasterizerCachedMemory:
if (protect_bytes > 0) {
@@ -123,16 +125,14 @@ struct Memory::Impl {
}
[[nodiscard]] u8* GetPointerFromRasterizerCachedMemory(u64 vaddr) const {
Common::PhysicalAddress const paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr;
if (paddr)
return system.DeviceMemory().GetPointer<u8>(paddr + vaddr);
if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr)
return reinterpret_cast<u8*>(paddr) + vaddr;
return {};
}
[[nodiscard]] u8* GetPointerFromDebugMemory(u64 vaddr) const {
const Common::PhysicalAddress paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr;
if (paddr != 0)
return system.DeviceMemory().GetPointer<u8>(paddr + vaddr);
if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr)
return reinterpret_cast<u8*>(paddr) + vaddr;
return {};
}
@@ -243,10 +243,12 @@ struct Memory::Impl {
std::size_t page_index = addr >> YUZU_PAGEBITS;
std::size_t page_offset = addr & YUZU_PAGEMASK;
bool user_accessible = true;
current_page_table->entries.CommitRegion(page_index, page_index + (size >> YUZU_PAGEBITS) + 1);
while (remaining_size != 0) {
const std::size_t copy_amount = (std::min)(std::size_t(YUZU_PAGESIZE) - page_offset, remaining_size);
const auto current_vaddr = u64((page_index << YUZU_PAGEBITS) + page_offset);
const auto [pointer, type] = current_page_table->entries[page_index].ptr.PointerType();
const auto [pointer, type, _] = current_page_table->entries.GetUnchecked(page_index).PointerTypeBlock();
switch (type) {
case Common::PageType::Unmapped: {
user_accessible = false;
@@ -297,10 +299,10 @@ struct Memory::Impl {
}
[[nodiscard]] inline const u8* GetSpan(const VAddr addr, const std::size_t size) const noexcept {
return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr;
return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr;
}
[[nodiscard]] inline u8* GetSpan(const VAddr addr, const std::size_t size) noexcept {
return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr;
return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr;
}
bool WriteBlockImpl(const Common::ProcessAddress addr, const void* buffer, const std::size_t size, bool unsafe) {
@@ -404,11 +406,14 @@ struct Memory::Impl {
// The region is at a granularity of CPU pages.
const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
const Common::PageType page_type = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type();
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
const auto [pointer, type, block] = entry.PointerTypeBlock(true);
if (debug) {
// Switch page type to debug if now debug
switch (page_type) {
switch (type) {
case Common::PageType::Unmapped:
ASSERT(false && "Attempted to mark unmapped pages as debug");
break;
@@ -417,14 +422,14 @@ struct Memory::Impl {
// Page is already marked.
break;
case Common::PageType::Memory:
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::DebugMemory);
entry.MarkDebug(pointer, block);
break;
default:
UNREACHABLE();
}
} else {
// Switch page type to non-debug if now non-debug
switch (page_type) {
switch (type) {
case Common::PageType::Unmapped:
ASSERT(false && "Attempted to mark unmapped pages as non-debug");
break;
@@ -433,8 +438,7 @@ struct Memory::Impl {
// Don't mess with already non-debug or rasterizer memory.
break;
case Common::PageType::DebugMemory: {
u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK);
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
entry.Store(false, Common::PageType::Memory, block, pointer);
break;
}
default:
@@ -466,8 +470,10 @@ struct Memory::Impl {
// is different). This assumes the specified GPU address region is contiguous as well.
const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
const Common::PageType page_type= current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type();
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
const Common::PageType page_type = entry.Type();
if (cached) {
// Switch page type to cached if now cached
switch (page_type) {
@@ -477,7 +483,7 @@ struct Memory::Impl {
break;
case Common::PageType::DebugMemory:
case Common::PageType::Memory:
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::RasterizerCachedMemory);
entry.MarkRasterizerCached();
break;
case Common::PageType::RasterizerCachedMemory:
// There can be more than one GPU region mapped per CPU region, so it's common
@@ -499,13 +505,13 @@ struct Memory::Impl {
// that this area is already unmarked as cached.
break;
case Common::PageType::RasterizerCachedMemory: {
if (u8* const pointer = GetPointerFromRasterizerCachedMemory(vaddr & ~YUZU_PAGEMASK); pointer == nullptr) {
if (auto [ptr, _, block] = entry.PointerTypeBlock(true); ptr == 0) {
// It's possible that this function has been called while updating the
// pagetable after unmapping a VMA. In that case the underlying VMA will no
// longer exist, and we should just leave the pagetable entry blank.
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::Unmapped);
entry.Store(false, Common::PageType::Unmapped, block, 0);
} else {
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
entry.Store(false, Common::PageType::Memory, block, ptr);
}
break;
}
@@ -539,22 +545,18 @@ struct Memory::Impl {
ASSERT_MSG(type != Common::PageType::Memory,
"Mapping memory page without a pointer @ {:016x}", base * YUZU_PAGESIZE);
while (base != end) {
page_table.entries[base].ptr.Store(0, type);
page_table.entries[base].addr = 0;
page_table.entries[base].block = 0;
base += 1;
}
page_table.entries.ZeroRegion(base, end);
} else {
auto orig_base = base;
while (base != end) {
auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer<u8>(target)) - (base << YUZU_PAGEBITS);
auto backing = GetInteger(target) - (base << YUZU_PAGEBITS);
page_table.entries[base].ptr.Store(host_ptr, type);
page_table.entries[base].addr = backing;
page_table.entries[base].block = orig_base << YUZU_PAGEBITS;
auto current_block = block_count.fetch_add(1, std::memory_order_relaxed);
ASSERT(current_block != 65535);
ASSERT_MSG(page_table.entries[base].ptr.Pointer(),
page_table.entries.CommitRegion(base, end);
while (base != end) {
auto host_ptr = reinterpret_cast<u64>(system.DeviceMemory().GetPointer<u8>(target)) - (base << YUZU_PAGEBITS);;
auto& entry = page_table.entries.GetUnchecked(base);
entry.Store(false, type, current_block, host_ptr);
ASSERT_MSG(page_table.entries[base].Pointer(),
"memory mapping base yield a nullptr within the table");
base += 1;
@@ -569,11 +571,11 @@ struct Memory::Impl {
vaddr &= 0xffffffffffffULL;
if (AddressSpaceContains(*current_page_table, vaddr, 1)) [[likely]] {
// Avoid adding any extra logic to this fast-path block
const uintptr_t raw_pointer = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Raw();
if (const uintptr_t pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) [[likely]] {
const auto raw = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Raw();
if (auto pointer = Common::PageTable::PageEntryData::ExtractPointer(raw); pointer) [[likely]] {
return reinterpret_cast<u8*>(pointer + vaddr);
} else {
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
switch (static_cast<Common::PageType>(raw.type)) {
case Common::PageType::Memory:
ASSERT_MSG(false, "Mapped memory page without a pointer @ {:#016x}", vaddr);
return nullptr;
@@ -773,6 +775,7 @@ struct Memory::Impl {
#else
Common::HostMemory* host_buffer{};
#endif
std::atomic<u16> block_count = 0;
};
Memory::Memory(Core::System& system_) : system{system_} {
@@ -811,7 +814,7 @@ bool Memory::IsValidVirtualAddress(const Common::ProcessAddress vaddr) const {
if (page >= page_table.entries.size()) {
return false;
}
const auto [pointer, type] = page_table.entries[page].ptr.PointerType();
const auto [pointer, type, _] = page_table.entries[page].PointerTypeBlock();
return pointer != 0 || type == Common::PageType::RasterizerCachedMemory ||
type == Common::PageType::DebugMemory;
}