debuggerx/eden
1
0
Fork 0
mirror of https://git.eden-emu.dev/eden-emu/eden.git synced 2026-10-07 06:18:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "[common/core/dynarmic] Optimize page table allocations (#4219)"

Browse Source
This commit is contained in:
CamilleLaVey
2026-09-20 14:18:28 -04:00
parent d82a9c3ba5
commit ffb75c2b68
27 changed files with 394 additions and 737 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/arm/dynarmic/arm_dynarmic_32.cpp
+5 -13
View File
@@ -172,12 +172,12 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) {
if (page_table) {
constexpr size_t PageBits = 12;
constexpr size_t NumPageTableEntries = 1 << (32 - PageBits);
constexpr size_t PageLog2Stride = 5;
static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData));
// Dynarmic will not write to the page table, const_cast is safe here
config.page_table = reinterpret_cast<std::array<std::uint8_t*, NumPageTableEntries>*>(
const_cast<Common::PageTable::PageEntryData*>(page_table->entries.data()));
config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK;
config.page_table_marked_bit = 0;
config.page_table = reinterpret_cast<std::array<std::uint8_t*, NumPageTableEntries>*>(page_table->entries.data());
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
config.page_table_log2_stride = PageLog2Stride;
config.absolute_offset_page_table = true;
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
config.only_detect_misalignment_via_page_table_on_page_boundary = true;
@@ -188,13 +188,6 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) {
config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt;
config.recompile_on_exclusive_fastmem_failure = true;
if (reinterpret_cast<u64>(m_system.DeviceMemory().buffer.BackingBasePointer() +
Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) {
// Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries,
// we have to manually sign extend when our actual pointer is negative.
config.page_table_sign_extension = Common::PageTable::SIGN_BIT;
}
}
// Multi-process state
@@ -427,7 +420,6 @@ void ArmDynarmic32::SignalInterrupt(Kernel::KThread* thread) {
}
void ArmDynarmic32::ClearInstructionCache() {
m_cb->last_code_addr = u64(-1);
m_jit->ClearCache();
}
src/core/arm/dynarmic/arm_dynarmic_64.cpp
+6 -13
View File
@@ -211,12 +211,13 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s
// Memory
if (page_table) {
// Dynarmic will not write to the page table, const_cast is safe here
config.page_table = reinterpret_cast<void**>(
const_cast<Common::PageTable::PageEntryData*>(page_table->entries.data()));
constexpr size_t PageLog2Stride = 5;
static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData));
config.page_table = reinterpret_cast<void**>(page_table->entries.data());
config.page_table_address_space_bits = std::uint32_t(address_space_bits);
config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK;
config.page_table_marked_bit = 0;
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
config.page_table_log2_stride = PageLog2Stride;
config.silently_mirror_page_table = false;
config.absolute_offset_page_table = true;
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
@@ -230,13 +231,6 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s
config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt;
config.recompile_on_exclusive_fastmem_failure = true;
if (reinterpret_cast<u64>(m_system.DeviceMemory().buffer.BackingBasePointer() +
Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) {
// Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries,
// we have to manually sign extend when our actual pointer is negative.
config.page_table_sign_extension = Common::PageTable::SIGN_BIT;
}
}
// Multi-process state
@@ -453,7 +447,6 @@ void ArmDynarmic64::SignalInterrupt(Kernel::KThread* thread) {
}
void ArmDynarmic64::ClearInstructionCache() {
m_cb->last_code_addr = u64(-1);
m_jit->ClearCache();
}
src/core/device_memory.h
+2 -8
View File
@@ -1,6 +1,3 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -29,11 +26,8 @@ public:
template <typename T>
Common::PhysicalAddress GetPhysicalAddr(const T* ptr) const {
return GetPhysicalAddr(reinterpret_cast<uintptr_t>(ptr));
}
Common::PhysicalAddress GetPhysicalAddr(uintptr_t ptr) const {
return (ptr - reinterpret_cast<uintptr_t>(buffer.BackingBasePointer())) +
return (reinterpret_cast<uintptr_t>(ptr) -
reinterpret_cast<uintptr_t>(buffer.BackingBasePointer())) +
DramMemoryMap::Base;
}
src/core/device_memory_manager.h
+5 -5
View File
@@ -18,7 +18,7 @@
#include "common/common_types.h"
#include "common/range_mutex.h"
#include "common/scratch_buffer.h"
#include "common/sparse_large_vector.h"
#include "common/virtual_buffer.h"
namespace Core {
@@ -178,8 +178,8 @@ private:
u32 continuity_tracker;
u32 compressed_physical_ptr;
};
Common::SparseLargeVector<u32> compressed_device_addr;
Common::SparseLargeVector<TrackedEntry> tracked_entries;
Common::VirtualBuffer<u32> compressed_device_addr;
Common::VirtualBuffer<TrackedEntry> tracked_entries;
// Process memory interfaces
@@ -200,8 +200,8 @@ private:
return std::make_pair(asid, address);
}
constexpr void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) {
tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit);
void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) {
tracked_entries[page_index].cpu_backing_address = address | (asid.id << asid_start_bit);
}
std::array<TranslationEntry, 4> t_slot{};
src/core/device_memory_manager.inc
+27 -22
View File
@@ -177,6 +177,17 @@ DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memo
{
impl = std::make_unique<DeviceMemoryManagerAllocator<Traits>>();
cached_pages = std::make_unique<CachedPages>();
const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS;
for (size_t i = 0; i < total_virtual; i++) {
tracked_entries[i].compressed_physical_ptr = 0;
tracked_entries[i].continuity_tracker = 1;
tracked_entries[i].cpu_backing_address = 0;
}
const size_t total_phys = 1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS);
for (size_t i = 0; i < total_phys; i++) {
compressed_device_addr[i] = 0;
}
}
template <typename Traits>
@@ -209,28 +220,26 @@ void DeviceMemoryManager<Traits>::Map(DAddr address, VAddr virtual_address, size
size_t start_page_d = address >> Memory::YUZU_PAGEBITS;
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
std::scoped_lock lk(mapping_guard);
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages);
for (size_t i = 0; i < num_pages; i++) {
const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE;
auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress));
if (ptr == nullptr) [[unlikely]] {
tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = 0;
tracked_entries[start_page_d + i].compressed_physical_ptr = 0;
continue;
}
auto phys_addr = static_cast<u32>(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U;
tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = phys_addr;
tracked_entries[start_page_d + i].compressed_physical_ptr = phys_addr;
InsertCPUBacking(start_page_d + i, new_vaddress, asid);
const u32 base_dev = compressed_device_addr[phys_addr - 1U];
const u32 new_dev = static_cast<u32>(start_page_d + i);
if (base_dev == 0) [[likely]] {
compressed_device_addr.GetAndFault(phys_addr - 1U) = new_dev;
compressed_device_addr[phys_addr - 1U] = new_dev;
continue;
}
u32 start_id = base_dev & MULTI_MASK;
if ((base_dev >> MULTI_FLAG_BITS) == 0) {
start_id = impl->multi_dev_address.Register(base_dev);
compressed_device_addr.GetAndFault(phys_addr - 1U) = MULTI_FLAG | start_id;
compressed_device_addr[phys_addr - 1U] = MULTI_FLAG | start_id;
}
impl->multi_dev_address.Register(new_dev, start_id);
}
@@ -246,26 +255,24 @@ void DeviceMemoryManager<Traits>::Unmap(DAddr address, size_t size) {
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
device_inter->InvalidateRegion(address, size);
std::scoped_lock lk(mapping_guard);
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); // should already be committed, but just in case
for (size_t i = 0; i < num_pages; i++) {
auto& entry = tracked_entries.GetUnchecked(start_page_d + i);
auto phys_addr = entry.compressed_physical_ptr;
entry.compressed_physical_ptr = 0;
entry.cpu_backing_address = 0;
auto phys_addr = tracked_entries[start_page_d + i].compressed_physical_ptr;
tracked_entries[start_page_d + i].compressed_physical_ptr = 0;
tracked_entries[start_page_d + i].cpu_backing_address = 0;
if (phys_addr != 0) [[likely]] {
u32& base_dev = compressed_device_addr.GetAndFault(phys_addr - 1U);
const u32 base_dev = compressed_device_addr[phys_addr - 1U];
if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] {
base_dev = 0;
compressed_device_addr[phys_addr - 1] = 0;
continue;
}
const auto [more_entries, new_start] = impl->multi_dev_address.Unregister(
static_cast<u32>(start_page_d + i), base_dev & MULTI_MASK);
if (!more_entries) {
base_dev = impl->multi_dev_address.ReleaseEntry(new_start);
compressed_device_addr[phys_addr - 1] =
impl->multi_dev_address.ReleaseEntry(new_start);
continue;
}
base_dev = new_start | MULTI_FLAG;
compressed_device_addr[phys_addr - 1] = new_start | MULTI_FLAG;
}
}
t_slot = {};
@@ -278,8 +285,6 @@ void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtu
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
uintptr_t last_ptr = 0;
size_t page_count = 1;
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages);
for (size_t i = num_pages; i > 0; i--) {
size_t index = i - 1;
const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE;
@@ -291,14 +296,14 @@ void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtu
page_count = 1;
}
last_ptr = new_ptr;
tracked_entries.GetUnchecked(start_page_d + index).continuity_tracker = static_cast<u32>(page_count) - 1;
tracked_entries[start_page_d + index].continuity_tracker = static_cast<u32>(page_count);
}
}
template <typename Traits>
u8* DeviceMemoryManager<Traits>::GetSpan(const DAddr src_addr, const std::size_t size) {
size_t page_index = src_addr >> page_bits;
size_t subbits = src_addr & page_mask;
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) {
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) {
return GetPointer<u8>(src_addr);
}
return nullptr;
@@ -308,7 +313,7 @@ template <typename Traits>
const u8* DeviceMemoryManager<Traits>::GetSpan(const DAddr src_addr, const std::size_t size) const {
size_t page_index = src_addr >> page_bits;
size_t subbits = src_addr & page_mask;
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) {
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) {
return GetPointer<u8>(src_addr);
}
return nullptr;
@@ -378,7 +383,7 @@ void DeviceMemoryManager<Traits>::WalkBlock(DAddr addr, std::size_t size, auto o
std::size_t page_index = addr >> Memory::YUZU_PAGEBITS;
std::size_t page_offset = addr & Memory::YUZU_PAGEMASK;
while (remaining_size) {
const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker+1);
const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker);
const std::size_t copy_amount = (std::min)((next_pages << Memory::YUZU_PAGEBITS) - page_offset, remaining_size);
const auto current_vaddr = u64((page_index << Memory::YUZU_PAGEBITS) + page_offset);
SCOPE_EXIT{
src/core/hle/kernel/k_page_table_base.cpp
+40 -70
View File
@@ -635,36 +635,6 @@ Result KPageTableBase::CheckMemoryState(const KMemoryInfo& info, KMemoryState st
R_SUCCEED();
}
bool KPageTableBase::BeginTraversal(const Common::PageTable &impl, TraversalEntry *out_entry, TraversalContext *out_context,
Common::ProcessAddress address) const {
out_context->next_offset = GetInteger(address);
out_context->next_page = GetInteger(address) >> PageBits;
return ContinueTraversal(impl, out_entry, out_context);
}
bool KPageTableBase::ContinueTraversal(const Common::PageTable &impl, TraversalEntry *out_entry,
TraversalContext *context) const {
// Setup invalid defaults.
out_entry->phys_addr = 0;
out_entry->block_size = PageSize;
// Validate that we can read the actual entry.
if (auto const page = context->next_page; page < impl.entries.size()) {
// Validate that the entry is mapped.
if (auto const paddr = impl.entries[page].Pointer(true); paddr != 0) {
// Populate the results and return true
out_entry->phys_addr = GetInteger(m_system.DeviceMemory().GetPhysicalAddr(paddr + context->next_offset));
context->next_page += 1;
context->next_offset += PageSize;
return true;
}
}
context->next_page += 1;
context->next_offset += PageSize;
// Otherwise return false
return false;
}
Result KPageTableBase::CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr,
size_t size, KMemoryState state_mask,
KMemoryState state, KMemoryPermission perm_mask,
@@ -970,7 +940,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a
size_t tot_size = 0;
next_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), region_start);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), region_start);
next_entry.block_size =
(next_entry.block_size - (GetInteger(region_start) & (next_entry.block_size - 1)));
@@ -1006,7 +976,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a
break;
}
next_valid = ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
next_valid = impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
}
// Check the last entry.
@@ -1784,7 +1754,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
R_UNLESS(BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr),
R_UNLESS(impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr),
ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -1794,7 +1764,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_
// Iterate, adding to group as we go.
while (tot_size < size) {
R_UNLESS(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)),
R_UNLESS(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)),
ResultInvalidCurrentMemory);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -1858,7 +1828,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
if (!BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr)) {
if (!impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr)) {
return false;
}
@@ -1869,7 +1839,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr
// Iterate, comparing expected to actual.
while (tot_size < size) {
if (!ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context))) {
if (!impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context))) {
return false;
}
@@ -1926,7 +1896,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState(
// Begin a traversal.
TraversalContext context;
TraversalEntry cur_entry = {.phys_addr = 0, .block_size = 0};
R_UNLESS(BeginTraversal(impl, std::addressof(cur_entry), std::addressof(context), address),
R_UNLESS(impl.BeginTraversal(std::addressof(cur_entry), std::addressof(context), address),
ResultInvalidCurrentMemory);
// Traverse until we have enough size or we aren't contiguous any more.
@@ -1935,7 +1905,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState(
for (contig_size =
cur_entry.block_size - (GetInteger(phys_address) & (cur_entry.block_size - 1));
contig_size < size; contig_size += cur_entry.block_size) {
if (!ContinueTraversal(impl, std::addressof(cur_entry), std::addressof(context))) {
if (!impl.ContinueTraversal(std::addressof(cur_entry), std::addressof(context))) {
break;
}
if (cur_entry.phys_addr != phys_address + contig_size) {
@@ -2364,7 +2334,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out,
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(m_impl, std::addressof(next_entry), std::addressof(context), virt_addr);
m_impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Set tracking variables.
@@ -2375,7 +2345,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out,
while (true) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(m_impl, std::addressof(next_entry), std::addressof(context));
m_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
if (!traverse_valid) {
break;
}
@@ -2597,7 +2567,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre
TraversalContext context;
TraversalEntry next_entry;
ASSERT(
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address));
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address));
// Check that the physical region matches.
R_UNLESS(next_entry.phys_addr == phys_addr, ResultInvalidMemoryRegion);
@@ -2607,7 +2577,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre
next_entry.block_size - (GetInteger(phys_addr) & (next_entry.block_size - 1));
checked_size < size; checked_size += next_entry.block_size) {
// Continue the traversal.
ASSERT(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)));
ASSERT(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)));
// Check that the physical region matches.
R_UNLESS(next_entry.phys_addr == phys_addr + checked_size, ResultInvalidMemoryRegion);
@@ -3059,7 +3029,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), address);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -3071,7 +3041,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3159,7 +3129,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_address);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -3197,7 +3167,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3255,7 +3225,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -3297,7 +3267,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3758,7 +3728,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -3798,7 +3768,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3852,7 +3822,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -3875,7 +3845,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3932,7 +3902,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -3971,7 +3941,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -4027,7 +3997,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -4050,7 +4020,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -4119,10 +4089,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
bool traverse_valid;
// Begin traversal.
traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry),
traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry),
std::addressof(src_context), src_addr);
ASSERT(traverse_valid);
traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry),
traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry),
std::addressof(dst_context), dst_addr);
ASSERT(traverse_valid);
@@ -4157,7 +4127,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
if (ofs + cur_copy_size != size) {
if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) {
// Continue the src traversal.
traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry),
traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry),
std::addressof(src_context));
ASSERT(traverse_valid);
@@ -4168,7 +4138,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
if (cur_dst_addr + cur_min_size ==
dst_next_entry.phys_addr + dst_next_entry.block_size) {
// Continue the dst traversal.
traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry),
traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry),
std::addressof(dst_context));
ASSERT(traverse_valid);
@@ -4253,10 +4223,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
bool traverse_valid;
// Begin traversal.
traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry),
traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry),
std::addressof(src_context), src_addr);
ASSERT(traverse_valid);
traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry),
traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry),
std::addressof(dst_context), dst_addr);
ASSERT(traverse_valid);
@@ -4291,7 +4261,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
if (ofs + cur_copy_size != size) {
if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) {
// Continue the src traversal.
traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry),
traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry),
std::addressof(src_context));
ASSERT(traverse_valid);
@@ -4302,7 +4272,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
if (cur_dst_addr + cur_min_size ==
dst_next_entry.phys_addr + dst_next_entry.block_size) {
// Continue the dst traversal.
traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry),
traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry),
std::addressof(dst_context));
ASSERT(traverse_valid);
@@ -4577,7 +4547,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid = BeginTraversal(src_impl, std::addressof(next_entry),
bool traverse_valid = src_impl.BeginTraversal(std::addressof(next_entry),
std::addressof(context), aligned_src_start);
ASSERT(traverse_valid);
@@ -4627,7 +4597,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
// If the block's size was one page, we may need to continue traversal.
if (cur_block_size == 0 && aligned_src_size > PageSize) {
traverse_valid =
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
cur_block_addr = next_entry.phys_addr;
@@ -4640,7 +4610,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
while (aligned_src_start + tot_block_size < mapping_src_end) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
// Process the block.
@@ -4683,7 +4653,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
if (mapped_block_end + cur_block_size < aligned_src_end &&
cur_block_size == last_block_size) {
traverse_valid =
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
cur_block_addr = next_entry.phys_addr;
@@ -5631,7 +5601,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si
ContiguousRangeInfo(KPageTableBase& pt, KProcessAddress address, size_t size)
: m_pt(pt), m_remaining_size(size) {
// Begin a traversal.
ASSERT(m_pt.BeginTraversal(m_pt.GetImpl(), std::addressof(m_entry),
ASSERT(m_pt.GetImpl().BeginTraversal(std::addressof(m_entry),
std::addressof(m_context), address));
// Setup tracking fields.
@@ -5662,7 +5632,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si
void DetermineContiguousBlockExtents() {
// Continue traversing until we're not contiguous, or we have enough.
while (m_cur_size < m_remaining_size) {
ASSERT(m_pt.ContinueTraversal(m_pt.GetImpl(), std::addressof(m_entry),
ASSERT(m_pt.GetImpl().ContinueTraversal(std::addressof(m_entry),
std::addressof(m_context)));
// If we're not contiguous, we're done.
src/core/hle/kernel/k_page_table_base.h
+1 -12
View File
@@ -370,10 +370,6 @@ private:
size_t num_pages, size_t alignment, size_t offset,
size_t guard_pages) const;
bool BeginTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* out_context,
Common::ProcessAddress address) const;
bool ContinueTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* context) const;
Result CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size,
KMemoryState state_mask, KMemoryState state,
KMemoryPermission perm_mask, KMemoryPermission perm,
@@ -478,14 +474,7 @@ private:
// Validate pre-conditions.
ASSERT(this->IsLockedByCurrentThread());
if (virt_addr > (1ULL << m_address_space_width)) {
return false;
}
*out = m_system.DeviceMemory().GetPhysicalAddr(
this->GetImpl().entries[GetInteger(virt_addr) >> PageBits].Pointer(true) + GetInteger(virt_addr));
return true;
return this->GetImpl().GetPhysicalAddress(out, virt_addr);
}
public:
src/core/memory.cpp
+39 -42
View File
@@ -101,10 +101,8 @@ struct Memory::Impl {
}
u64 protect_bytes = 0, protect_begin = 0;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr + size) >> YUZU_PAGEBITS);
for (u64 addr = vaddr; addr < vaddr + size; addr += YUZU_PAGESIZE) {
const Common::PageType page_type = current_page_table->entries.GetUnchecked(addr >> YUZU_PAGEBITS).Type();
const Common::PageType page_type = current_page_table->entries[addr >> YUZU_PAGEBITS].ptr.Type();
switch (page_type) {
case Common::PageType::RasterizerCachedMemory:
if (protect_bytes > 0) {
@@ -125,14 +123,16 @@ struct Memory::Impl {
}
[[nodiscard]] u8* GetPointerFromRasterizerCachedMemory(u64 vaddr) const {
if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr)
return reinterpret_cast<u8*>(paddr) + vaddr;
Common::PhysicalAddress const paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr;
if (paddr)
return system.DeviceMemory().GetPointer<u8>(paddr + vaddr);
return {};
}
[[nodiscard]] u8* GetPointerFromDebugMemory(u64 vaddr) const {
if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr)
return reinterpret_cast<u8*>(paddr) + vaddr;
const Common::PhysicalAddress paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr;
if (paddr != 0)
return system.DeviceMemory().GetPointer<u8>(paddr + vaddr);
return {};
}
@@ -243,12 +243,10 @@ struct Memory::Impl {
std::size_t page_index = addr >> YUZU_PAGEBITS;
std::size_t page_offset = addr & YUZU_PAGEMASK;
bool user_accessible = true;
current_page_table->entries.CommitRegion(page_index, page_index + (size >> YUZU_PAGEBITS) + 1);
while (remaining_size != 0) {
const std::size_t copy_amount = (std::min)(std::size_t(YUZU_PAGESIZE) - page_offset, remaining_size);
const auto current_vaddr = u64((page_index << YUZU_PAGEBITS) + page_offset);
const auto [pointer, type, _] = current_page_table->entries.GetUnchecked(page_index).PointerTypeBlock();
const auto [pointer, type] = current_page_table->entries[page_index].ptr.PointerType();
switch (type) {
case Common::PageType::Unmapped: {
user_accessible = false;
@@ -299,10 +297,10 @@ struct Memory::Impl {
}
[[nodiscard]] inline const u8* GetSpan(const VAddr addr, const std::size_t size) const noexcept {
return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr;
return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr;
}
[[nodiscard]] inline u8* GetSpan(const VAddr addr, const std::size_t size) noexcept {
return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr;
return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr;
}
bool WriteBlockImpl(const Common::ProcessAddress addr, const void* buffer, const std::size_t size, bool unsafe) {
@@ -406,14 +404,11 @@ struct Memory::Impl {
// The region is at a granularity of CPU pages.
const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
const auto [pointer, type, block] = entry.PointerTypeBlock(true);
const Common::PageType page_type = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type();
if (debug) {
// Switch page type to debug if now debug
switch (type) {
switch (page_type) {
case Common::PageType::Unmapped:
ASSERT(false && "Attempted to mark unmapped pages as debug");
break;
@@ -422,14 +417,14 @@ struct Memory::Impl {
// Page is already marked.
break;
case Common::PageType::Memory:
entry.MarkDebug(pointer, block);
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::DebugMemory);
break;
default:
UNREACHABLE();
}
} else {
// Switch page type to non-debug if now non-debug
switch (type) {
switch (page_type) {
case Common::PageType::Unmapped:
ASSERT(false && "Attempted to mark unmapped pages as non-debug");
break;
@@ -438,7 +433,8 @@ struct Memory::Impl {
// Don't mess with already non-debug or rasterizer memory.
break;
case Common::PageType::DebugMemory: {
entry.Store(false, Common::PageType::Memory, block, pointer);
u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK);
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
break;
}
default:
@@ -470,10 +466,8 @@ struct Memory::Impl {
// is different). This assumes the specified GPU address region is contiguous as well.
const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
const Common::PageType page_type = entry.Type();
const Common::PageType page_type= current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type();
if (cached) {
// Switch page type to cached if now cached
switch (page_type) {
@@ -483,7 +477,7 @@ struct Memory::Impl {
break;
case Common::PageType::DebugMemory:
case Common::PageType::Memory:
entry.MarkRasterizerCached();
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::RasterizerCachedMemory);
break;
case Common::PageType::RasterizerCachedMemory:
// There can be more than one GPU region mapped per CPU region, so it's common
@@ -505,13 +499,13 @@ struct Memory::Impl {
// that this area is already unmarked as cached.
break;
case Common::PageType::RasterizerCachedMemory: {
if (auto [ptr, _, block] = entry.PointerTypeBlock(true); ptr == 0) {
if (u8* const pointer = GetPointerFromRasterizerCachedMemory(vaddr & ~YUZU_PAGEMASK); pointer == nullptr) {
// It's possible that this function has been called while updating the
// pagetable after unmapping a VMA. In that case the underlying VMA will no
// longer exist, and we should just leave the pagetable entry blank.
entry.Store(false, Common::PageType::Unmapped, block, 0);
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::Unmapped);
} else {
entry.Store(false, Common::PageType::Memory, block, ptr);
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
}
break;
}
@@ -545,18 +539,22 @@ struct Memory::Impl {
ASSERT_MSG(type != Common::PageType::Memory,
"Mapping memory page without a pointer @ {:016x}", base * YUZU_PAGESIZE);
page_table.entries.ZeroRegion(base, end);
} else {
auto current_block = block_count.fetch_add(1, std::memory_order_relaxed);
ASSERT(current_block != 65535);
page_table.entries.CommitRegion(base, end);
while (base != end) {
auto host_ptr = reinterpret_cast<u64>(system.DeviceMemory().GetPointer<u8>(target)) - (base << YUZU_PAGEBITS);;
auto& entry = page_table.entries.GetUnchecked(base);
page_table.entries[base].ptr.Store(0, type);
page_table.entries[base].addr = 0;
page_table.entries[base].block = 0;
base += 1;
}
} else {
auto orig_base = base;
while (base != end) {
auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer<u8>(target)) - (base << YUZU_PAGEBITS);
auto backing = GetInteger(target) - (base << YUZU_PAGEBITS);
page_table.entries[base].ptr.Store(host_ptr, type);
page_table.entries[base].addr = backing;
page_table.entries[base].block = orig_base << YUZU_PAGEBITS;
entry.Store(false, type, current_block, host_ptr);
ASSERT_MSG(page_table.entries[base].Pointer(),
ASSERT_MSG(page_table.entries[base].ptr.Pointer(),
"memory mapping base yield a nullptr within the table");
base += 1;
@@ -571,11 +569,11 @@ struct Memory::Impl {
vaddr &= 0xffffffffffffULL;
if (AddressSpaceContains(*current_page_table, vaddr, 1)) [[likely]] {
// Avoid adding any extra logic to this fast-path block
const auto raw = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Raw();
if (auto pointer = Common::PageTable::PageEntryData::ExtractPointer(raw); pointer) [[likely]] {
const uintptr_t raw_pointer = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Raw();
if (const uintptr_t pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) [[likely]] {
return reinterpret_cast<u8*>(pointer + vaddr);
} else {
switch (static_cast<Common::PageType>(raw.type)) {
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
case Common::PageType::Memory:
ASSERT_MSG(false, "Mapped memory page without a pointer @ {:#016x}", vaddr);
return nullptr;
@@ -775,7 +773,6 @@ struct Memory::Impl {
#else
Common::HostMemory* host_buffer{};
#endif
std::atomic<u16> block_count = 0;
};
Memory::Memory(Core::System& system_) : system{system_} {
@@ -814,7 +811,7 @@ bool Memory::IsValidVirtualAddress(const Common::ProcessAddress vaddr) const {
if (page >= page_table.entries.size()) {
return false;
}
const auto [pointer, type, _] = page_table.entries[page].PointerTypeBlock();
const auto [pointer, type] = page_table.entries[page].ptr.PointerType();
return pointer != 0 || type == Common::PageType::RasterizerCachedMemory ||
type == Common::PageType::DebugMemory;
}