Compare commits

..

2 Commits

Author SHA1 Message Date
lizzie da07722056 Fix license headers 2026-09-12 20:39:09 +00:00
lizzie f75828da0c Revert "[common/core/dynarmic] Optimize page table allocations (#4219)"
This reverts commit 5f142c7926.
2026-09-12 20:35:16 +00:00
27 changed files with 391 additions and 725 deletions
+2 -2
View File
@@ -108,8 +108,6 @@ add_library(
settings_setting.h
slot_vector.h
socket_types.h
sparse_large_vector.cpp
sparse_large_vector.h
spin_lock.h
stb.cpp
stb.h
@@ -139,6 +137,8 @@ add_library(
uuid.cpp
uuid.h
vector_math.h
virtual_buffer.cpp
virtual_buffer.h
zstd_compression.cpp
zstd_compression.h
fs/ryujinx_compat.h fs/ryujinx_compat.cpp
+1
View File
@@ -9,6 +9,7 @@
#include "common/assert.h"
#include "common/fiber.h"
#include "common/virtual_buffer.h"
#include <boost/context/detail/fcontext.hpp>
+13 -41
View File
@@ -178,14 +178,6 @@ public:
Release();
}
void* Allocate(size_t size) {
auto* ptr = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
if (ptr == nullptr) {
LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, error {}", size, GetLastError());
}
return ptr;
}
void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) {
std::unique_lock lock{placeholder_mutex};
if (!IsNiechePlaceholder(virtual_offset, length)) {
@@ -406,10 +398,6 @@ private:
// For managarm: see https://github.com/managarm/managarm/issues/1370
#else // ^^^ Windows ^^^ vvv POSIX vvv
#ifndef MAP_NOCORE
#define MAP_NOCORE 0
#endif
#ifdef ARCHITECTURE_arm64
static void* ChooseVirtualBase(size_t virtual_size) {
@@ -434,7 +422,7 @@ static void* ChooseVirtualBase(size_t virtual_size) {
// Note: we may be able to take advantage of MAP_FIXED_NOREPLACE here.
void* map_pointer =
mmap(reinterpret_cast<void*>(hint_address), virtual_size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_NOCORE, -1, 0);
MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
// If we successfully mapped, we're done.
if (reinterpret_cast<uintptr_t>(map_pointer) == hint_address) {
@@ -454,11 +442,11 @@ static void* ChooseVirtualBase(size_t virtual_size) {
static void* ChooseVirtualBase(size_t virtual_size) {
#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__OpenBSD__) || defined(__sun__) || defined(__HAIKU__) || defined(__managarm__) || defined(__AIX__)
void* virtual_base = mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_ALIGNED_SUPER | MAP_NOCORE, -1, 0);
void* virtual_base = mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_ALIGNED_SUPER, -1, 0);
if (virtual_base != MAP_FAILED)
return virtual_base;
#endif
return mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_NOCORE, -1, 0);
return mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
}
#endif
@@ -552,13 +540,13 @@ public:
}
if (use_anon) {
LOG_WARNING(Common_Memory, "Using private mappings instead of shared ones");
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_NOCORE, -1, 0));
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0));
if (fd > 0) {
fd = -1;
close(fd);
}
} else {
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_NOCORE, fd, 0));
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
}
if (backing_base == MAP_FAILED) {
LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno));
@@ -582,14 +570,6 @@ public:
Release();
}
void* Allocate(size_t size) {
auto* ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (ptr == MAP_FAILED) {
LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, {}", size, strerror(errno));
}
return ptr;
}
void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) {
// Intersect the range with our address space.
AdjustMap(&virtual_offset, &length);
@@ -710,10 +690,12 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_)
{
#if defined(__OPENORBIS__) || defined(__managarm__)
LOG_WARNING(HW_Memory, "Platform doesn't support fastmem");
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
fallback_buffer.emplace(backing_size);
backing_base = fallback_buffer->data();
virtual_base = nullptr;
#else
// Try to allocate a fastmem arena.
// The implementation will fail with std::bad_alloc on errors.
impl = std::make_unique<HostMemory::Impl>(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize);
if (impl->Init()) {
backing_base = impl->backing_base;
@@ -724,26 +706,16 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_)
virtual_base_offset = virtual_base - impl->virtual_base;
}
} else {
LOG_WARNING(HW_Memory, "Platform can support fastmem, but can't create it");
fallback_buffer = true;
backing_base = static_cast<u8*>(impl->Allocate(backing_size));
virtual_base = nullptr;
impl.reset();
LOG_WARNING(HW_Memory, "Platform can support fastmem, but can't create it");
fallback_buffer.emplace(backing_size);
backing_base = fallback_buffer->data();
virtual_base = nullptr;
}
#endif
}
HostMemory::~HostMemory() {
#ifdef _WIN32
if (fallback_buffer) {
VirtualFree(backing_base, backing_size, MEM_RELEASE);
}
#else
if (fallback_buffer) {
munmap(backing_base, backing_size);
}
#endif
}
HostMemory::~HostMemory() = default;
HostMemory::HostMemory(HostMemory&&) noexcept = default;
+2 -1
View File
@@ -10,6 +10,7 @@
#include <optional>
#include "common/common_funcs.h"
#include "common/common_types.h"
#include "common/virtual_buffer.h"
namespace Common {
@@ -85,7 +86,7 @@ private:
u8* virtual_base{};
size_t virtual_base_offset{};
// Windows requires it for kernels whom lack proper support for some functions!
bool fallback_buffer{false};
std::optional<Common::VirtualBuffer<u8>> fallback_buffer;
};
} // namespace Common
+32 -4
View File
@@ -13,11 +13,39 @@ PageTable::PageTable() = default;
PageTable::~PageTable() noexcept = default;
void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_bits) {
auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_bits);
entries.ResizeAndClear(num_page_table_entries);
bool PageTable::BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context,
Common::ProcessAddress address) const {
out_context->next_offset = GetInteger(address);
out_context->next_page = address / page_size;
return this->ContinueTraversal(out_entry, out_context);
}
bool PageTable::ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const {
// Setup invalid defaults.
out_entry->phys_addr = 0;
out_entry->block_size = page_size;
// Validate that we can read the actual entry.
if (auto const page = context->next_page; page < entries.size()) {
// Validate that the entry is mapped.
if (auto const paddr = entries[page].addr; paddr != 0) {
// Populate the results.
out_entry->phys_addr = paddr + context->next_offset;
context->next_page += 1;
context->next_offset += page_size;
return true;
}
}
context->next_page += 1;
context->next_offset += page_size;
return false;
}
void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits) {
auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_size_in_bits);
entries.resize(num_page_table_entries);
current_address_space_width_in_bits = address_space_width_in_bits;
current_page_bits = page_bits;
page_size = 1ULL << page_size_in_bits;
}
} // namespace Common
+56 -64
View File
@@ -9,22 +9,22 @@
#include <atomic>
#include "common/common_types.h"
#include "common/sparse_large_vector.h"
#include "common/typed_address.h"
#include "common/virtual_buffer.h"
namespace Common {
enum class PageType : u8 {
/// Page is unmapped and should cause an access error.
Unmapped = 0b00,
Unmapped,
/// Page is mapped to regular memory. This is the only type you can get pointers to.
Memory = 0b01,
Memory,
/// Page is mapped to regular memory, but inaccessible from CPU fastmem and must use
/// the callbacks.
DebugMemory = 0b10,
DebugMemory,
/// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and
/// invalidation
RasterizerCachedMemory = 0b11,
RasterizerCachedMemory,
};
/**
@@ -42,86 +42,57 @@ struct PageTable {
u64 next_offset{};
};
/// Masks out bits reserved for attribute tagging.
static constexpr u64 ATTRIBUTE_MASK = ((1ULL << 44) - 1) << 12;
/// Specifies sign bit for page table entries.
static constexpr u64 SIGN_BIT = 45 + 12; // 44 bits of data + page offset
/// Number of bits reserved for attribute tagging.
/// This can be at most the guaranteed alignment of the pointers in the page table.
static constexpr int ATTRIBUTE_BITS = 2;
/**
* Atomic tuple of host pointer, page type, and block id.
* This uses the lower bits of a given pointer to store the attributes.
* Pair of host pointer and page type attribute.
* This uses the lower bits of a given pointer to store the attribute tag.
* Writing and reading the pointer attribute pair is guaranteed to be atomic for the same method
* call. In other words, they are guaranteed to be synchronized at all times.
*/
class PageEntryData {
class PageInfo {
public:
struct Data {
Data(bool marked_, PageType type_, u16 block_, u64 page_)
: marked(static_cast<u64>(marked_) & 0b1)
, type(static_cast<u64>(type_) & ((1ULL << 2) - 1))
, block(static_cast<u64>(block_) & ((1ULL << 9) - 1))
, page((page_ >> 12) & ((1ULL << 45) - 1))
, block2((static_cast<u64>(block_) >> 9) & ((1ULL << 7) - 1)) {}
u64 marked : 1;
u64 type : 2;
u64 block : 9;
u64 page : 45; // 44 bits of actual data (64 - page offset (12) - reserved (8)) + a sign bit
u64 block2 : 7;
};
[[nodiscard]] Data Raw() const noexcept {
return std::bit_cast<Data>(data_raw.load(std::memory_order_relaxed));
}
/// Returns the page pointer
[[nodiscard]] uintptr_t Pointer(bool ignored_marked = false) const noexcept {
return ExtractPointer(std::bit_cast<Data>(data_raw.load(std::memory_order_relaxed)), ignored_marked);
[[nodiscard]] uintptr_t Pointer() const noexcept {
return ExtractPointer(raw.load(std::memory_order_relaxed));
}
/// Returns the page type attribute
[[nodiscard]] PageType Type() const noexcept {
return static_cast<PageType>(std::bit_cast<Data>(data_raw.load(std::memory_order_relaxed)).type);
}
/// Returns the block identifier.
[[nodiscard]] u16 Block() const noexcept {
return ExtractBlock(std::bit_cast<Data>(data_raw.load(std::memory_order_relaxed)));
return ExtractType(raw.load(std::memory_order_relaxed));
}
/// Returns the page pointer and attribute pair, extracted from the same atomic read
[[nodiscard]] std::tuple<uintptr_t, PageType, u16> PointerTypeBlock(bool ignore_marked = false) const noexcept {
const auto non_atomic_raw = std::bit_cast<Data>(data_raw.load(std::memory_order_relaxed));
return {ExtractPointer(non_atomic_raw, ignore_marked), static_cast<PageType>(non_atomic_raw.type), ExtractBlock(non_atomic_raw)};
[[nodiscard]] std::pair<uintptr_t, PageType> PointerType() const noexcept {
const uintptr_t non_atomic_raw = raw.load(std::memory_order_relaxed);
return {ExtractPointer(non_atomic_raw), ExtractType(non_atomic_raw)};
}
/// Write page info atomically
constexpr void Store(bool marked, PageType type, u16 block, uintptr_t pointer) noexcept {
data_raw.store(std::bit_cast<u64>(Data{marked, type, block, pointer}));
/// Returns the raw representation of the page information.
/// Use ExtractPointer and ExtractType to unpack the value.
[[nodiscard]] uintptr_t Raw() const noexcept {
return raw.load(std::memory_order_relaxed);
}
constexpr void MarkRasterizerCached() noexcept {
data_raw.fetch_or(0b111);
}
constexpr void MarkDebug(u64 ptr, u16 block) noexcept {
Store(true, PageType::DebugMemory, block, ptr);
/// Write a page pointer and type pair atomically
void Store(uintptr_t pointer, PageType type) noexcept {
raw.store(pointer | uintptr_t(type));
}
/// Unpack a pointer from a page info raw representation
[[nodiscard]] static uintptr_t ExtractPointer(Data raw, bool ignore_marked = false) noexcept {
return raw.marked && !ignore_marked ? 0
// shift raw.page's fake sign bit to the actual sign bit, then sign extend
: ((s64)(raw.page << (64 - 44))) >> (64 - 44 - 12);
[[nodiscard]] static uintptr_t ExtractPointer(uintptr_t raw) noexcept {
return raw & (~uintptr_t{0} << ATTRIBUTE_BITS);
}
[[nodiscard]] static u16 ExtractBlock(Data raw) noexcept {
return static_cast<u16>(raw.block | (raw.block2 << 9));
/// Unpack a page type from a page info raw representation
[[nodiscard]] static PageType ExtractType(uintptr_t raw) noexcept {
return static_cast<PageType>(raw & ((uintptr_t{1} << ATTRIBUTE_BITS) - 1));
}
private:
std::atomic<u64> data_raw;
static_assert(sizeof(Data) == sizeof(std::atomic<u64>));
std::atomic<uintptr_t> raw;
};
PageTable();
@@ -129,8 +100,13 @@ struct PageTable {
PageTable(const PageTable&) = delete;
PageTable& operator=(const PageTable&) = delete;
PageTable(PageTable&&) noexcept = delete;
PageTable& operator=(PageTable&&) noexcept = delete;
PageTable(PageTable&&) noexcept = default;
PageTable& operator=(PageTable&&) noexcept = default;
bool BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context,
Common::ProcessAddress address) const;
bool ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const;
/**
* Resizes the page table to be able to accommodate enough pages within
@@ -145,14 +121,30 @@ struct PageTable {
return current_address_space_width_in_bits;
}
bool GetPhysicalAddress(Common::PhysicalAddress* out_phys_addr,
Common::ProcessAddress virt_addr) const {
if (virt_addr > (1ULL << this->GetAddressSpaceBits())) {
return false;
}
*out_phys_addr = entries[virt_addr / page_size].addr + GetInteger(virt_addr);
return true;
}
/// Vector of memory pointers backing each page. An entry can only be non-null if the
/// corresponding attribute element is of type `Memory`.
SparseLargeVector<PageEntryData> entries;
static_assert(sizeof(PageEntryData) == 8);
struct PageEntryData {
PageInfo ptr;
u64 block;
u64 addr;
u64 padding;
};
VirtualBuffer<PageEntryData> entries;
static_assert(sizeof(PageEntryData) == 32);
u8* fastmem_arena{};
std::size_t current_address_space_width_in_bits{};
std::size_t current_page_bits{};
std::size_t page_size{};
};
} // namespace Common
-143
View File
@@ -1,143 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
/* virtual_buffer.cpp */
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#ifdef _WIN32
#include <windows.h>
#include <mutex>
#else
#include <sys/mman.h>
#endif
#include "common/alignment.h"
#include "common/assert.h"
#include "common/sparse_large_vector.h"
namespace Common {
#ifdef _WIN32
static std::vector<std::pair<u64, u64>> vector_regions {};
// Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error
static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) {
DWORD code = info->ExceptionRecord->ExceptionCode;
u64 exception_addr = reinterpret_cast<u64>(info->ExceptionRecord->ExceptionAddress);
if (code != EXCEPTION_ACCESS_VIOLATION) {
// Not our problem
return EXCEPTION_CONTINUE_SEARCH;
}
u64 addr = 0, addr2 = 0;
for (auto region: vector_regions) {
auto addr_shifted = exception_addr >> HostPageBits;
if (region.first <= addr_shifted && addr_shifted <= region.second) {
addr = addr_shifted;
}
// Page-boundary accesses
if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ <= region.second) {
addr2 = addr_;
}
if (addr != 0 || addr2 != 0) {
break;
}
}
if (addr == 0 && addr2 == 0) {
// Not our problem
return EXCEPTION_CONTINUE_SEARCH;
}
LOG_ERROR(HW_Memory, "Accessing an unallocated region of a SparseLargeVector at {:#x}; this shouldn't happen and is likely a Dynarmic error!", exception_addr);
// Commit this region
if (addr != 0) {
if (!CommitVectorPage(addr << HostPageBits, false)) {
return EXCEPTION_CONTINUE_SEARCH;
}
}
// Commit next region if needed
if (addr2 != 0) {
if (!CommitVectorPage(addr2 << HostPageBits, false)) {
return EXCEPTION_CONTINUE_SEARCH;
}
}
return EXCEPTION_CONTINUE_EXECUTION;
}
bool CommitVectorPage(uintptr_t addr, bool write) noexcept {
MEMORY_BASIC_INFORMATION info {};
auto res = VirtualQuery(reinterpret_cast<void*>(addr), &info, sizeof(info));
if (res == 0) {
LOG_CRITICAL(HW_Memory, "Failed to query large buffer region at {:#x} with error {}, will try committing anyway", addr, GetLastError());
} else if (info.State != MEM_RESERVE) {
LOG_ERROR(HW_Memory, "Tried to commit an unreserved large buffer region at {:#x} that is not mapped or is already committed (state {:#x})", addr, info.State);
return false;
}
auto perm = write ? PAGE_READWRITE : PAGE_READONLY;
void* res2 = VirtualAlloc(reinterpret_cast<LPVOID>(addr), HostPageSize, MEM_COMMIT, perm);
if (res2 == nullptr) {
LOG_ERROR(HW_Memory, "Failed to commit large buffer region at {:#x}, error {}", addr, GetLastError());
return false;
}
return true;
}
#endif
#ifndef MAP_NOCORE
#define MAP_NOCORE 0
#endif
void* AllocateMemoryPages(std::size_t size) noexcept {
if (auto page = HostPageSize; size % page != 0) {
LOG_WARNING(HW_Memory, "Allocating unaligned large vector with size {:#x}; aligning to {} page size", size, page);
size = AlignUp(size, page);
}
#ifdef _WIN32
// We will never use this memory entirely so instead of committing it up front let's just reserve it and commit each page individually
void* base = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE);
if (base != nullptr) {
vector_regions.emplace_back(reinterpret_cast<u64>(base), reinterpret_cast<u64>(base) + size);
static std::once_flag flag;
std::call_once(flag, []() { AddVectoredExceptionHandler(1, FakePageFaultHandler); });
} else {
// Try committing everything instead??
LOG_WARNING(HW_Memory, "Failed to reserve large vector region with error {}, trying to commit instead..", GetLastError());
base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE);
}
ASSERT_MSG(base, "Failed to reserve {:#x} sized region with error {}", size, GetLastError());
#else
void* base = mmap(nullptr, size, PROT_READ, MAP_ANON | MAP_PRIVATE | MAP_NOCORE, -1, 0);
if (base == MAP_FAILED)
base = nullptr;
ASSERT_MSG(base, "Failed to allocate {:#x} sized region with error {}", size, strerror(errno));
#endif
return base;
}
void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept {
if (auto page = HostPageSize; size % page != 0) {
size = AlignUp(size, page);
}
if (!base)
return;
#ifdef _WIN32
ASSERT(VirtualFree(base, 0, MEM_RELEASE));
#else
ASSERT(munmap(base, size) == 0);
#endif
}
} // namespace Common
-194
View File
@@ -1,194 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
/* virtual_buffer.h */
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <atomic>
#include <bit>
#include <utility>
#include <vector>
#ifndef _WIN32
#include <unistd.h>
#include <sys/mman.h>
#endif
#include "common/alignment.h"
#include "common/assert.h"
namespace Common {
#ifdef _WIN32
constexpr u64 HostPageSize = 0x1000;
constexpr u64 HostPageBits = 12;
constexpr u64 HostPageMask = ~(HostPageSize - 1);
bool CommitVectorPage(uintptr_t addr, bool write) noexcept;
#else
const u64 HostPageSize = sysconf(_SC_PAGESIZE);
const u64 HostPageBits = std::countr_zero(HostPageSize);
const u64 HostPageMask = ~(HostPageSize - 1);
#endif
void* AllocateMemoryPages(std::size_t size) noexcept;
void FreeMemoryPages(void* base, std::size_t size) noexcept;
/// A large page-aligned buffer that has optimized memory usage for zero-writes.
template <typename T>
requires std::is_trivially_copyable_v<T>
class SparseLargeVector final {
public:
constexpr SparseLargeVector() = default;
explicit SparseLargeVector(std::size_t count) noexcept
: alloc_size{count * sizeof(T)}
{
base_ptr = static_cast<T*>(AllocateMemoryPages(alloc_size));
// each item in vector holds information for 64 pages
auto denom = HostPageSize * 64;
committed_pages = std::vector<std::atomic<u64>>((alloc_size + denom - 1) / denom);
}
~SparseLargeVector() noexcept {
FreeMemoryPages(base_ptr, alloc_size);
}
SparseLargeVector(const SparseLargeVector&) = delete;
SparseLargeVector& operator=(const SparseLargeVector&) = delete;
SparseLargeVector(SparseLargeVector&& other) = delete;
SparseLargeVector& operator=(SparseLargeVector&& other) = delete;
void ResizeAndClear(std::size_t count) noexcept {
if (auto const new_size = count * sizeof(T); new_size != alloc_size) {
FreeMemoryPages(base_ptr, alloc_size);
alloc_size = new_size;
base_ptr = static_cast<T*>(AllocateMemoryPages(alloc_size));
auto denom = HostPageSize * 64;
committed_pages = std::vector<std::atomic<u64>>((alloc_size + denom - 1) / denom);
}
}
/// Returns a reference to the value of the requested index and allocates memory if needed.
T& GetAndFault(std::size_t index) noexcept {
if (index > alloc_size / sizeof(T)) {
UNREACHABLE_MSG("Out of bounds RW access on SparseLargeVector @ {}", index);
}
if (!IsCommittedPage(index)) {
CommitPage(index);
}
return base_ptr[index];
}
/// Returns a reference to the value of the requested index if initialized, or will otherwise return a zero-initialized object.
const T& GetOrDefault(std::size_t index) const {
#ifdef _WIN32
if (!IsCommittedPage(index)) {
return *reinterpret_cast<const T*>(&default_val);
}
#endif
// On non-Windows, OS page table should optimize this by pointing to a zero page if unallocated.
return base_ptr[index];
}
void Set(std::size_t index, const T& value) noexcept {
if (index > alloc_size / sizeof(T)) {
LOG_CRITICAL(Common_Memory, "Out of bounds write on SparseLargeVector @ {}", index);
return;
}
if (!IsCommittedPage(index))
CommitPage(index);
base_ptr[index] = value;
}
void ZeroRegion(std::size_t start, std::size_t end_) noexcept {
u64 base = reinterpret_cast<u64>(&base_ptr[start]);
const u64 end = reinterpret_cast<u64>(&base_ptr[end_]);
const u64 end_page = AlignUp(base, HostPageSize);
const u64 first_size = (std::min)(end_page, end) - base;
if (IsCommittedPage(start / sizeof(T))) {
std::memset(reinterpret_cast<void*>(base), 0, first_size);
}
if (end <= end_page)
return;
base = end_page;
for (u64 page = base; page < end; page += HostPageSize) {
if (!IsCommittedPage((page - reinterpret_cast<u64>(base_ptr)) / sizeof(T))) {
continue;
}
std::memset(reinterpret_cast<void*>(page), 0, (std::min)( HostPageSize, end - page));
}
}
constexpr void CommitRegion(size_t index, size_t end_) {
const u64 base = static_cast<u64>(index) * sizeof(T);
const u64 end = static_cast<u64>(end_) * sizeof(T);
for (u64 page = AlignDown(base, HostPageSize); page < end; page += HostPageSize) {
if (!IsCommittedPage(page / sizeof(T))) {
CommitPage(page / sizeof(T));
}
}
}
constexpr T& GetUnchecked(size_t index) {
return base_ptr[index];
}
[[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept {
return GetOrDefault(index);
}
[[nodiscard]] constexpr const T* data() const noexcept {
return base_ptr;
}
[[nodiscard]] constexpr std::size_t size() const noexcept {
return alloc_size / sizeof(T);
}
private:
[[nodiscard]] constexpr bool IsCommittedPage(std::size_t index) const noexcept {
if (index > alloc_size / sizeof(T)) {
LOG_CRITICAL(Common_Memory, "Out of bounds access on large vector @ {}", index);
return false;
}
auto page = (index * sizeof(T)) >> HostPageBits;
auto val = committed_pages[page >> 6].load(std::memory_order_acquire);
return (val >> (page & 63)) & 1;
}
constexpr void CommitPage(std::size_t index) noexcept {
auto page_index = (index * sizeof(T)) >> HostPageBits;
auto page = reinterpret_cast<uintptr_t>(base_ptr + index) & HostPageMask;
#if defined(_WIN32)
CommitVectorPage(page, true);
#else
mprotect(reinterpret_cast<void*>(page), HostPageSize, PROT_READ | PROT_WRITE);
#endif
committed_pages[page_index >> 6].fetch_or(1ULL << (page_index & 63), std::memory_order_release);
}
std::size_t alloc_size{};
T* base_ptr{};
std::vector<std::atomic<u64>> committed_pages{};
#ifdef _WIN32
const std::array<u8, sizeof(T)> default_val{};
#endif
};
} // namespace Common
+44
View File
@@ -0,0 +1,44 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#ifdef _WIN32
#include <windows.h>
#else
#include <sys/mman.h>
#endif
#include "common/assert.h"
#include "common/virtual_buffer.h"
namespace Common {
void* AllocateMemoryPages(std::size_t size) noexcept {
#ifdef _WIN32
void* base = VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
if (base == nullptr) {
// Probably failing to reserve is less likely than failing to commit
base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE);
}
#else
void* base = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
if (base == MAP_FAILED)
base = nullptr;
#endif
ASSERT(base);
return base;
}
void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept {
if (!base)
return;
#ifdef _WIN32
ASSERT(VirtualFree(base, 0, MEM_RELEASE));
#else
ASSERT(munmap(base, size) == 0);
#endif
}
} // namespace Common
+84
View File
@@ -0,0 +1,84 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <utility>
namespace Common {
void* AllocateMemoryPages(std::size_t size) noexcept;
void FreeMemoryPages(void* base, std::size_t size) noexcept;
template <typename T>
class VirtualBuffer final {
public:
// TODO: Uncomment this and change Common::PageTable::PageInfo to be trivially constructible
// using std::atomic_ref once libc++ has support for it
// static_assert(
// std::is_trivially_constructible_v<T>,
// "T must be trivially constructible, as non-trivial constructors will not be executed "
// "with the current allocator");
constexpr VirtualBuffer() = default;
explicit VirtualBuffer(std::size_t count) noexcept
: alloc_size{count * sizeof(T)}
{
base_ptr = reinterpret_cast<T*>(AllocateMemoryPages(alloc_size));
}
~VirtualBuffer() noexcept {
FreeMemoryPages(base_ptr, alloc_size);
}
VirtualBuffer(const VirtualBuffer&) = delete;
VirtualBuffer& operator=(const VirtualBuffer&) = delete;
VirtualBuffer(VirtualBuffer&& other) noexcept
: alloc_size{std::exchange(other.alloc_size, 0)}
, base_ptr{std::exchange(other.base_ptr, nullptr)}
{}
VirtualBuffer& operator=(VirtualBuffer&& other) noexcept {
alloc_size = std::exchange(other.alloc_size, 0);
base_ptr = std::exchange(other.base_ptr, nullptr);
return *this;
}
void resize(std::size_t count) noexcept {
if (auto const new_size = count * sizeof(T); new_size != alloc_size) {
FreeMemoryPages(base_ptr, alloc_size);
alloc_size = new_size;
base_ptr = reinterpret_cast<T*>(AllocateMemoryPages(alloc_size));
}
}
[[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept {
return base_ptr[index];
}
[[nodiscard]] constexpr T& operator[](std::size_t index) noexcept {
return base_ptr[index];
}
[[nodiscard]] constexpr T* data() noexcept {
return base_ptr;
}
[[nodiscard]] constexpr const T* data() const noexcept {
return base_ptr;
}
[[nodiscard]] constexpr std::size_t size() const noexcept {
return alloc_size / sizeof(T);
}
private:
std::size_t alloc_size{};
T* base_ptr{};
};
} // namespace Common
+5 -13
View File
@@ -172,12 +172,12 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) {
if (page_table) {
constexpr size_t PageBits = 12;
constexpr size_t NumPageTableEntries = 1 << (32 - PageBits);
constexpr size_t PageLog2Stride = 5;
static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData));
// Dynarmic will not write to the page table, const_cast is safe here
config.page_table = reinterpret_cast<std::array<std::uint8_t*, NumPageTableEntries>*>(
const_cast<Common::PageTable::PageEntryData*>(page_table->entries.data()));
config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK;
config.page_table_marked_bit = 0;
config.page_table = reinterpret_cast<std::array<std::uint8_t*, NumPageTableEntries>*>(page_table->entries.data());
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
config.page_table_log2_stride = PageLog2Stride;
config.absolute_offset_page_table = true;
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
config.only_detect_misalignment_via_page_table_on_page_boundary = true;
@@ -188,13 +188,6 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) {
config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt;
config.recompile_on_exclusive_fastmem_failure = true;
if (reinterpret_cast<u64>(m_system.DeviceMemory().buffer.BackingBasePointer() +
Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) {
// Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries,
// we have to manually sign extend when our actual pointer is negative.
config.page_table_sign_extension = Common::PageTable::SIGN_BIT;
}
}
// Multi-process state
@@ -427,7 +420,6 @@ void ArmDynarmic32::SignalInterrupt(Kernel::KThread* thread) {
}
void ArmDynarmic32::ClearInstructionCache() {
m_cb->last_code_addr = u64(-1);
m_jit->ClearCache();
}
+6 -13
View File
@@ -211,12 +211,13 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s
// Memory
if (page_table) {
// Dynarmic will not write to the page table, const_cast is safe here
config.page_table = reinterpret_cast<void**>(
const_cast<Common::PageTable::PageEntryData*>(page_table->entries.data()));
constexpr size_t PageLog2Stride = 5;
static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData));
config.page_table = reinterpret_cast<void**>(page_table->entries.data());
config.page_table_address_space_bits = std::uint32_t(address_space_bits);
config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK;
config.page_table_marked_bit = 0;
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
config.page_table_log2_stride = PageLog2Stride;
config.silently_mirror_page_table = false;
config.absolute_offset_page_table = true;
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
@@ -230,13 +231,6 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s
config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt;
config.recompile_on_exclusive_fastmem_failure = true;
if (reinterpret_cast<u64>(m_system.DeviceMemory().buffer.BackingBasePointer() +
Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) {
// Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries,
// we have to manually sign extend when our actual pointer is negative.
config.page_table_sign_extension = Common::PageTable::SIGN_BIT;
}
}
// Multi-process state
@@ -453,7 +447,6 @@ void ArmDynarmic64::SignalInterrupt(Kernel::KThread* thread) {
}
void ArmDynarmic64::ClearInstructionCache() {
m_cb->last_code_addr = u64(-1);
m_jit->ClearCache();
}
+2 -5
View File
@@ -29,11 +29,8 @@ public:
template <typename T>
Common::PhysicalAddress GetPhysicalAddr(const T* ptr) const {
return GetPhysicalAddr(reinterpret_cast<uintptr_t>(ptr));
}
Common::PhysicalAddress GetPhysicalAddr(uintptr_t ptr) const {
return (ptr - reinterpret_cast<uintptr_t>(buffer.BackingBasePointer())) +
return (reinterpret_cast<uintptr_t>(ptr) -
reinterpret_cast<uintptr_t>(buffer.BackingBasePointer())) +
DramMemoryMap::Base;
}
+5 -5
View File
@@ -18,7 +18,7 @@
#include "common/common_types.h"
#include "common/range_mutex.h"
#include "common/scratch_buffer.h"
#include "common/sparse_large_vector.h"
#include "common/virtual_buffer.h"
namespace Core {
@@ -178,8 +178,8 @@ private:
u32 continuity_tracker;
u32 compressed_physical_ptr;
};
Common::SparseLargeVector<u32> compressed_device_addr;
Common::SparseLargeVector<TrackedEntry> tracked_entries;
Common::VirtualBuffer<u32> compressed_device_addr;
Common::VirtualBuffer<TrackedEntry> tracked_entries;
// Process memory interfaces
@@ -200,8 +200,8 @@ private:
return std::make_pair(asid, address);
}
constexpr void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) {
tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit);
void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) {
tracked_entries[page_index].cpu_backing_address = address | (asid.id << asid_start_bit);
}
std::array<TranslationEntry, 4> t_slot{};
+27 -22
View File
@@ -177,6 +177,17 @@ DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memo
{
impl = std::make_unique<DeviceMemoryManagerAllocator<Traits>>();
cached_pages = std::make_unique<CachedPages>();
const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS;
for (size_t i = 0; i < total_virtual; i++) {
tracked_entries[i].compressed_physical_ptr = 0;
tracked_entries[i].continuity_tracker = 1;
tracked_entries[i].cpu_backing_address = 0;
}
const size_t total_phys = 1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS);
for (size_t i = 0; i < total_phys; i++) {
compressed_device_addr[i] = 0;
}
}
template <typename Traits>
@@ -209,28 +220,26 @@ void DeviceMemoryManager<Traits>::Map(DAddr address, VAddr virtual_address, size
size_t start_page_d = address >> Memory::YUZU_PAGEBITS;
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
std::scoped_lock lk(mapping_guard);
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages);
for (size_t i = 0; i < num_pages; i++) {
const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE;
auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress));
if (ptr == nullptr) [[unlikely]] {
tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = 0;
tracked_entries[start_page_d + i].compressed_physical_ptr = 0;
continue;
}
auto phys_addr = static_cast<u32>(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U;
tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = phys_addr;
tracked_entries[start_page_d + i].compressed_physical_ptr = phys_addr;
InsertCPUBacking(start_page_d + i, new_vaddress, asid);
const u32 base_dev = compressed_device_addr[phys_addr - 1U];
const u32 new_dev = static_cast<u32>(start_page_d + i);
if (base_dev == 0) [[likely]] {
compressed_device_addr.GetAndFault(phys_addr - 1U) = new_dev;
compressed_device_addr[phys_addr - 1U] = new_dev;
continue;
}
u32 start_id = base_dev & MULTI_MASK;
if ((base_dev >> MULTI_FLAG_BITS) == 0) {
start_id = impl->multi_dev_address.Register(base_dev);
compressed_device_addr.GetAndFault(phys_addr - 1U) = MULTI_FLAG | start_id;
compressed_device_addr[phys_addr - 1U] = MULTI_FLAG | start_id;
}
impl->multi_dev_address.Register(new_dev, start_id);
}
@@ -246,26 +255,24 @@ void DeviceMemoryManager<Traits>::Unmap(DAddr address, size_t size) {
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
device_inter->InvalidateRegion(address, size);
std::scoped_lock lk(mapping_guard);
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); // should already be committed, but just in case
for (size_t i = 0; i < num_pages; i++) {
auto& entry = tracked_entries.GetUnchecked(start_page_d + i);
auto phys_addr = entry.compressed_physical_ptr;
entry.compressed_physical_ptr = 0;
entry.cpu_backing_address = 0;
auto phys_addr = tracked_entries[start_page_d + i].compressed_physical_ptr;
tracked_entries[start_page_d + i].compressed_physical_ptr = 0;
tracked_entries[start_page_d + i].cpu_backing_address = 0;
if (phys_addr != 0) [[likely]] {
u32& base_dev = compressed_device_addr.GetAndFault(phys_addr - 1U);
const u32 base_dev = compressed_device_addr[phys_addr - 1U];
if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] {
base_dev = 0;
compressed_device_addr[phys_addr - 1] = 0;
continue;
}
const auto [more_entries, new_start] = impl->multi_dev_address.Unregister(
static_cast<u32>(start_page_d + i), base_dev & MULTI_MASK);
if (!more_entries) {
base_dev = impl->multi_dev_address.ReleaseEntry(new_start);
compressed_device_addr[phys_addr - 1] =
impl->multi_dev_address.ReleaseEntry(new_start);
continue;
}
base_dev = new_start | MULTI_FLAG;
compressed_device_addr[phys_addr - 1] = new_start | MULTI_FLAG;
}
}
t_slot = {};
@@ -278,8 +285,6 @@ void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtu
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
uintptr_t last_ptr = 0;
size_t page_count = 1;
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages);
for (size_t i = num_pages; i > 0; i--) {
size_t index = i - 1;
const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE;
@@ -291,14 +296,14 @@ void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtu
page_count = 1;
}
last_ptr = new_ptr;
tracked_entries.GetUnchecked(start_page_d + index).continuity_tracker = static_cast<u32>(page_count) - 1;
tracked_entries[start_page_d + index].continuity_tracker = static_cast<u32>(page_count);
}
}
template <typename Traits>
u8* DeviceMemoryManager<Traits>::GetSpan(const DAddr src_addr, const std::size_t size) {
size_t page_index = src_addr >> page_bits;
size_t subbits = src_addr & page_mask;
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) {
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) {
return GetPointer<u8>(src_addr);
}
return nullptr;
@@ -308,7 +313,7 @@ template <typename Traits>
const u8* DeviceMemoryManager<Traits>::GetSpan(const DAddr src_addr, const std::size_t size) const {
size_t page_index = src_addr >> page_bits;
size_t subbits = src_addr & page_mask;
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) {
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) {
return GetPointer<u8>(src_addr);
}
return nullptr;
@@ -378,7 +383,7 @@ void DeviceMemoryManager<Traits>::WalkBlock(DAddr addr, std::size_t size, auto o
std::size_t page_index = addr >> Memory::YUZU_PAGEBITS;
std::size_t page_offset = addr & Memory::YUZU_PAGEMASK;
while (remaining_size) {
const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker+1);
const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker);
const std::size_t copy_amount = (std::min)((next_pages << Memory::YUZU_PAGEBITS) - page_offset, remaining_size);
const auto current_vaddr = u64((page_index << Memory::YUZU_PAGEBITS) + page_offset);
SCOPE_EXIT{
+40 -70
View File
@@ -635,36 +635,6 @@ Result KPageTableBase::CheckMemoryState(const KMemoryInfo& info, KMemoryState st
R_SUCCEED();
}
bool KPageTableBase::BeginTraversal(const Common::PageTable &impl, TraversalEntry *out_entry, TraversalContext *out_context,
Common::ProcessAddress address) const {
out_context->next_offset = GetInteger(address);
out_context->next_page = GetInteger(address) >> PageBits;
return ContinueTraversal(impl, out_entry, out_context);
}
bool KPageTableBase::ContinueTraversal(const Common::PageTable &impl, TraversalEntry *out_entry,
TraversalContext *context) const {
// Setup invalid defaults.
out_entry->phys_addr = 0;
out_entry->block_size = PageSize;
// Validate that we can read the actual entry.
if (auto const page = context->next_page; page < impl.entries.size()) {
// Validate that the entry is mapped.
if (auto const paddr = impl.entries[page].Pointer(true); paddr != 0) {
// Populate the results and return true
out_entry->phys_addr = GetInteger(m_system.DeviceMemory().GetPhysicalAddr(paddr + context->next_offset));
context->next_page += 1;
context->next_offset += PageSize;
return true;
}
}
context->next_page += 1;
context->next_offset += PageSize;
// Otherwise return false
return false;
}
Result KPageTableBase::CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr,
size_t size, KMemoryState state_mask,
KMemoryState state, KMemoryPermission perm_mask,
@@ -970,7 +940,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a
size_t tot_size = 0;
next_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), region_start);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), region_start);
next_entry.block_size =
(next_entry.block_size - (GetInteger(region_start) & (next_entry.block_size - 1)));
@@ -1006,7 +976,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a
break;
}
next_valid = ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
next_valid = impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
}
// Check the last entry.
@@ -1784,7 +1754,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
R_UNLESS(BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr),
R_UNLESS(impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr),
ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -1794,7 +1764,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_
// Iterate, adding to group as we go.
while (tot_size < size) {
R_UNLESS(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)),
R_UNLESS(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)),
ResultInvalidCurrentMemory);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -1858,7 +1828,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
if (!BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr)) {
if (!impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr)) {
return false;
}
@@ -1869,7 +1839,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr
// Iterate, comparing expected to actual.
while (tot_size < size) {
if (!ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context))) {
if (!impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context))) {
return false;
}
@@ -1926,7 +1896,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState(
// Begin a traversal.
TraversalContext context;
TraversalEntry cur_entry = {.phys_addr = 0, .block_size = 0};
R_UNLESS(BeginTraversal(impl, std::addressof(cur_entry), std::addressof(context), address),
R_UNLESS(impl.BeginTraversal(std::addressof(cur_entry), std::addressof(context), address),
ResultInvalidCurrentMemory);
// Traverse until we have enough size or we aren't contiguous any more.
@@ -1935,7 +1905,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState(
for (contig_size =
cur_entry.block_size - (GetInteger(phys_address) & (cur_entry.block_size - 1));
contig_size < size; contig_size += cur_entry.block_size) {
if (!ContinueTraversal(impl, std::addressof(cur_entry), std::addressof(context))) {
if (!impl.ContinueTraversal(std::addressof(cur_entry), std::addressof(context))) {
break;
}
if (cur_entry.phys_addr != phys_address + contig_size) {
@@ -2364,7 +2334,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out,
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(m_impl, std::addressof(next_entry), std::addressof(context), virt_addr);
m_impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Set tracking variables.
@@ -2375,7 +2345,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out,
while (true) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(m_impl, std::addressof(next_entry), std::addressof(context));
m_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
if (!traverse_valid) {
break;
}
@@ -2597,7 +2567,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre
TraversalContext context;
TraversalEntry next_entry;
ASSERT(
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address));
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address));
// Check that the physical region matches.
R_UNLESS(next_entry.phys_addr == phys_addr, ResultInvalidMemoryRegion);
@@ -2607,7 +2577,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre
next_entry.block_size - (GetInteger(phys_addr) & (next_entry.block_size - 1));
checked_size < size; checked_size += next_entry.block_size) {
// Continue the traversal.
ASSERT(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)));
ASSERT(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)));
// Check that the physical region matches.
R_UNLESS(next_entry.phys_addr == phys_addr + checked_size, ResultInvalidMemoryRegion);
@@ -3059,7 +3029,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), address);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -3071,7 +3041,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3159,7 +3129,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_address);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -3197,7 +3167,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3255,7 +3225,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@@ -3297,7 +3267,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3758,7 +3728,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -3798,7 +3768,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3852,7 +3822,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -3875,7 +3845,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -3932,7 +3902,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -3971,7 +3941,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -4027,7 +3997,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr);
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@@ -4050,7 +4020,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@@ -4119,10 +4089,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
bool traverse_valid;
// Begin traversal.
traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry),
traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry),
std::addressof(src_context), src_addr);
ASSERT(traverse_valid);
traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry),
traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry),
std::addressof(dst_context), dst_addr);
ASSERT(traverse_valid);
@@ -4157,7 +4127,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
if (ofs + cur_copy_size != size) {
if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) {
// Continue the src traversal.
traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry),
traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry),
std::addressof(src_context));
ASSERT(traverse_valid);
@@ -4168,7 +4138,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
if (cur_dst_addr + cur_min_size ==
dst_next_entry.phys_addr + dst_next_entry.block_size) {
// Continue the dst traversal.
traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry),
traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry),
std::addressof(dst_context));
ASSERT(traverse_valid);
@@ -4253,10 +4223,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
bool traverse_valid;
// Begin traversal.
traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry),
traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry),
std::addressof(src_context), src_addr);
ASSERT(traverse_valid);
traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry),
traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry),
std::addressof(dst_context), dst_addr);
ASSERT(traverse_valid);
@@ -4291,7 +4261,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
if (ofs + cur_copy_size != size) {
if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) {
// Continue the src traversal.
traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry),
traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry),
std::addressof(src_context));
ASSERT(traverse_valid);
@@ -4302,7 +4272,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
if (cur_dst_addr + cur_min_size ==
dst_next_entry.phys_addr + dst_next_entry.block_size) {
// Continue the dst traversal.
traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry),
traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry),
std::addressof(dst_context));
ASSERT(traverse_valid);
@@ -4577,7 +4547,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid = BeginTraversal(src_impl, std::addressof(next_entry),
bool traverse_valid = src_impl.BeginTraversal(std::addressof(next_entry),
std::addressof(context), aligned_src_start);
ASSERT(traverse_valid);
@@ -4627,7 +4597,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
// If the block's size was one page, we may need to continue traversal.
if (cur_block_size == 0 && aligned_src_size > PageSize) {
traverse_valid =
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
cur_block_addr = next_entry.phys_addr;
@@ -4640,7 +4610,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
while (aligned_src_start + tot_block_size < mapping_src_end) {
// Continue the traversal.
traverse_valid =
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
// Process the block.
@@ -4683,7 +4653,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
if (mapped_block_end + cur_block_size < aligned_src_end &&
cur_block_size == last_block_size) {
traverse_valid =
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
cur_block_addr = next_entry.phys_addr;
@@ -5631,7 +5601,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si
ContiguousRangeInfo(KPageTableBase& pt, KProcessAddress address, size_t size)
: m_pt(pt), m_remaining_size(size) {
// Begin a traversal.
ASSERT(m_pt.BeginTraversal(m_pt.GetImpl(), std::addressof(m_entry),
ASSERT(m_pt.GetImpl().BeginTraversal(std::addressof(m_entry),
std::addressof(m_context), address));
// Setup tracking fields.
@@ -5662,7 +5632,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si
void DetermineContiguousBlockExtents() {
// Continue traversing until we're not contiguous, or we have enough.
while (m_cur_size < m_remaining_size) {
ASSERT(m_pt.ContinueTraversal(m_pt.GetImpl(), std::addressof(m_entry),
ASSERT(m_pt.GetImpl().ContinueTraversal(std::addressof(m_entry),
std::addressof(m_context)));
// If we're not contiguous, we're done.
+1 -12
View File
@@ -370,10 +370,6 @@ private:
size_t num_pages, size_t alignment, size_t offset,
size_t guard_pages) const;
bool BeginTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* out_context,
Common::ProcessAddress address) const;
bool ContinueTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* context) const;
Result CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size,
KMemoryState state_mask, KMemoryState state,
KMemoryPermission perm_mask, KMemoryPermission perm,
@@ -478,14 +474,7 @@ private:
// Validate pre-conditions.
ASSERT(this->IsLockedByCurrentThread());
if (virt_addr > (1ULL << m_address_space_width)) {
return false;
}
*out = m_system.DeviceMemory().GetPhysicalAddr(
this->GetImpl().entries[GetInteger(virt_addr) >> PageBits].Pointer(true) + GetInteger(virt_addr));
return true;
return this->GetImpl().GetPhysicalAddress(out, virt_addr);
}
public:
+39 -42
View File
@@ -101,10 +101,8 @@ struct Memory::Impl {
}
u64 protect_bytes = 0, protect_begin = 0;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr + size) >> YUZU_PAGEBITS);
for (u64 addr = vaddr; addr < vaddr + size; addr += YUZU_PAGESIZE) {
const Common::PageType page_type = current_page_table->entries.GetUnchecked(addr >> YUZU_PAGEBITS).Type();
const Common::PageType page_type = current_page_table->entries[addr >> YUZU_PAGEBITS].ptr.Type();
switch (page_type) {
case Common::PageType::RasterizerCachedMemory:
if (protect_bytes > 0) {
@@ -125,14 +123,16 @@ struct Memory::Impl {
}
[[nodiscard]] u8* GetPointerFromRasterizerCachedMemory(u64 vaddr) const {
if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr)
return reinterpret_cast<u8*>(paddr) + vaddr;
Common::PhysicalAddress const paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr;
if (paddr)
return system.DeviceMemory().GetPointer<u8>(paddr + vaddr);
return {};
}
[[nodiscard]] u8* GetPointerFromDebugMemory(u64 vaddr) const {
if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr)
return reinterpret_cast<u8*>(paddr) + vaddr;
const Common::PhysicalAddress paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr;
if (paddr != 0)
return system.DeviceMemory().GetPointer<u8>(paddr + vaddr);
return {};
}
@@ -243,12 +243,10 @@ struct Memory::Impl {
std::size_t page_index = addr >> YUZU_PAGEBITS;
std::size_t page_offset = addr & YUZU_PAGEMASK;
bool user_accessible = true;
current_page_table->entries.CommitRegion(page_index, page_index + (size >> YUZU_PAGEBITS) + 1);
while (remaining_size != 0) {
const std::size_t copy_amount = (std::min)(std::size_t(YUZU_PAGESIZE) - page_offset, remaining_size);
const auto current_vaddr = u64((page_index << YUZU_PAGEBITS) + page_offset);
const auto [pointer, type, _] = current_page_table->entries.GetUnchecked(page_index).PointerTypeBlock();
const auto [pointer, type] = current_page_table->entries[page_index].ptr.PointerType();
switch (type) {
case Common::PageType::Unmapped: {
user_accessible = false;
@@ -299,10 +297,10 @@ struct Memory::Impl {
}
[[nodiscard]] inline const u8* GetSpan(const VAddr addr, const std::size_t size) const noexcept {
return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr;
return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr;
}
[[nodiscard]] inline u8* GetSpan(const VAddr addr, const std::size_t size) noexcept {
return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr;
return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr;
}
bool WriteBlockImpl(const Common::ProcessAddress addr, const void* buffer, const std::size_t size, bool unsafe) {
@@ -406,14 +404,11 @@ struct Memory::Impl {
// The region is at a granularity of CPU pages.
const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
const auto [pointer, type, block] = entry.PointerTypeBlock(true);
const Common::PageType page_type = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type();
if (debug) {
// Switch page type to debug if now debug
switch (type) {
switch (page_type) {
case Common::PageType::Unmapped:
ASSERT(false && "Attempted to mark unmapped pages as debug");
break;
@@ -422,14 +417,14 @@ struct Memory::Impl {
// Page is already marked.
break;
case Common::PageType::Memory:
entry.MarkDebug(pointer, block);
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::DebugMemory);
break;
default:
UNREACHABLE();
}
} else {
// Switch page type to non-debug if now non-debug
switch (type) {
switch (page_type) {
case Common::PageType::Unmapped:
ASSERT(false && "Attempted to mark unmapped pages as non-debug");
break;
@@ -438,7 +433,8 @@ struct Memory::Impl {
// Don't mess with already non-debug or rasterizer memory.
break;
case Common::PageType::DebugMemory: {
entry.Store(false, Common::PageType::Memory, block, pointer);
u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK);
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
break;
}
default:
@@ -470,10 +466,8 @@ struct Memory::Impl {
// is different). This assumes the specified GPU address region is contiguous as well.
const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
const Common::PageType page_type = entry.Type();
const Common::PageType page_type= current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type();
if (cached) {
// Switch page type to cached if now cached
switch (page_type) {
@@ -483,7 +477,7 @@ struct Memory::Impl {
break;
case Common::PageType::DebugMemory:
case Common::PageType::Memory:
entry.MarkRasterizerCached();
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::RasterizerCachedMemory);
break;
case Common::PageType::RasterizerCachedMemory:
// There can be more than one GPU region mapped per CPU region, so it's common
@@ -505,13 +499,13 @@ struct Memory::Impl {
// that this area is already unmarked as cached.
break;
case Common::PageType::RasterizerCachedMemory: {
if (auto [ptr, _, block] = entry.PointerTypeBlock(true); ptr == 0) {
if (u8* const pointer = GetPointerFromRasterizerCachedMemory(vaddr & ~YUZU_PAGEMASK); pointer == nullptr) {
// It's possible that this function has been called while updating the
// pagetable after unmapping a VMA. In that case the underlying VMA will no
// longer exist, and we should just leave the pagetable entry blank.
entry.Store(false, Common::PageType::Unmapped, block, 0);
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::Unmapped);
} else {
entry.Store(false, Common::PageType::Memory, block, ptr);
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
}
break;
}
@@ -545,18 +539,22 @@ struct Memory::Impl {
ASSERT_MSG(type != Common::PageType::Memory,
"Mapping memory page without a pointer @ {:016x}", base * YUZU_PAGESIZE);
page_table.entries.ZeroRegion(base, end);
} else {
auto current_block = block_count.fetch_add(1, std::memory_order_relaxed);
ASSERT(current_block != 65535);
page_table.entries.CommitRegion(base, end);
while (base != end) {
auto host_ptr = reinterpret_cast<u64>(system.DeviceMemory().GetPointer<u8>(target)) - (base << YUZU_PAGEBITS);;
auto& entry = page_table.entries.GetUnchecked(base);
page_table.entries[base].ptr.Store(0, type);
page_table.entries[base].addr = 0;
page_table.entries[base].block = 0;
base += 1;
}
} else {
auto orig_base = base;
while (base != end) {
auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer<u8>(target)) - (base << YUZU_PAGEBITS);
auto backing = GetInteger(target) - (base << YUZU_PAGEBITS);
page_table.entries[base].ptr.Store(host_ptr, type);
page_table.entries[base].addr = backing;
page_table.entries[base].block = orig_base << YUZU_PAGEBITS;
entry.Store(false, type, current_block, host_ptr);
ASSERT_MSG(page_table.entries[base].Pointer(),
ASSERT_MSG(page_table.entries[base].ptr.Pointer(),
"memory mapping base yield a nullptr within the table");
base += 1;
@@ -571,11 +569,11 @@ struct Memory::Impl {
vaddr &= 0xffffffffffffULL;
if (AddressSpaceContains(*current_page_table, vaddr, 1)) [[likely]] {
// Avoid adding any extra logic to this fast-path block
const auto raw = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Raw();
if (auto pointer = Common::PageTable::PageEntryData::ExtractPointer(raw); pointer) [[likely]] {
const uintptr_t raw_pointer = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Raw();
if (const uintptr_t pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) [[likely]] {
return reinterpret_cast<u8*>(pointer + vaddr);
} else {
switch (static_cast<Common::PageType>(raw.type)) {
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
case Common::PageType::Memory:
ASSERT_MSG(false, "Mapped memory page without a pointer @ {:#016x}", vaddr);
return nullptr;
@@ -775,7 +773,6 @@ struct Memory::Impl {
#else
Common::HostMemory* host_buffer{};
#endif
std::atomic<u16> block_count = 0;
};
Memory::Memory(Core::System& system_) : system{system_} {
@@ -814,7 +811,7 @@ bool Memory::IsValidVirtualAddress(const Common::ProcessAddress vaddr) const {
if (page >= page_table.entries.size()) {
return false;
}
const auto [pointer, type, _] = page_table.entries[page].PointerTypeBlock();
const auto [pointer, type] = page_table.entries[page].ptr.PointerType();
return pointer != 0 || type == Common::PageType::RasterizerCachedMemory ||
type == Common::PageType::DebugMemory;
}
@@ -371,10 +371,8 @@ EmitConfig A32AddressSpace::GetEmitConfig() {
.page_table_pointer = std::bit_cast<u64>(conf.page_table),
.page_table_address_space_bits = 32,
.page_table_pointer_mask = conf.page_table_pointer_mask,
.page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits,
.page_table_log2_stride = conf.page_table_log2_stride,
.page_table_marked_bit = conf.page_table_marked_bit,
.page_table_sign_extension = conf.page_table_sign_extension,
.silently_mirror_page_table = true,
.absolute_offset_page_table = conf.absolute_offset_page_table,
.detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table,
@@ -545,10 +545,8 @@ EmitConfig A64AddressSpace::GetEmitConfig() {
.page_table_pointer = std::bit_cast<u64>(conf.page_table),
.page_table_address_space_bits = conf.page_table_address_space_bits,
.page_table_pointer_mask = conf.page_table_pointer_mask,
.page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits,
.page_table_log2_stride = conf.page_table_log2_stride,
.page_table_marked_bit = conf.page_table_marked_bit,
.page_table_sign_extension = conf.page_table_sign_extension,
.silently_mirror_page_table = conf.silently_mirror_page_table,
.absolute_offset_page_table = conf.absolute_offset_page_table,
.detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table,
@@ -128,10 +128,8 @@ struct EmitConfig {
// Page table
u64 page_table_pointer;
std::size_t page_table_address_space_bits;
u64 page_table_pointer_mask;
int page_table_pointer_mask_bits;
std::size_t page_table_log2_stride;
std::optional<std::uint8_t> page_table_marked_bit;
std::optional<std::uint8_t> page_table_sign_extension;
bool silently_mirror_page_table;
bool absolute_offset_page_table;
u8 detect_misaligned_access_via_page_table;
@@ -273,18 +273,9 @@ std::pair<oaknut::XReg, oaknut::XReg> InlinePageTableEmitVAddrLookup(oaknut::Cod
// load x0 = *<(u8*)pagetable + index>
code.LDR(Xscratch0, Xpagetable, Xscratch0);
if (ctx.conf.page_table_marked_bit) {
code.TST(Xscratch0, 1ULL << *ctx.conf.page_table_marked_bit);
code.B(NE, *fallback);
}
if (ctx.conf.page_table_pointer_mask != 0) {
code.AND(Xscratch0, Xscratch0, ctx.conf.page_table_pointer_mask);
}
// TODO: combine this with page_table_pointer_mask
if (ctx.conf.page_table_sign_extension) {
code.SBFM(Xscratch0, Xscratch0, 0, *ctx.conf.page_table_sign_extension);
if (ctx.conf.page_table_pointer_mask_bits != 0) {
const u64 mask = u64(~u64(0)) << ctx.conf.page_table_pointer_mask_bits;
code.AND(Xscratch0, Xscratch0, mask);
}
code.CBZ(Xscratch0, *fallback);
@@ -9,7 +9,6 @@
#pragma once
#include <bit>
#include <utility>
#include "dynarmic/backend/x64/xbyak.h"
#include "dynarmic/backend/x64/a32_emit_x64.h"
@@ -79,40 +78,27 @@ Xbyak::RegExp EmitVAddrLookup(BlockOfCode& code, EmitContext& ctx, size_t bitsiz
template<>
[[maybe_unused]] Xbyak::RegExp EmitVAddrLookup<A32EmitContext>(BlockOfCode& code, A32EmitContext& ctx, size_t bitsize, Xbyak::Label& abort, Xbyak::Reg64 vaddr) {
const Xbyak::Reg64 page = ctx.reg_alloc.ScratchGpr(code);
const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table && ctx.conf.page_table_pointer_mask == 0 ? page : ctx.reg_alloc.ScratchGpr(code);
const Xbyak::Reg32 tmp = ctx.conf.absolute_offset_page_table ? page.cvt32() : ctx.reg_alloc.ScratchGpr(code).cvt32();
EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp);
EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp.cvt64());
code.mov(tmp, vaddr);
// TODO: This code assumes vaddr has been zext from 32-bits to 64-bits.
code.mov(tmp, vaddr.cvt32());
code.shr(tmp, int(page_table_const_bits));
code.shl(tmp, int(ctx.conf.page_table_log2_stride));
code.mov(page, qword[r14 + tmp.cvt64()]);
// check for marked bit, use as unmapped if marked
if (ctx.conf.page_table_marked_bit) {
code.bt(page, *ctx.conf.page_table_marked_bit);
code.jc(abort, code.T_NEAR);
}
// mask away attributes
if (ctx.conf.page_table_pointer_mask == 0) {
if (ctx.conf.page_table_pointer_mask_bits == 0) {
code.test(page, page);
} else if (std::in_range<s32>(ctx.conf.page_table_pointer_mask)) {
code.and_(page, ctx.conf.page_table_pointer_mask);
} else {
code.mov(tmp, ctx.conf.page_table_pointer_mask);
code.and_(page, tmp);
code.and_(page, ~u32(0) << ctx.conf.page_table_pointer_mask_bits);
}
if (ctx.conf.page_table_sign_extension) {
code.shl(page, *ctx.conf.page_table_sign_extension);
code.sar(page, *ctx.conf.page_table_sign_extension);
}
code.jz(abort, code.T_NEAR);
if (ctx.conf.absolute_offset_page_table) {
return page + vaddr;
}
code.mov(tmp, vaddr);
code.and_(tmp, u32(page_table_const_mask));
code.mov(tmp, vaddr.cvt32());
code.and_(tmp, static_cast<u32>(page_table_const_mask));
return page + tmp.cvt64();
}
@@ -122,7 +108,7 @@ template<>
const size_t unused_top_bits = 64 - ctx.conf.page_table_address_space_bits;
const Xbyak::Reg64 page = ctx.reg_alloc.ScratchGpr(code);
const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table && ctx.conf.page_table_pointer_mask == 0 ? page : ctx.reg_alloc.ScratchGpr(code);
const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table ? page : ctx.reg_alloc.ScratchGpr(code);
EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp);
@@ -157,26 +143,11 @@ template<>
code.shl(tmp, int(ctx.conf.page_table_log2_stride));
code.mov(page, qword[r14 + tmp]);
// check for marked bit, use as unmapped if marked
if (ctx.conf.page_table_marked_bit) {
code.bt(page, *ctx.conf.page_table_marked_bit);
code.jc(abort, code.T_NEAR);
}
// mask away attributes
if (ctx.conf.page_table_pointer_mask == 0) {
if (ctx.conf.page_table_pointer_mask_bits == 0) {
code.test(page, page);
} else if (std::in_range<s32>(ctx.conf.page_table_pointer_mask)) {
code.and_(page, ctx.conf.page_table_pointer_mask);
} else {
code.mov(tmp, ctx.conf.page_table_pointer_mask);
code.and_(page, tmp);
code.and_(page, ~u32(0) << ctx.conf.page_table_pointer_mask_bits);
}
if (ctx.conf.page_table_sign_extension) {
code.shl(page, *ctx.conf.page_table_sign_extension);
code.sar(page, *ctx.conf.page_table_sign_extension);
}
code.jz(abort, code.T_NEAR);
if (ctx.conf.absolute_offset_page_table) {
return page + vaddr;
@@ -159,23 +159,14 @@ struct UserConfig {
/// Maximum size is limited by the maximum length of a x86_64 / arm64 jump.
std::uint32_t code_cache_size = 128 * 1024 * 1024; // bytes
/// Applies a bit mask to the bits in host pointers from the page table.
/// Masks out the first N bits in host pointers from the page table.
/// The intention behind this is to allow users of Dynarmic to pack attributes in the
/// same integer and update the pointer attribute pair atomically.
/// If the configured value is ~(0b111ULL), all pointers will be forcefully aligned to 8 bytes.
std::uint64_t page_table_pointer_mask = 0;
/// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes.
std::int32_t page_table_pointer_mask_bits = 0;
/// Log2 of the size per page entry, value should be either 3 or 4
std::uint32_t page_table_log2_stride = 3;
/// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table.
/// If the bit is set to 1, Dynarmic will treat it as unmapped.
/// This bit should be included as part of `page_table_pointer_mask_bits`.
std::optional<std::uint8_t> page_table_marked_bit = std::nullopt;
/// If this value is set, Dynarmic will sign extend the page table pointer by this bit.
/// Useful for compacting bits into the page table and should be used as part of `page_table_pointer_mask`.
std::optional<std::uint8_t> page_table_sign_extension = std::nullopt;
// Log2 of the size per page entry, value should be either 3 or 4
std::size_t page_table_log2_stride = 3;
/// Select the architecture version to use.
/// There are minor behavioural differences between versions.
@@ -173,23 +173,14 @@ struct UserConfig {
/// This is only used if page_table is not nullptr.
std::uint32_t page_table_address_space_bits = 36;
/// Applies a bit mask to the bits in host pointers from the page table.
/// Masks out the first N bits in host pointers from the page table.
/// The intention behind this is to allow users of Dynarmic to pack attributes in the
/// same integer and update the pointer attribute pair atomically.
/// If the configured value is ~(0b111ULL), all pointers will be forcefully aligned to 8 bytes.
std::uint64_t page_table_pointer_mask = 0;
/// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes.
std::int32_t page_table_pointer_mask_bits = 0;
/// Log2 of the size per page entry, value should be either 3 or 4
std::uint32_t page_table_log2_stride = 3;
/// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table.
/// If the bit is set to 1, Dynarmic will treat it as unmapped.
/// This bit should be included as part of `page_table_pointer_mask`.
std::optional<std::uint8_t> page_table_marked_bit = std::nullopt;
/// If this value is set, Dynarmic will sign extend the page table pointer by this bit.
/// Useful for compacting bits into the page table and should be used as part of `page_table_pointer_mask`.
std::optional<std::uint8_t> page_table_sign_extension = std::nullopt;
// Log2 of the size per page entry, value should be either 3 or 4
std::size_t page_table_log2_stride = 3;
/// Counter-timer frequency register. The value of the register is not interpreted by
/// dynarmic.
+2 -2
View File
@@ -46,8 +46,8 @@ MemoryManager::MemoryManager(Core::System& system_, MaxwellDeviceMemoryManager&
page_table_mask = page_table_size - 1;
big_page_table_mask = big_page_table_size - 1;
big_page_table_dev.ResizeAndClear(big_page_table_size);
big_entries.resize(big_page_table_size / 32, 0);
big_page_table_dev.resize(big_page_table_size);
big_page_continuous.resize(big_page_table_size / continuous_bits, 0);
entries.resize(page_table_size / 32, 0);
}
@@ -143,7 +143,7 @@ GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr
const DAddr current_dev_addr = dev_addr + offset;
const auto index = PageEntryIndex(current_gpu_addr, true);
const u32 sub_value = static_cast<u32>(current_dev_addr >> cpu_page_bits);
big_page_table_dev.Set(index, sub_value);
big_page_table_dev[index] = sub_value;
const bool is_continuous = ([&] {
uintptr_t base_ptr{
reinterpret_cast<uintptr_t>(memory.GetPointer<u8>(current_dev_addr))};
+2 -2
View File
@@ -17,7 +17,7 @@
#include "common/multi_level_page_table.h"
#include "common/range_map.h"
#include "common/scratch_buffer.h"
#include "common/sparse_large_vector.h"
#include "common/virtual_buffer.h"
#include "video_core/invalidation_accumulator.h"
#include "video_core/cache_types.h"
#include "video_core/host1x/gpu_device_memory_manager.h"
@@ -214,7 +214,7 @@ private:
Common::MultiLevelPageTable<u32> page_table;
Common::RangeMap<GPUVAddr, PTEKind> kind_map;
Common::SparseLargeVector<u32> big_page_table_dev;
Common::VirtualBuffer<u32> big_page_table_dev;
std::vector<u64> big_page_continuous;
boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash{};