mirror of
https://git.eden-emu.dev/eden-emu/eden.git
synced 2026-09-13 06:58:13 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| da07722056 | |||
| f75828da0c |
@@ -108,8 +108,6 @@ add_library(
|
||||
settings_setting.h
|
||||
slot_vector.h
|
||||
socket_types.h
|
||||
sparse_large_vector.cpp
|
||||
sparse_large_vector.h
|
||||
spin_lock.h
|
||||
stb.cpp
|
||||
stb.h
|
||||
@@ -139,6 +137,8 @@ add_library(
|
||||
uuid.cpp
|
||||
uuid.h
|
||||
vector_math.h
|
||||
virtual_buffer.cpp
|
||||
virtual_buffer.h
|
||||
zstd_compression.cpp
|
||||
zstd_compression.h
|
||||
fs/ryujinx_compat.h fs/ryujinx_compat.cpp
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/fiber.h"
|
||||
#include "common/virtual_buffer.h"
|
||||
|
||||
#include <boost/context/detail/fcontext.hpp>
|
||||
|
||||
|
||||
+13
-41
@@ -178,14 +178,6 @@ public:
|
||||
Release();
|
||||
}
|
||||
|
||||
void* Allocate(size_t size) {
|
||||
auto* ptr = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
||||
if (ptr == nullptr) {
|
||||
LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, error {}", size, GetLastError());
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) {
|
||||
std::unique_lock lock{placeholder_mutex};
|
||||
if (!IsNiechePlaceholder(virtual_offset, length)) {
|
||||
@@ -406,10 +398,6 @@ private:
|
||||
// For managarm: see https://github.com/managarm/managarm/issues/1370
|
||||
#else // ^^^ Windows ^^^ vvv POSIX vvv
|
||||
|
||||
#ifndef MAP_NOCORE
|
||||
#define MAP_NOCORE 0
|
||||
#endif
|
||||
|
||||
#ifdef ARCHITECTURE_arm64
|
||||
|
||||
static void* ChooseVirtualBase(size_t virtual_size) {
|
||||
@@ -434,7 +422,7 @@ static void* ChooseVirtualBase(size_t virtual_size) {
|
||||
// Note: we may be able to take advantage of MAP_FIXED_NOREPLACE here.
|
||||
void* map_pointer =
|
||||
mmap(reinterpret_cast<void*>(hint_address), virtual_size, PROT_READ | PROT_WRITE,
|
||||
MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_NOCORE, -1, 0);
|
||||
MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
|
||||
|
||||
// If we successfully mapped, we're done.
|
||||
if (reinterpret_cast<uintptr_t>(map_pointer) == hint_address) {
|
||||
@@ -454,11 +442,11 @@ static void* ChooseVirtualBase(size_t virtual_size) {
|
||||
|
||||
static void* ChooseVirtualBase(size_t virtual_size) {
|
||||
#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__OpenBSD__) || defined(__sun__) || defined(__HAIKU__) || defined(__managarm__) || defined(__AIX__)
|
||||
void* virtual_base = mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_ALIGNED_SUPER | MAP_NOCORE, -1, 0);
|
||||
void* virtual_base = mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_ALIGNED_SUPER, -1, 0);
|
||||
if (virtual_base != MAP_FAILED)
|
||||
return virtual_base;
|
||||
#endif
|
||||
return mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_NOCORE, -1, 0);
|
||||
return mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -552,13 +540,13 @@ public:
|
||||
}
|
||||
if (use_anon) {
|
||||
LOG_WARNING(Common_Memory, "Using private mappings instead of shared ones");
|
||||
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_NOCORE, -1, 0));
|
||||
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0));
|
||||
if (fd > 0) {
|
||||
fd = -1;
|
||||
close(fd);
|
||||
}
|
||||
} else {
|
||||
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_NOCORE, fd, 0));
|
||||
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
|
||||
}
|
||||
if (backing_base == MAP_FAILED) {
|
||||
LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno));
|
||||
@@ -582,14 +570,6 @@ public:
|
||||
Release();
|
||||
}
|
||||
|
||||
void* Allocate(size_t size) {
|
||||
auto* ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
|
||||
if (ptr == MAP_FAILED) {
|
||||
LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, {}", size, strerror(errno));
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) {
|
||||
// Intersect the range with our address space.
|
||||
AdjustMap(&virtual_offset, &length);
|
||||
@@ -710,10 +690,12 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_)
|
||||
{
|
||||
#if defined(__OPENORBIS__) || defined(__managarm__)
|
||||
LOG_WARNING(HW_Memory, "Platform doesn't support fastmem");
|
||||
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
|
||||
fallback_buffer.emplace(backing_size);
|
||||
backing_base = fallback_buffer->data();
|
||||
virtual_base = nullptr;
|
||||
#else
|
||||
// Try to allocate a fastmem arena.
|
||||
// The implementation will fail with std::bad_alloc on errors.
|
||||
impl = std::make_unique<HostMemory::Impl>(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize);
|
||||
if (impl->Init()) {
|
||||
backing_base = impl->backing_base;
|
||||
@@ -724,26 +706,16 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_)
|
||||
virtual_base_offset = virtual_base - impl->virtual_base;
|
||||
}
|
||||
} else {
|
||||
LOG_WARNING(HW_Memory, "Platform can support fastmem, but can't create it");
|
||||
fallback_buffer = true;
|
||||
backing_base = static_cast<u8*>(impl->Allocate(backing_size));
|
||||
virtual_base = nullptr;
|
||||
impl.reset();
|
||||
LOG_WARNING(HW_Memory, "Platform can support fastmem, but can't create it");
|
||||
fallback_buffer.emplace(backing_size);
|
||||
backing_base = fallback_buffer->data();
|
||||
virtual_base = nullptr;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
HostMemory::~HostMemory() {
|
||||
#ifdef _WIN32
|
||||
if (fallback_buffer) {
|
||||
VirtualFree(backing_base, backing_size, MEM_RELEASE);
|
||||
}
|
||||
#else
|
||||
if (fallback_buffer) {
|
||||
munmap(backing_base, backing_size);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
HostMemory::~HostMemory() = default;
|
||||
|
||||
HostMemory::HostMemory(HostMemory&&) noexcept = default;
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <optional>
|
||||
#include "common/common_funcs.h"
|
||||
#include "common/common_types.h"
|
||||
#include "common/virtual_buffer.h"
|
||||
|
||||
namespace Common {
|
||||
|
||||
@@ -85,7 +86,7 @@ private:
|
||||
u8* virtual_base{};
|
||||
size_t virtual_base_offset{};
|
||||
// Windows requires it for kernels whom lack proper support for some functions!
|
||||
bool fallback_buffer{false};
|
||||
std::optional<Common::VirtualBuffer<u8>> fallback_buffer;
|
||||
};
|
||||
|
||||
} // namespace Common
|
||||
|
||||
@@ -13,11 +13,39 @@ PageTable::PageTable() = default;
|
||||
|
||||
PageTable::~PageTable() noexcept = default;
|
||||
|
||||
void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_bits) {
|
||||
auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_bits);
|
||||
entries.ResizeAndClear(num_page_table_entries);
|
||||
bool PageTable::BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context,
|
||||
Common::ProcessAddress address) const {
|
||||
out_context->next_offset = GetInteger(address);
|
||||
out_context->next_page = address / page_size;
|
||||
|
||||
return this->ContinueTraversal(out_entry, out_context);
|
||||
}
|
||||
|
||||
bool PageTable::ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const {
|
||||
// Setup invalid defaults.
|
||||
out_entry->phys_addr = 0;
|
||||
out_entry->block_size = page_size;
|
||||
// Validate that we can read the actual entry.
|
||||
if (auto const page = context->next_page; page < entries.size()) {
|
||||
// Validate that the entry is mapped.
|
||||
if (auto const paddr = entries[page].addr; paddr != 0) {
|
||||
// Populate the results.
|
||||
out_entry->phys_addr = paddr + context->next_offset;
|
||||
context->next_page += 1;
|
||||
context->next_offset += page_size;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
context->next_page += 1;
|
||||
context->next_offset += page_size;
|
||||
return false;
|
||||
}
|
||||
|
||||
void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits) {
|
||||
auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_size_in_bits);
|
||||
entries.resize(num_page_table_entries);
|
||||
current_address_space_width_in_bits = address_space_width_in_bits;
|
||||
current_page_bits = page_bits;
|
||||
page_size = 1ULL << page_size_in_bits;
|
||||
}
|
||||
|
||||
} // namespace Common
|
||||
|
||||
+56
-64
@@ -9,22 +9,22 @@
|
||||
#include <atomic>
|
||||
|
||||
#include "common/common_types.h"
|
||||
#include "common/sparse_large_vector.h"
|
||||
#include "common/typed_address.h"
|
||||
#include "common/virtual_buffer.h"
|
||||
|
||||
namespace Common {
|
||||
|
||||
enum class PageType : u8 {
|
||||
/// Page is unmapped and should cause an access error.
|
||||
Unmapped = 0b00,
|
||||
Unmapped,
|
||||
/// Page is mapped to regular memory. This is the only type you can get pointers to.
|
||||
Memory = 0b01,
|
||||
Memory,
|
||||
/// Page is mapped to regular memory, but inaccessible from CPU fastmem and must use
|
||||
/// the callbacks.
|
||||
DebugMemory = 0b10,
|
||||
DebugMemory,
|
||||
/// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and
|
||||
/// invalidation
|
||||
RasterizerCachedMemory = 0b11,
|
||||
RasterizerCachedMemory,
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -42,86 +42,57 @@ struct PageTable {
|
||||
u64 next_offset{};
|
||||
};
|
||||
|
||||
/// Masks out bits reserved for attribute tagging.
|
||||
static constexpr u64 ATTRIBUTE_MASK = ((1ULL << 44) - 1) << 12;
|
||||
|
||||
/// Specifies sign bit for page table entries.
|
||||
static constexpr u64 SIGN_BIT = 45 + 12; // 44 bits of data + page offset
|
||||
/// Number of bits reserved for attribute tagging.
|
||||
/// This can be at most the guaranteed alignment of the pointers in the page table.
|
||||
static constexpr int ATTRIBUTE_BITS = 2;
|
||||
|
||||
/**
|
||||
* Atomic tuple of host pointer, page type, and block id.
|
||||
* This uses the lower bits of a given pointer to store the attributes.
|
||||
* Pair of host pointer and page type attribute.
|
||||
* This uses the lower bits of a given pointer to store the attribute tag.
|
||||
* Writing and reading the pointer attribute pair is guaranteed to be atomic for the same method
|
||||
* call. In other words, they are guaranteed to be synchronized at all times.
|
||||
*/
|
||||
class PageEntryData {
|
||||
class PageInfo {
|
||||
public:
|
||||
struct Data {
|
||||
Data(bool marked_, PageType type_, u16 block_, u64 page_)
|
||||
: marked(static_cast<u64>(marked_) & 0b1)
|
||||
, type(static_cast<u64>(type_) & ((1ULL << 2) - 1))
|
||||
, block(static_cast<u64>(block_) & ((1ULL << 9) - 1))
|
||||
, page((page_ >> 12) & ((1ULL << 45) - 1))
|
||||
, block2((static_cast<u64>(block_) >> 9) & ((1ULL << 7) - 1)) {}
|
||||
u64 marked : 1;
|
||||
u64 type : 2;
|
||||
u64 block : 9;
|
||||
u64 page : 45; // 44 bits of actual data (64 - page offset (12) - reserved (8)) + a sign bit
|
||||
u64 block2 : 7;
|
||||
};
|
||||
|
||||
[[nodiscard]] Data Raw() const noexcept {
|
||||
return std::bit_cast<Data>(data_raw.load(std::memory_order_relaxed));
|
||||
}
|
||||
|
||||
/// Returns the page pointer
|
||||
[[nodiscard]] uintptr_t Pointer(bool ignored_marked = false) const noexcept {
|
||||
return ExtractPointer(std::bit_cast<Data>(data_raw.load(std::memory_order_relaxed)), ignored_marked);
|
||||
[[nodiscard]] uintptr_t Pointer() const noexcept {
|
||||
return ExtractPointer(raw.load(std::memory_order_relaxed));
|
||||
}
|
||||
|
||||
/// Returns the page type attribute
|
||||
[[nodiscard]] PageType Type() const noexcept {
|
||||
return static_cast<PageType>(std::bit_cast<Data>(data_raw.load(std::memory_order_relaxed)).type);
|
||||
}
|
||||
|
||||
/// Returns the block identifier.
|
||||
[[nodiscard]] u16 Block() const noexcept {
|
||||
return ExtractBlock(std::bit_cast<Data>(data_raw.load(std::memory_order_relaxed)));
|
||||
return ExtractType(raw.load(std::memory_order_relaxed));
|
||||
}
|
||||
|
||||
/// Returns the page pointer and attribute pair, extracted from the same atomic read
|
||||
[[nodiscard]] std::tuple<uintptr_t, PageType, u16> PointerTypeBlock(bool ignore_marked = false) const noexcept {
|
||||
const auto non_atomic_raw = std::bit_cast<Data>(data_raw.load(std::memory_order_relaxed));
|
||||
return {ExtractPointer(non_atomic_raw, ignore_marked), static_cast<PageType>(non_atomic_raw.type), ExtractBlock(non_atomic_raw)};
|
||||
[[nodiscard]] std::pair<uintptr_t, PageType> PointerType() const noexcept {
|
||||
const uintptr_t non_atomic_raw = raw.load(std::memory_order_relaxed);
|
||||
return {ExtractPointer(non_atomic_raw), ExtractType(non_atomic_raw)};
|
||||
}
|
||||
|
||||
/// Write page info atomically
|
||||
constexpr void Store(bool marked, PageType type, u16 block, uintptr_t pointer) noexcept {
|
||||
data_raw.store(std::bit_cast<u64>(Data{marked, type, block, pointer}));
|
||||
/// Returns the raw representation of the page information.
|
||||
/// Use ExtractPointer and ExtractType to unpack the value.
|
||||
[[nodiscard]] uintptr_t Raw() const noexcept {
|
||||
return raw.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
constexpr void MarkRasterizerCached() noexcept {
|
||||
data_raw.fetch_or(0b111);
|
||||
}
|
||||
|
||||
constexpr void MarkDebug(u64 ptr, u16 block) noexcept {
|
||||
Store(true, PageType::DebugMemory, block, ptr);
|
||||
/// Write a page pointer and type pair atomically
|
||||
void Store(uintptr_t pointer, PageType type) noexcept {
|
||||
raw.store(pointer | uintptr_t(type));
|
||||
}
|
||||
|
||||
/// Unpack a pointer from a page info raw representation
|
||||
[[nodiscard]] static uintptr_t ExtractPointer(Data raw, bool ignore_marked = false) noexcept {
|
||||
return raw.marked && !ignore_marked ? 0
|
||||
// shift raw.page's fake sign bit to the actual sign bit, then sign extend
|
||||
: ((s64)(raw.page << (64 - 44))) >> (64 - 44 - 12);
|
||||
[[nodiscard]] static uintptr_t ExtractPointer(uintptr_t raw) noexcept {
|
||||
return raw & (~uintptr_t{0} << ATTRIBUTE_BITS);
|
||||
}
|
||||
|
||||
[[nodiscard]] static u16 ExtractBlock(Data raw) noexcept {
|
||||
return static_cast<u16>(raw.block | (raw.block2 << 9));
|
||||
/// Unpack a page type from a page info raw representation
|
||||
[[nodiscard]] static PageType ExtractType(uintptr_t raw) noexcept {
|
||||
return static_cast<PageType>(raw & ((uintptr_t{1} << ATTRIBUTE_BITS) - 1));
|
||||
}
|
||||
|
||||
private:
|
||||
std::atomic<u64> data_raw;
|
||||
static_assert(sizeof(Data) == sizeof(std::atomic<u64>));
|
||||
std::atomic<uintptr_t> raw;
|
||||
};
|
||||
|
||||
PageTable();
|
||||
@@ -129,8 +100,13 @@ struct PageTable {
|
||||
|
||||
PageTable(const PageTable&) = delete;
|
||||
PageTable& operator=(const PageTable&) = delete;
|
||||
PageTable(PageTable&&) noexcept = delete;
|
||||
PageTable& operator=(PageTable&&) noexcept = delete;
|
||||
|
||||
PageTable(PageTable&&) noexcept = default;
|
||||
PageTable& operator=(PageTable&&) noexcept = default;
|
||||
|
||||
bool BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context,
|
||||
Common::ProcessAddress address) const;
|
||||
bool ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const;
|
||||
|
||||
/**
|
||||
* Resizes the page table to be able to accommodate enough pages within
|
||||
@@ -145,14 +121,30 @@ struct PageTable {
|
||||
return current_address_space_width_in_bits;
|
||||
}
|
||||
|
||||
bool GetPhysicalAddress(Common::PhysicalAddress* out_phys_addr,
|
||||
Common::ProcessAddress virt_addr) const {
|
||||
if (virt_addr > (1ULL << this->GetAddressSpaceBits())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
*out_phys_addr = entries[virt_addr / page_size].addr + GetInteger(virt_addr);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Vector of memory pointers backing each page. An entry can only be non-null if the
|
||||
/// corresponding attribute element is of type `Memory`.
|
||||
SparseLargeVector<PageEntryData> entries;
|
||||
static_assert(sizeof(PageEntryData) == 8);
|
||||
struct PageEntryData {
|
||||
PageInfo ptr;
|
||||
u64 block;
|
||||
u64 addr;
|
||||
u64 padding;
|
||||
};
|
||||
VirtualBuffer<PageEntryData> entries;
|
||||
static_assert(sizeof(PageEntryData) == 32);
|
||||
|
||||
u8* fastmem_arena{};
|
||||
std::size_t current_address_space_width_in_bits{};
|
||||
std::size_t current_page_bits{};
|
||||
std::size_t page_size{};
|
||||
};
|
||||
|
||||
} // namespace Common
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
/* virtual_buffer.cpp */
|
||||
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#include <mutex>
|
||||
#else
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
|
||||
#include "common/alignment.h"
|
||||
#include "common/assert.h"
|
||||
#include "common/sparse_large_vector.h"
|
||||
|
||||
namespace Common {
|
||||
|
||||
#ifdef _WIN32
|
||||
static std::vector<std::pair<u64, u64>> vector_regions {};
|
||||
|
||||
// Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error
|
||||
static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) {
|
||||
DWORD code = info->ExceptionRecord->ExceptionCode;
|
||||
u64 exception_addr = reinterpret_cast<u64>(info->ExceptionRecord->ExceptionAddress);
|
||||
|
||||
if (code != EXCEPTION_ACCESS_VIOLATION) {
|
||||
// Not our problem
|
||||
return EXCEPTION_CONTINUE_SEARCH;
|
||||
}
|
||||
|
||||
u64 addr = 0, addr2 = 0;
|
||||
|
||||
for (auto region: vector_regions) {
|
||||
auto addr_shifted = exception_addr >> HostPageBits;
|
||||
if (region.first <= addr_shifted && addr_shifted <= region.second) {
|
||||
addr = addr_shifted;
|
||||
}
|
||||
|
||||
// Page-boundary accesses
|
||||
if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ <= region.second) {
|
||||
addr2 = addr_;
|
||||
}
|
||||
|
||||
if (addr != 0 || addr2 != 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (addr == 0 && addr2 == 0) {
|
||||
// Not our problem
|
||||
return EXCEPTION_CONTINUE_SEARCH;
|
||||
}
|
||||
|
||||
LOG_ERROR(HW_Memory, "Accessing an unallocated region of a SparseLargeVector at {:#x}; this shouldn't happen and is likely a Dynarmic error!", exception_addr);
|
||||
|
||||
// Commit this region
|
||||
if (addr != 0) {
|
||||
if (!CommitVectorPage(addr << HostPageBits, false)) {
|
||||
return EXCEPTION_CONTINUE_SEARCH;
|
||||
}
|
||||
}
|
||||
// Commit next region if needed
|
||||
if (addr2 != 0) {
|
||||
if (!CommitVectorPage(addr2 << HostPageBits, false)) {
|
||||
return EXCEPTION_CONTINUE_SEARCH;
|
||||
}
|
||||
}
|
||||
|
||||
return EXCEPTION_CONTINUE_EXECUTION;
|
||||
}
|
||||
|
||||
bool CommitVectorPage(uintptr_t addr, bool write) noexcept {
|
||||
MEMORY_BASIC_INFORMATION info {};
|
||||
auto res = VirtualQuery(reinterpret_cast<void*>(addr), &info, sizeof(info));
|
||||
if (res == 0) {
|
||||
LOG_CRITICAL(HW_Memory, "Failed to query large buffer region at {:#x} with error {}, will try committing anyway", addr, GetLastError());
|
||||
} else if (info.State != MEM_RESERVE) {
|
||||
LOG_ERROR(HW_Memory, "Tried to commit an unreserved large buffer region at {:#x} that is not mapped or is already committed (state {:#x})", addr, info.State);
|
||||
return false;
|
||||
}
|
||||
|
||||
auto perm = write ? PAGE_READWRITE : PAGE_READONLY;
|
||||
void* res2 = VirtualAlloc(reinterpret_cast<LPVOID>(addr), HostPageSize, MEM_COMMIT, perm);
|
||||
if (res2 == nullptr) {
|
||||
LOG_ERROR(HW_Memory, "Failed to commit large buffer region at {:#x}, error {}", addr, GetLastError());
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef MAP_NOCORE
|
||||
#define MAP_NOCORE 0
|
||||
#endif
|
||||
|
||||
void* AllocateMemoryPages(std::size_t size) noexcept {
|
||||
if (auto page = HostPageSize; size % page != 0) {
|
||||
LOG_WARNING(HW_Memory, "Allocating unaligned large vector with size {:#x}; aligning to {} page size", size, page);
|
||||
size = AlignUp(size, page);
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
// We will never use this memory entirely so instead of committing it up front let's just reserve it and commit each page individually
|
||||
void* base = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE);
|
||||
|
||||
if (base != nullptr) {
|
||||
vector_regions.emplace_back(reinterpret_cast<u64>(base), reinterpret_cast<u64>(base) + size);
|
||||
|
||||
static std::once_flag flag;
|
||||
std::call_once(flag, []() { AddVectoredExceptionHandler(1, FakePageFaultHandler); });
|
||||
} else {
|
||||
// Try committing everything instead??
|
||||
LOG_WARNING(HW_Memory, "Failed to reserve large vector region with error {}, trying to commit instead..", GetLastError());
|
||||
base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE);
|
||||
}
|
||||
ASSERT_MSG(base, "Failed to reserve {:#x} sized region with error {}", size, GetLastError());
|
||||
#else
|
||||
void* base = mmap(nullptr, size, PROT_READ, MAP_ANON | MAP_PRIVATE | MAP_NOCORE, -1, 0);
|
||||
if (base == MAP_FAILED)
|
||||
base = nullptr;
|
||||
ASSERT_MSG(base, "Failed to allocate {:#x} sized region with error {}", size, strerror(errno));
|
||||
#endif
|
||||
return base;
|
||||
}
|
||||
|
||||
void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept {
|
||||
if (auto page = HostPageSize; size % page != 0) {
|
||||
size = AlignUp(size, page);
|
||||
}
|
||||
if (!base)
|
||||
return;
|
||||
#ifdef _WIN32
|
||||
ASSERT(VirtualFree(base, 0, MEM_RELEASE));
|
||||
#else
|
||||
ASSERT(munmap(base, size) == 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace Common
|
||||
@@ -1,194 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
/* virtual_buffer.h */
|
||||
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <bit>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#ifndef _WIN32
|
||||
#include <unistd.h>
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
|
||||
#include "common/alignment.h"
|
||||
#include "common/assert.h"
|
||||
|
||||
namespace Common {
|
||||
|
||||
#ifdef _WIN32
|
||||
constexpr u64 HostPageSize = 0x1000;
|
||||
constexpr u64 HostPageBits = 12;
|
||||
constexpr u64 HostPageMask = ~(HostPageSize - 1);
|
||||
bool CommitVectorPage(uintptr_t addr, bool write) noexcept;
|
||||
#else
|
||||
const u64 HostPageSize = sysconf(_SC_PAGESIZE);
|
||||
const u64 HostPageBits = std::countr_zero(HostPageSize);
|
||||
const u64 HostPageMask = ~(HostPageSize - 1);
|
||||
#endif
|
||||
|
||||
void* AllocateMemoryPages(std::size_t size) noexcept;
|
||||
void FreeMemoryPages(void* base, std::size_t size) noexcept;
|
||||
|
||||
/// A large page-aligned buffer that has optimized memory usage for zero-writes.
|
||||
template <typename T>
|
||||
requires std::is_trivially_copyable_v<T>
|
||||
class SparseLargeVector final {
|
||||
public:
|
||||
constexpr SparseLargeVector() = default;
|
||||
|
||||
explicit SparseLargeVector(std::size_t count) noexcept
|
||||
: alloc_size{count * sizeof(T)}
|
||||
{
|
||||
base_ptr = static_cast<T*>(AllocateMemoryPages(alloc_size));
|
||||
|
||||
// each item in vector holds information for 64 pages
|
||||
auto denom = HostPageSize * 64;
|
||||
committed_pages = std::vector<std::atomic<u64>>((alloc_size + denom - 1) / denom);
|
||||
}
|
||||
|
||||
~SparseLargeVector() noexcept {
|
||||
FreeMemoryPages(base_ptr, alloc_size);
|
||||
}
|
||||
|
||||
SparseLargeVector(const SparseLargeVector&) = delete;
|
||||
SparseLargeVector& operator=(const SparseLargeVector&) = delete;
|
||||
SparseLargeVector(SparseLargeVector&& other) = delete;
|
||||
SparseLargeVector& operator=(SparseLargeVector&& other) = delete;
|
||||
|
||||
void ResizeAndClear(std::size_t count) noexcept {
|
||||
if (auto const new_size = count * sizeof(T); new_size != alloc_size) {
|
||||
FreeMemoryPages(base_ptr, alloc_size);
|
||||
alloc_size = new_size;
|
||||
base_ptr = static_cast<T*>(AllocateMemoryPages(alloc_size));
|
||||
|
||||
auto denom = HostPageSize * 64;
|
||||
committed_pages = std::vector<std::atomic<u64>>((alloc_size + denom - 1) / denom);
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a reference to the value of the requested index and allocates memory if needed.
|
||||
T& GetAndFault(std::size_t index) noexcept {
|
||||
if (index > alloc_size / sizeof(T)) {
|
||||
UNREACHABLE_MSG("Out of bounds RW access on SparseLargeVector @ {}", index);
|
||||
}
|
||||
|
||||
if (!IsCommittedPage(index)) {
|
||||
CommitPage(index);
|
||||
}
|
||||
return base_ptr[index];
|
||||
}
|
||||
|
||||
/// Returns a reference to the value of the requested index if initialized, or will otherwise return a zero-initialized object.
|
||||
const T& GetOrDefault(std::size_t index) const {
|
||||
#ifdef _WIN32
|
||||
if (!IsCommittedPage(index)) {
|
||||
return *reinterpret_cast<const T*>(&default_val);
|
||||
}
|
||||
#endif
|
||||
// On non-Windows, OS page table should optimize this by pointing to a zero page if unallocated.
|
||||
return base_ptr[index];
|
||||
}
|
||||
|
||||
void Set(std::size_t index, const T& value) noexcept {
|
||||
if (index > alloc_size / sizeof(T)) {
|
||||
LOG_CRITICAL(Common_Memory, "Out of bounds write on SparseLargeVector @ {}", index);
|
||||
return;
|
||||
}
|
||||
if (!IsCommittedPage(index))
|
||||
CommitPage(index);
|
||||
base_ptr[index] = value;
|
||||
}
|
||||
|
||||
void ZeroRegion(std::size_t start, std::size_t end_) noexcept {
|
||||
u64 base = reinterpret_cast<u64>(&base_ptr[start]);
|
||||
const u64 end = reinterpret_cast<u64>(&base_ptr[end_]);
|
||||
|
||||
const u64 end_page = AlignUp(base, HostPageSize);
|
||||
const u64 first_size = (std::min)(end_page, end) - base;
|
||||
|
||||
if (IsCommittedPage(start / sizeof(T))) {
|
||||
std::memset(reinterpret_cast<void*>(base), 0, first_size);
|
||||
}
|
||||
|
||||
if (end <= end_page)
|
||||
return;
|
||||
|
||||
base = end_page;
|
||||
|
||||
for (u64 page = base; page < end; page += HostPageSize) {
|
||||
if (!IsCommittedPage((page - reinterpret_cast<u64>(base_ptr)) / sizeof(T))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::memset(reinterpret_cast<void*>(page), 0, (std::min)( HostPageSize, end - page));
|
||||
}
|
||||
}
|
||||
|
||||
constexpr void CommitRegion(size_t index, size_t end_) {
|
||||
const u64 base = static_cast<u64>(index) * sizeof(T);
|
||||
const u64 end = static_cast<u64>(end_) * sizeof(T);
|
||||
|
||||
for (u64 page = AlignDown(base, HostPageSize); page < end; page += HostPageSize) {
|
||||
if (!IsCommittedPage(page / sizeof(T))) {
|
||||
CommitPage(page / sizeof(T));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
constexpr T& GetUnchecked(size_t index) {
|
||||
return base_ptr[index];
|
||||
}
|
||||
|
||||
[[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept {
|
||||
return GetOrDefault(index);
|
||||
}
|
||||
|
||||
[[nodiscard]] constexpr const T* data() const noexcept {
|
||||
return base_ptr;
|
||||
}
|
||||
|
||||
[[nodiscard]] constexpr std::size_t size() const noexcept {
|
||||
return alloc_size / sizeof(T);
|
||||
}
|
||||
|
||||
private:
|
||||
[[nodiscard]] constexpr bool IsCommittedPage(std::size_t index) const noexcept {
|
||||
if (index > alloc_size / sizeof(T)) {
|
||||
LOG_CRITICAL(Common_Memory, "Out of bounds access on large vector @ {}", index);
|
||||
return false;
|
||||
}
|
||||
|
||||
auto page = (index * sizeof(T)) >> HostPageBits;
|
||||
auto val = committed_pages[page >> 6].load(std::memory_order_acquire);
|
||||
return (val >> (page & 63)) & 1;
|
||||
}
|
||||
|
||||
constexpr void CommitPage(std::size_t index) noexcept {
|
||||
auto page_index = (index * sizeof(T)) >> HostPageBits;
|
||||
auto page = reinterpret_cast<uintptr_t>(base_ptr + index) & HostPageMask;
|
||||
#if defined(_WIN32)
|
||||
CommitVectorPage(page, true);
|
||||
#else
|
||||
mprotect(reinterpret_cast<void*>(page), HostPageSize, PROT_READ | PROT_WRITE);
|
||||
#endif
|
||||
|
||||
committed_pages[page_index >> 6].fetch_or(1ULL << (page_index & 63), std::memory_order_release);
|
||||
}
|
||||
|
||||
std::size_t alloc_size{};
|
||||
T* base_ptr{};
|
||||
|
||||
std::vector<std::atomic<u64>> committed_pages{};
|
||||
#ifdef _WIN32
|
||||
const std::array<u8, sizeof(T)> default_val{};
|
||||
#endif
|
||||
};
|
||||
|
||||
} // namespace Common
|
||||
@@ -0,0 +1,44 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "common/virtual_buffer.h"
|
||||
|
||||
namespace Common {
|
||||
|
||||
void* AllocateMemoryPages(std::size_t size) noexcept {
|
||||
#ifdef _WIN32
|
||||
void* base = VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
|
||||
if (base == nullptr) {
|
||||
// Probably failing to reserve is less likely than failing to commit
|
||||
base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE);
|
||||
}
|
||||
#else
|
||||
void* base = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
|
||||
if (base == MAP_FAILED)
|
||||
base = nullptr;
|
||||
#endif
|
||||
ASSERT(base);
|
||||
return base;
|
||||
}
|
||||
|
||||
void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept {
|
||||
if (!base)
|
||||
return;
|
||||
#ifdef _WIN32
|
||||
ASSERT(VirtualFree(base, 0, MEM_RELEASE));
|
||||
#else
|
||||
ASSERT(munmap(base, size) == 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace Common
|
||||
@@ -0,0 +1,84 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace Common {
|
||||
|
||||
void* AllocateMemoryPages(std::size_t size) noexcept;
|
||||
void FreeMemoryPages(void* base, std::size_t size) noexcept;
|
||||
|
||||
template <typename T>
|
||||
class VirtualBuffer final {
|
||||
public:
|
||||
// TODO: Uncomment this and change Common::PageTable::PageInfo to be trivially constructible
|
||||
// using std::atomic_ref once libc++ has support for it
|
||||
// static_assert(
|
||||
// std::is_trivially_constructible_v<T>,
|
||||
// "T must be trivially constructible, as non-trivial constructors will not be executed "
|
||||
// "with the current allocator");
|
||||
|
||||
constexpr VirtualBuffer() = default;
|
||||
explicit VirtualBuffer(std::size_t count) noexcept
|
||||
: alloc_size{count * sizeof(T)}
|
||||
{
|
||||
base_ptr = reinterpret_cast<T*>(AllocateMemoryPages(alloc_size));
|
||||
}
|
||||
|
||||
~VirtualBuffer() noexcept {
|
||||
FreeMemoryPages(base_ptr, alloc_size);
|
||||
}
|
||||
|
||||
VirtualBuffer(const VirtualBuffer&) = delete;
|
||||
VirtualBuffer& operator=(const VirtualBuffer&) = delete;
|
||||
|
||||
VirtualBuffer(VirtualBuffer&& other) noexcept
|
||||
: alloc_size{std::exchange(other.alloc_size, 0)}
|
||||
, base_ptr{std::exchange(other.base_ptr, nullptr)}
|
||||
{}
|
||||
|
||||
VirtualBuffer& operator=(VirtualBuffer&& other) noexcept {
|
||||
alloc_size = std::exchange(other.alloc_size, 0);
|
||||
base_ptr = std::exchange(other.base_ptr, nullptr);
|
||||
return *this;
|
||||
}
|
||||
|
||||
void resize(std::size_t count) noexcept {
|
||||
if (auto const new_size = count * sizeof(T); new_size != alloc_size) {
|
||||
FreeMemoryPages(base_ptr, alloc_size);
|
||||
alloc_size = new_size;
|
||||
base_ptr = reinterpret_cast<T*>(AllocateMemoryPages(alloc_size));
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept {
|
||||
return base_ptr[index];
|
||||
}
|
||||
|
||||
[[nodiscard]] constexpr T& operator[](std::size_t index) noexcept {
|
||||
return base_ptr[index];
|
||||
}
|
||||
|
||||
[[nodiscard]] constexpr T* data() noexcept {
|
||||
return base_ptr;
|
||||
}
|
||||
|
||||
[[nodiscard]] constexpr const T* data() const noexcept {
|
||||
return base_ptr;
|
||||
}
|
||||
|
||||
[[nodiscard]] constexpr std::size_t size() const noexcept {
|
||||
return alloc_size / sizeof(T);
|
||||
}
|
||||
|
||||
private:
|
||||
std::size_t alloc_size{};
|
||||
T* base_ptr{};
|
||||
};
|
||||
|
||||
} // namespace Common
|
||||
@@ -172,12 +172,12 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) {
|
||||
if (page_table) {
|
||||
constexpr size_t PageBits = 12;
|
||||
constexpr size_t NumPageTableEntries = 1 << (32 - PageBits);
|
||||
constexpr size_t PageLog2Stride = 5;
|
||||
static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData));
|
||||
|
||||
// Dynarmic will not write to the page table, const_cast is safe here
|
||||
config.page_table = reinterpret_cast<std::array<std::uint8_t*, NumPageTableEntries>*>(
|
||||
const_cast<Common::PageTable::PageEntryData*>(page_table->entries.data()));
|
||||
config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK;
|
||||
config.page_table_marked_bit = 0;
|
||||
config.page_table = reinterpret_cast<std::array<std::uint8_t*, NumPageTableEntries>*>(page_table->entries.data());
|
||||
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
|
||||
config.page_table_log2_stride = PageLog2Stride;
|
||||
config.absolute_offset_page_table = true;
|
||||
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
|
||||
config.only_detect_misalignment_via_page_table_on_page_boundary = true;
|
||||
@@ -188,13 +188,6 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) {
|
||||
|
||||
config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt;
|
||||
config.recompile_on_exclusive_fastmem_failure = true;
|
||||
|
||||
if (reinterpret_cast<u64>(m_system.DeviceMemory().buffer.BackingBasePointer() +
|
||||
Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) {
|
||||
// Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries,
|
||||
// we have to manually sign extend when our actual pointer is negative.
|
||||
config.page_table_sign_extension = Common::PageTable::SIGN_BIT;
|
||||
}
|
||||
}
|
||||
|
||||
// Multi-process state
|
||||
@@ -427,7 +420,6 @@ void ArmDynarmic32::SignalInterrupt(Kernel::KThread* thread) {
|
||||
}
|
||||
|
||||
void ArmDynarmic32::ClearInstructionCache() {
|
||||
m_cb->last_code_addr = u64(-1);
|
||||
m_jit->ClearCache();
|
||||
}
|
||||
|
||||
|
||||
@@ -211,12 +211,13 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s
|
||||
|
||||
// Memory
|
||||
if (page_table) {
|
||||
// Dynarmic will not write to the page table, const_cast is safe here
|
||||
config.page_table = reinterpret_cast<void**>(
|
||||
const_cast<Common::PageTable::PageEntryData*>(page_table->entries.data()));
|
||||
constexpr size_t PageLog2Stride = 5;
|
||||
static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData));
|
||||
|
||||
config.page_table = reinterpret_cast<void**>(page_table->entries.data());
|
||||
config.page_table_address_space_bits = std::uint32_t(address_space_bits);
|
||||
config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK;
|
||||
config.page_table_marked_bit = 0;
|
||||
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
|
||||
config.page_table_log2_stride = PageLog2Stride;
|
||||
config.silently_mirror_page_table = false;
|
||||
config.absolute_offset_page_table = true;
|
||||
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
|
||||
@@ -230,13 +231,6 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s
|
||||
|
||||
config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt;
|
||||
config.recompile_on_exclusive_fastmem_failure = true;
|
||||
|
||||
if (reinterpret_cast<u64>(m_system.DeviceMemory().buffer.BackingBasePointer() +
|
||||
Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) {
|
||||
// Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries,
|
||||
// we have to manually sign extend when our actual pointer is negative.
|
||||
config.page_table_sign_extension = Common::PageTable::SIGN_BIT;
|
||||
}
|
||||
}
|
||||
|
||||
// Multi-process state
|
||||
@@ -453,7 +447,6 @@ void ArmDynarmic64::SignalInterrupt(Kernel::KThread* thread) {
|
||||
}
|
||||
|
||||
void ArmDynarmic64::ClearInstructionCache() {
|
||||
m_cb->last_code_addr = u64(-1);
|
||||
m_jit->ClearCache();
|
||||
}
|
||||
|
||||
|
||||
@@ -29,11 +29,8 @@ public:
|
||||
|
||||
template <typename T>
|
||||
Common::PhysicalAddress GetPhysicalAddr(const T* ptr) const {
|
||||
return GetPhysicalAddr(reinterpret_cast<uintptr_t>(ptr));
|
||||
}
|
||||
|
||||
Common::PhysicalAddress GetPhysicalAddr(uintptr_t ptr) const {
|
||||
return (ptr - reinterpret_cast<uintptr_t>(buffer.BackingBasePointer())) +
|
||||
return (reinterpret_cast<uintptr_t>(ptr) -
|
||||
reinterpret_cast<uintptr_t>(buffer.BackingBasePointer())) +
|
||||
DramMemoryMap::Base;
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
#include "common/common_types.h"
|
||||
#include "common/range_mutex.h"
|
||||
#include "common/scratch_buffer.h"
|
||||
#include "common/sparse_large_vector.h"
|
||||
#include "common/virtual_buffer.h"
|
||||
|
||||
namespace Core {
|
||||
|
||||
@@ -178,8 +178,8 @@ private:
|
||||
u32 continuity_tracker;
|
||||
u32 compressed_physical_ptr;
|
||||
};
|
||||
Common::SparseLargeVector<u32> compressed_device_addr;
|
||||
Common::SparseLargeVector<TrackedEntry> tracked_entries;
|
||||
Common::VirtualBuffer<u32> compressed_device_addr;
|
||||
Common::VirtualBuffer<TrackedEntry> tracked_entries;
|
||||
|
||||
// Process memory interfaces
|
||||
|
||||
@@ -200,8 +200,8 @@ private:
|
||||
return std::make_pair(asid, address);
|
||||
}
|
||||
|
||||
constexpr void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) {
|
||||
tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit);
|
||||
void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) {
|
||||
tracked_entries[page_index].cpu_backing_address = address | (asid.id << asid_start_bit);
|
||||
}
|
||||
|
||||
std::array<TranslationEntry, 4> t_slot{};
|
||||
|
||||
@@ -177,6 +177,17 @@ DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memo
|
||||
{
|
||||
impl = std::make_unique<DeviceMemoryManagerAllocator<Traits>>();
|
||||
cached_pages = std::make_unique<CachedPages>();
|
||||
|
||||
const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS;
|
||||
for (size_t i = 0; i < total_virtual; i++) {
|
||||
tracked_entries[i].compressed_physical_ptr = 0;
|
||||
tracked_entries[i].continuity_tracker = 1;
|
||||
tracked_entries[i].cpu_backing_address = 0;
|
||||
}
|
||||
const size_t total_phys = 1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS);
|
||||
for (size_t i = 0; i < total_phys; i++) {
|
||||
compressed_device_addr[i] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
@@ -209,28 +220,26 @@ void DeviceMemoryManager<Traits>::Map(DAddr address, VAddr virtual_address, size
|
||||
size_t start_page_d = address >> Memory::YUZU_PAGEBITS;
|
||||
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
|
||||
std::scoped_lock lk(mapping_guard);
|
||||
|
||||
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages);
|
||||
for (size_t i = 0; i < num_pages; i++) {
|
||||
const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE;
|
||||
auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress));
|
||||
if (ptr == nullptr) [[unlikely]] {
|
||||
tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = 0;
|
||||
tracked_entries[start_page_d + i].compressed_physical_ptr = 0;
|
||||
continue;
|
||||
}
|
||||
auto phys_addr = static_cast<u32>(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U;
|
||||
tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = phys_addr;
|
||||
tracked_entries[start_page_d + i].compressed_physical_ptr = phys_addr;
|
||||
InsertCPUBacking(start_page_d + i, new_vaddress, asid);
|
||||
const u32 base_dev = compressed_device_addr[phys_addr - 1U];
|
||||
const u32 new_dev = static_cast<u32>(start_page_d + i);
|
||||
if (base_dev == 0) [[likely]] {
|
||||
compressed_device_addr.GetAndFault(phys_addr - 1U) = new_dev;
|
||||
compressed_device_addr[phys_addr - 1U] = new_dev;
|
||||
continue;
|
||||
}
|
||||
u32 start_id = base_dev & MULTI_MASK;
|
||||
if ((base_dev >> MULTI_FLAG_BITS) == 0) {
|
||||
start_id = impl->multi_dev_address.Register(base_dev);
|
||||
compressed_device_addr.GetAndFault(phys_addr - 1U) = MULTI_FLAG | start_id;
|
||||
compressed_device_addr[phys_addr - 1U] = MULTI_FLAG | start_id;
|
||||
}
|
||||
impl->multi_dev_address.Register(new_dev, start_id);
|
||||
}
|
||||
@@ -246,26 +255,24 @@ void DeviceMemoryManager<Traits>::Unmap(DAddr address, size_t size) {
|
||||
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
|
||||
device_inter->InvalidateRegion(address, size);
|
||||
std::scoped_lock lk(mapping_guard);
|
||||
|
||||
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); // should already be committed, but just in case
|
||||
for (size_t i = 0; i < num_pages; i++) {
|
||||
auto& entry = tracked_entries.GetUnchecked(start_page_d + i);
|
||||
auto phys_addr = entry.compressed_physical_ptr;
|
||||
entry.compressed_physical_ptr = 0;
|
||||
entry.cpu_backing_address = 0;
|
||||
auto phys_addr = tracked_entries[start_page_d + i].compressed_physical_ptr;
|
||||
tracked_entries[start_page_d + i].compressed_physical_ptr = 0;
|
||||
tracked_entries[start_page_d + i].cpu_backing_address = 0;
|
||||
if (phys_addr != 0) [[likely]] {
|
||||
u32& base_dev = compressed_device_addr.GetAndFault(phys_addr - 1U);
|
||||
const u32 base_dev = compressed_device_addr[phys_addr - 1U];
|
||||
if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] {
|
||||
base_dev = 0;
|
||||
compressed_device_addr[phys_addr - 1] = 0;
|
||||
continue;
|
||||
}
|
||||
const auto [more_entries, new_start] = impl->multi_dev_address.Unregister(
|
||||
static_cast<u32>(start_page_d + i), base_dev & MULTI_MASK);
|
||||
if (!more_entries) {
|
||||
base_dev = impl->multi_dev_address.ReleaseEntry(new_start);
|
||||
compressed_device_addr[phys_addr - 1] =
|
||||
impl->multi_dev_address.ReleaseEntry(new_start);
|
||||
continue;
|
||||
}
|
||||
base_dev = new_start | MULTI_FLAG;
|
||||
compressed_device_addr[phys_addr - 1] = new_start | MULTI_FLAG;
|
||||
}
|
||||
}
|
||||
t_slot = {};
|
||||
@@ -278,8 +285,6 @@ void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtu
|
||||
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
|
||||
uintptr_t last_ptr = 0;
|
||||
size_t page_count = 1;
|
||||
|
||||
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages);
|
||||
for (size_t i = num_pages; i > 0; i--) {
|
||||
size_t index = i - 1;
|
||||
const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE;
|
||||
@@ -291,14 +296,14 @@ void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtu
|
||||
page_count = 1;
|
||||
}
|
||||
last_ptr = new_ptr;
|
||||
tracked_entries.GetUnchecked(start_page_d + index).continuity_tracker = static_cast<u32>(page_count) - 1;
|
||||
tracked_entries[start_page_d + index].continuity_tracker = static_cast<u32>(page_count);
|
||||
}
|
||||
}
|
||||
template <typename Traits>
|
||||
u8* DeviceMemoryManager<Traits>::GetSpan(const DAddr src_addr, const std::size_t size) {
|
||||
size_t page_index = src_addr >> page_bits;
|
||||
size_t subbits = src_addr & page_mask;
|
||||
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) {
|
||||
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) {
|
||||
return GetPointer<u8>(src_addr);
|
||||
}
|
||||
return nullptr;
|
||||
@@ -308,7 +313,7 @@ template <typename Traits>
|
||||
const u8* DeviceMemoryManager<Traits>::GetSpan(const DAddr src_addr, const std::size_t size) const {
|
||||
size_t page_index = src_addr >> page_bits;
|
||||
size_t subbits = src_addr & page_mask;
|
||||
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) {
|
||||
if ((static_cast<size_t>(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) {
|
||||
return GetPointer<u8>(src_addr);
|
||||
}
|
||||
return nullptr;
|
||||
@@ -378,7 +383,7 @@ void DeviceMemoryManager<Traits>::WalkBlock(DAddr addr, std::size_t size, auto o
|
||||
std::size_t page_index = addr >> Memory::YUZU_PAGEBITS;
|
||||
std::size_t page_offset = addr & Memory::YUZU_PAGEMASK;
|
||||
while (remaining_size) {
|
||||
const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker+1);
|
||||
const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker);
|
||||
const std::size_t copy_amount = (std::min)((next_pages << Memory::YUZU_PAGEBITS) - page_offset, remaining_size);
|
||||
const auto current_vaddr = u64((page_index << Memory::YUZU_PAGEBITS) + page_offset);
|
||||
SCOPE_EXIT{
|
||||
|
||||
@@ -635,36 +635,6 @@ Result KPageTableBase::CheckMemoryState(const KMemoryInfo& info, KMemoryState st
|
||||
R_SUCCEED();
|
||||
}
|
||||
|
||||
bool KPageTableBase::BeginTraversal(const Common::PageTable &impl, TraversalEntry *out_entry, TraversalContext *out_context,
|
||||
Common::ProcessAddress address) const {
|
||||
out_context->next_offset = GetInteger(address);
|
||||
out_context->next_page = GetInteger(address) >> PageBits;
|
||||
|
||||
return ContinueTraversal(impl, out_entry, out_context);
|
||||
}
|
||||
|
||||
bool KPageTableBase::ContinueTraversal(const Common::PageTable &impl, TraversalEntry *out_entry,
|
||||
TraversalContext *context) const {
|
||||
// Setup invalid defaults.
|
||||
out_entry->phys_addr = 0;
|
||||
out_entry->block_size = PageSize;
|
||||
// Validate that we can read the actual entry.
|
||||
if (auto const page = context->next_page; page < impl.entries.size()) {
|
||||
// Validate that the entry is mapped.
|
||||
if (auto const paddr = impl.entries[page].Pointer(true); paddr != 0) {
|
||||
// Populate the results and return true
|
||||
out_entry->phys_addr = GetInteger(m_system.DeviceMemory().GetPhysicalAddr(paddr + context->next_offset));
|
||||
context->next_page += 1;
|
||||
context->next_offset += PageSize;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
context->next_page += 1;
|
||||
context->next_offset += PageSize;
|
||||
// Otherwise return false
|
||||
return false;
|
||||
}
|
||||
|
||||
Result KPageTableBase::CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr,
|
||||
size_t size, KMemoryState state_mask,
|
||||
KMemoryState state, KMemoryPermission perm_mask,
|
||||
@@ -970,7 +940,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a
|
||||
size_t tot_size = 0;
|
||||
|
||||
next_valid =
|
||||
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), region_start);
|
||||
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), region_start);
|
||||
next_entry.block_size =
|
||||
(next_entry.block_size - (GetInteger(region_start) & (next_entry.block_size - 1)));
|
||||
|
||||
@@ -1006,7 +976,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a
|
||||
break;
|
||||
}
|
||||
|
||||
next_valid = ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
|
||||
next_valid = impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
}
|
||||
|
||||
// Check the last entry.
|
||||
@@ -1784,7 +1754,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_
|
||||
// Begin traversal.
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
R_UNLESS(BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr),
|
||||
R_UNLESS(impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr),
|
||||
ResultInvalidCurrentMemory);
|
||||
|
||||
// Prepare tracking variables.
|
||||
@@ -1794,7 +1764,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_
|
||||
|
||||
// Iterate, adding to group as we go.
|
||||
while (tot_size < size) {
|
||||
R_UNLESS(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)),
|
||||
R_UNLESS(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)),
|
||||
ResultInvalidCurrentMemory);
|
||||
|
||||
if (next_entry.phys_addr != (cur_addr + cur_size)) {
|
||||
@@ -1858,7 +1828,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr
|
||||
// Begin traversal.
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
if (!BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr)) {
|
||||
if (!impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1869,7 +1839,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr
|
||||
|
||||
// Iterate, comparing expected to actual.
|
||||
while (tot_size < size) {
|
||||
if (!ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context))) {
|
||||
if (!impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1926,7 +1896,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState(
|
||||
// Begin a traversal.
|
||||
TraversalContext context;
|
||||
TraversalEntry cur_entry = {.phys_addr = 0, .block_size = 0};
|
||||
R_UNLESS(BeginTraversal(impl, std::addressof(cur_entry), std::addressof(context), address),
|
||||
R_UNLESS(impl.BeginTraversal(std::addressof(cur_entry), std::addressof(context), address),
|
||||
ResultInvalidCurrentMemory);
|
||||
|
||||
// Traverse until we have enough size or we aren't contiguous any more.
|
||||
@@ -1935,7 +1905,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState(
|
||||
for (contig_size =
|
||||
cur_entry.block_size - (GetInteger(phys_address) & (cur_entry.block_size - 1));
|
||||
contig_size < size; contig_size += cur_entry.block_size) {
|
||||
if (!ContinueTraversal(impl, std::addressof(cur_entry), std::addressof(context))) {
|
||||
if (!impl.ContinueTraversal(std::addressof(cur_entry), std::addressof(context))) {
|
||||
break;
|
||||
}
|
||||
if (cur_entry.phys_addr != phys_address + contig_size) {
|
||||
@@ -2364,7 +2334,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out,
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
bool traverse_valid =
|
||||
BeginTraversal(m_impl, std::addressof(next_entry), std::addressof(context), virt_addr);
|
||||
m_impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr);
|
||||
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
|
||||
|
||||
// Set tracking variables.
|
||||
@@ -2375,7 +2345,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out,
|
||||
while (true) {
|
||||
// Continue the traversal.
|
||||
traverse_valid =
|
||||
ContinueTraversal(m_impl, std::addressof(next_entry), std::addressof(context));
|
||||
m_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
if (!traverse_valid) {
|
||||
break;
|
||||
}
|
||||
@@ -2597,7 +2567,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
ASSERT(
|
||||
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address));
|
||||
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address));
|
||||
|
||||
// Check that the physical region matches.
|
||||
R_UNLESS(next_entry.phys_addr == phys_addr, ResultInvalidMemoryRegion);
|
||||
@@ -2607,7 +2577,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre
|
||||
next_entry.block_size - (GetInteger(phys_addr) & (next_entry.block_size - 1));
|
||||
checked_size < size; checked_size += next_entry.block_size) {
|
||||
// Continue the traversal.
|
||||
ASSERT(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)));
|
||||
ASSERT(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)));
|
||||
|
||||
// Check that the physical region matches.
|
||||
R_UNLESS(next_entry.phys_addr == phys_addr + checked_size, ResultInvalidMemoryRegion);
|
||||
@@ -3059,7 +3029,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
bool traverse_valid =
|
||||
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), address);
|
||||
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), address);
|
||||
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
|
||||
|
||||
// Prepare tracking variables.
|
||||
@@ -3071,7 +3041,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_
|
||||
while (tot_size < size) {
|
||||
// Continue the traversal.
|
||||
traverse_valid =
|
||||
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
|
||||
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
|
||||
|
||||
if (next_entry.phys_addr != (cur_addr + cur_size)) {
|
||||
@@ -3159,7 +3129,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
bool traverse_valid =
|
||||
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_address);
|
||||
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_address);
|
||||
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
|
||||
|
||||
// Prepare tracking variables.
|
||||
@@ -3197,7 +3167,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr
|
||||
while (tot_size < size) {
|
||||
// Continue the traversal.
|
||||
traverse_valid =
|
||||
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
|
||||
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
if (next_entry.phys_addr != (cur_addr + cur_size)) {
|
||||
@@ -3255,7 +3225,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
bool traverse_valid =
|
||||
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address);
|
||||
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address);
|
||||
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
|
||||
|
||||
// Prepare tracking variables.
|
||||
@@ -3297,7 +3267,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd
|
||||
while (tot_size < size) {
|
||||
// Continue the traversal.
|
||||
traverse_valid =
|
||||
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
|
||||
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
if (next_entry.phys_addr != (cur_addr + cur_size)) {
|
||||
@@ -3758,7 +3728,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser(
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
bool traverse_valid =
|
||||
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr);
|
||||
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr);
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
// Prepare tracking variables.
|
||||
@@ -3798,7 +3768,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser(
|
||||
while (tot_size < size) {
|
||||
// Continue the traversal.
|
||||
traverse_valid =
|
||||
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
|
||||
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
if (next_entry.phys_addr != (cur_addr + cur_size)) {
|
||||
@@ -3852,7 +3822,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel(
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
bool traverse_valid =
|
||||
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr);
|
||||
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr);
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
// Prepare tracking variables.
|
||||
@@ -3875,7 +3845,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel(
|
||||
while (tot_size < size) {
|
||||
// Continue the traversal.
|
||||
traverse_valid =
|
||||
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
|
||||
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
if (next_entry.phys_addr != (cur_addr + cur_size)) {
|
||||
@@ -3932,7 +3902,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear(
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
bool traverse_valid =
|
||||
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr);
|
||||
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr);
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
// Prepare tracking variables.
|
||||
@@ -3971,7 +3941,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear(
|
||||
while (tot_size < size) {
|
||||
// Continue the traversal.
|
||||
traverse_valid =
|
||||
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
|
||||
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
if (next_entry.phys_addr != (cur_addr + cur_size)) {
|
||||
@@ -4027,7 +3997,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
bool traverse_valid =
|
||||
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr);
|
||||
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr);
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
// Prepare tracking variables.
|
||||
@@ -4050,7 +4020,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si
|
||||
while (tot_size < size) {
|
||||
// Continue the traversal.
|
||||
traverse_valid =
|
||||
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
|
||||
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
if (next_entry.phys_addr != (cur_addr + cur_size)) {
|
||||
@@ -4119,10 +4089,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
|
||||
bool traverse_valid;
|
||||
|
||||
// Begin traversal.
|
||||
traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry),
|
||||
traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry),
|
||||
std::addressof(src_context), src_addr);
|
||||
ASSERT(traverse_valid);
|
||||
traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry),
|
||||
traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry),
|
||||
std::addressof(dst_context), dst_addr);
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
@@ -4157,7 +4127,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
|
||||
if (ofs + cur_copy_size != size) {
|
||||
if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) {
|
||||
// Continue the src traversal.
|
||||
traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry),
|
||||
traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry),
|
||||
std::addressof(src_context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
@@ -4168,7 +4138,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
|
||||
if (cur_dst_addr + cur_min_size ==
|
||||
dst_next_entry.phys_addr + dst_next_entry.block_size) {
|
||||
// Continue the dst traversal.
|
||||
traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry),
|
||||
traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry),
|
||||
std::addressof(dst_context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
@@ -4253,10 +4223,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
|
||||
bool traverse_valid;
|
||||
|
||||
// Begin traversal.
|
||||
traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry),
|
||||
traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry),
|
||||
std::addressof(src_context), src_addr);
|
||||
ASSERT(traverse_valid);
|
||||
traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry),
|
||||
traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry),
|
||||
std::addressof(dst_context), dst_addr);
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
@@ -4291,7 +4261,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
|
||||
if (ofs + cur_copy_size != size) {
|
||||
if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) {
|
||||
// Continue the src traversal.
|
||||
traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry),
|
||||
traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry),
|
||||
std::addressof(src_context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
@@ -4302,7 +4272,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
|
||||
if (cur_dst_addr + cur_min_size ==
|
||||
dst_next_entry.phys_addr + dst_next_entry.block_size) {
|
||||
// Continue the dst traversal.
|
||||
traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry),
|
||||
traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry),
|
||||
std::addressof(dst_context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
@@ -4577,7 +4547,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
|
||||
// Begin traversal.
|
||||
TraversalContext context;
|
||||
TraversalEntry next_entry;
|
||||
bool traverse_valid = BeginTraversal(src_impl, std::addressof(next_entry),
|
||||
bool traverse_valid = src_impl.BeginTraversal(std::addressof(next_entry),
|
||||
std::addressof(context), aligned_src_start);
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
@@ -4627,7 +4597,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
|
||||
// If the block's size was one page, we may need to continue traversal.
|
||||
if (cur_block_size == 0 && aligned_src_size > PageSize) {
|
||||
traverse_valid =
|
||||
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
|
||||
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
cur_block_addr = next_entry.phys_addr;
|
||||
@@ -4640,7 +4610,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
|
||||
while (aligned_src_start + tot_block_size < mapping_src_end) {
|
||||
// Continue the traversal.
|
||||
traverse_valid =
|
||||
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
|
||||
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
// Process the block.
|
||||
@@ -4683,7 +4653,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
|
||||
if (mapped_block_end + cur_block_size < aligned_src_end &&
|
||||
cur_block_size == last_block_size) {
|
||||
traverse_valid =
|
||||
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
|
||||
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
|
||||
ASSERT(traverse_valid);
|
||||
|
||||
cur_block_addr = next_entry.phys_addr;
|
||||
@@ -5631,7 +5601,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si
|
||||
ContiguousRangeInfo(KPageTableBase& pt, KProcessAddress address, size_t size)
|
||||
: m_pt(pt), m_remaining_size(size) {
|
||||
// Begin a traversal.
|
||||
ASSERT(m_pt.BeginTraversal(m_pt.GetImpl(), std::addressof(m_entry),
|
||||
ASSERT(m_pt.GetImpl().BeginTraversal(std::addressof(m_entry),
|
||||
std::addressof(m_context), address));
|
||||
|
||||
// Setup tracking fields.
|
||||
@@ -5662,7 +5632,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si
|
||||
void DetermineContiguousBlockExtents() {
|
||||
// Continue traversing until we're not contiguous, or we have enough.
|
||||
while (m_cur_size < m_remaining_size) {
|
||||
ASSERT(m_pt.ContinueTraversal(m_pt.GetImpl(), std::addressof(m_entry),
|
||||
ASSERT(m_pt.GetImpl().ContinueTraversal(std::addressof(m_entry),
|
||||
std::addressof(m_context)));
|
||||
|
||||
// If we're not contiguous, we're done.
|
||||
|
||||
@@ -370,10 +370,6 @@ private:
|
||||
size_t num_pages, size_t alignment, size_t offset,
|
||||
size_t guard_pages) const;
|
||||
|
||||
bool BeginTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* out_context,
|
||||
Common::ProcessAddress address) const;
|
||||
bool ContinueTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* context) const;
|
||||
|
||||
Result CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size,
|
||||
KMemoryState state_mask, KMemoryState state,
|
||||
KMemoryPermission perm_mask, KMemoryPermission perm,
|
||||
@@ -478,14 +474,7 @@ private:
|
||||
// Validate pre-conditions.
|
||||
ASSERT(this->IsLockedByCurrentThread());
|
||||
|
||||
if (virt_addr > (1ULL << m_address_space_width)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
*out = m_system.DeviceMemory().GetPhysicalAddr(
|
||||
this->GetImpl().entries[GetInteger(virt_addr) >> PageBits].Pointer(true) + GetInteger(virt_addr));
|
||||
|
||||
return true;
|
||||
return this->GetImpl().GetPhysicalAddress(out, virt_addr);
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
+39
-42
@@ -101,10 +101,8 @@ struct Memory::Impl {
|
||||
}
|
||||
|
||||
u64 protect_bytes = 0, protect_begin = 0;
|
||||
|
||||
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr + size) >> YUZU_PAGEBITS);
|
||||
for (u64 addr = vaddr; addr < vaddr + size; addr += YUZU_PAGESIZE) {
|
||||
const Common::PageType page_type = current_page_table->entries.GetUnchecked(addr >> YUZU_PAGEBITS).Type();
|
||||
const Common::PageType page_type = current_page_table->entries[addr >> YUZU_PAGEBITS].ptr.Type();
|
||||
switch (page_type) {
|
||||
case Common::PageType::RasterizerCachedMemory:
|
||||
if (protect_bytes > 0) {
|
||||
@@ -125,14 +123,16 @@ struct Memory::Impl {
|
||||
}
|
||||
|
||||
[[nodiscard]] u8* GetPointerFromRasterizerCachedMemory(u64 vaddr) const {
|
||||
if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr)
|
||||
return reinterpret_cast<u8*>(paddr) + vaddr;
|
||||
Common::PhysicalAddress const paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr;
|
||||
if (paddr)
|
||||
return system.DeviceMemory().GetPointer<u8>(paddr + vaddr);
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] u8* GetPointerFromDebugMemory(u64 vaddr) const {
|
||||
if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr)
|
||||
return reinterpret_cast<u8*>(paddr) + vaddr;
|
||||
const Common::PhysicalAddress paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr;
|
||||
if (paddr != 0)
|
||||
return system.DeviceMemory().GetPointer<u8>(paddr + vaddr);
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -243,12 +243,10 @@ struct Memory::Impl {
|
||||
std::size_t page_index = addr >> YUZU_PAGEBITS;
|
||||
std::size_t page_offset = addr & YUZU_PAGEMASK;
|
||||
bool user_accessible = true;
|
||||
|
||||
current_page_table->entries.CommitRegion(page_index, page_index + (size >> YUZU_PAGEBITS) + 1);
|
||||
while (remaining_size != 0) {
|
||||
const std::size_t copy_amount = (std::min)(std::size_t(YUZU_PAGESIZE) - page_offset, remaining_size);
|
||||
const auto current_vaddr = u64((page_index << YUZU_PAGEBITS) + page_offset);
|
||||
const auto [pointer, type, _] = current_page_table->entries.GetUnchecked(page_index).PointerTypeBlock();
|
||||
const auto [pointer, type] = current_page_table->entries[page_index].ptr.PointerType();
|
||||
switch (type) {
|
||||
case Common::PageType::Unmapped: {
|
||||
user_accessible = false;
|
||||
@@ -299,10 +297,10 @@ struct Memory::Impl {
|
||||
}
|
||||
|
||||
[[nodiscard]] inline const u8* GetSpan(const VAddr addr, const std::size_t size) const noexcept {
|
||||
return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr;
|
||||
return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr;
|
||||
}
|
||||
[[nodiscard]] inline u8* GetSpan(const VAddr addr, const std::size_t size) noexcept {
|
||||
return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr;
|
||||
return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr;
|
||||
}
|
||||
|
||||
bool WriteBlockImpl(const Common::ProcessAddress addr, const void* buffer, const std::size_t size, bool unsafe) {
|
||||
@@ -406,14 +404,11 @@ struct Memory::Impl {
|
||||
// The region is at a granularity of CPU pages.
|
||||
|
||||
const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1;
|
||||
|
||||
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
|
||||
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
|
||||
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
|
||||
const auto [pointer, type, block] = entry.PointerTypeBlock(true);
|
||||
const Common::PageType page_type = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type();
|
||||
if (debug) {
|
||||
// Switch page type to debug if now debug
|
||||
switch (type) {
|
||||
switch (page_type) {
|
||||
case Common::PageType::Unmapped:
|
||||
ASSERT(false && "Attempted to mark unmapped pages as debug");
|
||||
break;
|
||||
@@ -422,14 +417,14 @@ struct Memory::Impl {
|
||||
// Page is already marked.
|
||||
break;
|
||||
case Common::PageType::Memory:
|
||||
entry.MarkDebug(pointer, block);
|
||||
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::DebugMemory);
|
||||
break;
|
||||
default:
|
||||
UNREACHABLE();
|
||||
}
|
||||
} else {
|
||||
// Switch page type to non-debug if now non-debug
|
||||
switch (type) {
|
||||
switch (page_type) {
|
||||
case Common::PageType::Unmapped:
|
||||
ASSERT(false && "Attempted to mark unmapped pages as non-debug");
|
||||
break;
|
||||
@@ -438,7 +433,8 @@ struct Memory::Impl {
|
||||
// Don't mess with already non-debug or rasterizer memory.
|
||||
break;
|
||||
case Common::PageType::DebugMemory: {
|
||||
entry.Store(false, Common::PageType::Memory, block, pointer);
|
||||
u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK);
|
||||
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
@@ -470,10 +466,8 @@ struct Memory::Impl {
|
||||
// is different). This assumes the specified GPU address region is contiguous as well.
|
||||
|
||||
const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1;
|
||||
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
|
||||
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
|
||||
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
|
||||
const Common::PageType page_type = entry.Type();
|
||||
const Common::PageType page_type= current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type();
|
||||
if (cached) {
|
||||
// Switch page type to cached if now cached
|
||||
switch (page_type) {
|
||||
@@ -483,7 +477,7 @@ struct Memory::Impl {
|
||||
break;
|
||||
case Common::PageType::DebugMemory:
|
||||
case Common::PageType::Memory:
|
||||
entry.MarkRasterizerCached();
|
||||
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::RasterizerCachedMemory);
|
||||
break;
|
||||
case Common::PageType::RasterizerCachedMemory:
|
||||
// There can be more than one GPU region mapped per CPU region, so it's common
|
||||
@@ -505,13 +499,13 @@ struct Memory::Impl {
|
||||
// that this area is already unmarked as cached.
|
||||
break;
|
||||
case Common::PageType::RasterizerCachedMemory: {
|
||||
if (auto [ptr, _, block] = entry.PointerTypeBlock(true); ptr == 0) {
|
||||
if (u8* const pointer = GetPointerFromRasterizerCachedMemory(vaddr & ~YUZU_PAGEMASK); pointer == nullptr) {
|
||||
// It's possible that this function has been called while updating the
|
||||
// pagetable after unmapping a VMA. In that case the underlying VMA will no
|
||||
// longer exist, and we should just leave the pagetable entry blank.
|
||||
entry.Store(false, Common::PageType::Unmapped, block, 0);
|
||||
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::Unmapped);
|
||||
} else {
|
||||
entry.Store(false, Common::PageType::Memory, block, ptr);
|
||||
current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -545,18 +539,22 @@ struct Memory::Impl {
|
||||
ASSERT_MSG(type != Common::PageType::Memory,
|
||||
"Mapping memory page without a pointer @ {:016x}", base * YUZU_PAGESIZE);
|
||||
|
||||
page_table.entries.ZeroRegion(base, end);
|
||||
} else {
|
||||
auto current_block = block_count.fetch_add(1, std::memory_order_relaxed);
|
||||
ASSERT(current_block != 65535);
|
||||
|
||||
page_table.entries.CommitRegion(base, end);
|
||||
while (base != end) {
|
||||
auto host_ptr = reinterpret_cast<u64>(system.DeviceMemory().GetPointer<u8>(target)) - (base << YUZU_PAGEBITS);;
|
||||
auto& entry = page_table.entries.GetUnchecked(base);
|
||||
page_table.entries[base].ptr.Store(0, type);
|
||||
page_table.entries[base].addr = 0;
|
||||
page_table.entries[base].block = 0;
|
||||
base += 1;
|
||||
}
|
||||
} else {
|
||||
auto orig_base = base;
|
||||
while (base != end) {
|
||||
auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer<u8>(target)) - (base << YUZU_PAGEBITS);
|
||||
auto backing = GetInteger(target) - (base << YUZU_PAGEBITS);
|
||||
page_table.entries[base].ptr.Store(host_ptr, type);
|
||||
page_table.entries[base].addr = backing;
|
||||
page_table.entries[base].block = orig_base << YUZU_PAGEBITS;
|
||||
|
||||
entry.Store(false, type, current_block, host_ptr);
|
||||
ASSERT_MSG(page_table.entries[base].Pointer(),
|
||||
ASSERT_MSG(page_table.entries[base].ptr.Pointer(),
|
||||
"memory mapping base yield a nullptr within the table");
|
||||
|
||||
base += 1;
|
||||
@@ -571,11 +569,11 @@ struct Memory::Impl {
|
||||
vaddr &= 0xffffffffffffULL;
|
||||
if (AddressSpaceContains(*current_page_table, vaddr, 1)) [[likely]] {
|
||||
// Avoid adding any extra logic to this fast-path block
|
||||
const auto raw = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Raw();
|
||||
if (auto pointer = Common::PageTable::PageEntryData::ExtractPointer(raw); pointer) [[likely]] {
|
||||
const uintptr_t raw_pointer = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Raw();
|
||||
if (const uintptr_t pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) [[likely]] {
|
||||
return reinterpret_cast<u8*>(pointer + vaddr);
|
||||
} else {
|
||||
switch (static_cast<Common::PageType>(raw.type)) {
|
||||
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
|
||||
case Common::PageType::Memory:
|
||||
ASSERT_MSG(false, "Mapped memory page without a pointer @ {:#016x}", vaddr);
|
||||
return nullptr;
|
||||
@@ -775,7 +773,6 @@ struct Memory::Impl {
|
||||
#else
|
||||
Common::HostMemory* host_buffer{};
|
||||
#endif
|
||||
std::atomic<u16> block_count = 0;
|
||||
};
|
||||
|
||||
Memory::Memory(Core::System& system_) : system{system_} {
|
||||
@@ -814,7 +811,7 @@ bool Memory::IsValidVirtualAddress(const Common::ProcessAddress vaddr) const {
|
||||
if (page >= page_table.entries.size()) {
|
||||
return false;
|
||||
}
|
||||
const auto [pointer, type, _] = page_table.entries[page].PointerTypeBlock();
|
||||
const auto [pointer, type] = page_table.entries[page].ptr.PointerType();
|
||||
return pointer != 0 || type == Common::PageType::RasterizerCachedMemory ||
|
||||
type == Common::PageType::DebugMemory;
|
||||
}
|
||||
|
||||
@@ -371,10 +371,8 @@ EmitConfig A32AddressSpace::GetEmitConfig() {
|
||||
|
||||
.page_table_pointer = std::bit_cast<u64>(conf.page_table),
|
||||
.page_table_address_space_bits = 32,
|
||||
.page_table_pointer_mask = conf.page_table_pointer_mask,
|
||||
.page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits,
|
||||
.page_table_log2_stride = conf.page_table_log2_stride,
|
||||
.page_table_marked_bit = conf.page_table_marked_bit,
|
||||
.page_table_sign_extension = conf.page_table_sign_extension,
|
||||
.silently_mirror_page_table = true,
|
||||
.absolute_offset_page_table = conf.absolute_offset_page_table,
|
||||
.detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table,
|
||||
|
||||
@@ -545,10 +545,8 @@ EmitConfig A64AddressSpace::GetEmitConfig() {
|
||||
|
||||
.page_table_pointer = std::bit_cast<u64>(conf.page_table),
|
||||
.page_table_address_space_bits = conf.page_table_address_space_bits,
|
||||
.page_table_pointer_mask = conf.page_table_pointer_mask,
|
||||
.page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits,
|
||||
.page_table_log2_stride = conf.page_table_log2_stride,
|
||||
.page_table_marked_bit = conf.page_table_marked_bit,
|
||||
.page_table_sign_extension = conf.page_table_sign_extension,
|
||||
.silently_mirror_page_table = conf.silently_mirror_page_table,
|
||||
.absolute_offset_page_table = conf.absolute_offset_page_table,
|
||||
.detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table,
|
||||
|
||||
@@ -128,10 +128,8 @@ struct EmitConfig {
|
||||
// Page table
|
||||
u64 page_table_pointer;
|
||||
std::size_t page_table_address_space_bits;
|
||||
u64 page_table_pointer_mask;
|
||||
int page_table_pointer_mask_bits;
|
||||
std::size_t page_table_log2_stride;
|
||||
std::optional<std::uint8_t> page_table_marked_bit;
|
||||
std::optional<std::uint8_t> page_table_sign_extension;
|
||||
bool silently_mirror_page_table;
|
||||
bool absolute_offset_page_table;
|
||||
u8 detect_misaligned_access_via_page_table;
|
||||
|
||||
@@ -273,18 +273,9 @@ std::pair<oaknut::XReg, oaknut::XReg> InlinePageTableEmitVAddrLookup(oaknut::Cod
|
||||
// load x0 = *<(u8*)pagetable + index>
|
||||
code.LDR(Xscratch0, Xpagetable, Xscratch0);
|
||||
|
||||
if (ctx.conf.page_table_marked_bit) {
|
||||
code.TST(Xscratch0, 1ULL << *ctx.conf.page_table_marked_bit);
|
||||
code.B(NE, *fallback);
|
||||
}
|
||||
|
||||
if (ctx.conf.page_table_pointer_mask != 0) {
|
||||
code.AND(Xscratch0, Xscratch0, ctx.conf.page_table_pointer_mask);
|
||||
}
|
||||
|
||||
// TODO: combine this with page_table_pointer_mask
|
||||
if (ctx.conf.page_table_sign_extension) {
|
||||
code.SBFM(Xscratch0, Xscratch0, 0, *ctx.conf.page_table_sign_extension);
|
||||
if (ctx.conf.page_table_pointer_mask_bits != 0) {
|
||||
const u64 mask = u64(~u64(0)) << ctx.conf.page_table_pointer_mask_bits;
|
||||
code.AND(Xscratch0, Xscratch0, mask);
|
||||
}
|
||||
|
||||
code.CBZ(Xscratch0, *fallback);
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <bit>
|
||||
#include <utility>
|
||||
#include "dynarmic/backend/x64/xbyak.h"
|
||||
|
||||
#include "dynarmic/backend/x64/a32_emit_x64.h"
|
||||
@@ -79,40 +78,27 @@ Xbyak::RegExp EmitVAddrLookup(BlockOfCode& code, EmitContext& ctx, size_t bitsiz
|
||||
template<>
|
||||
[[maybe_unused]] Xbyak::RegExp EmitVAddrLookup<A32EmitContext>(BlockOfCode& code, A32EmitContext& ctx, size_t bitsize, Xbyak::Label& abort, Xbyak::Reg64 vaddr) {
|
||||
const Xbyak::Reg64 page = ctx.reg_alloc.ScratchGpr(code);
|
||||
const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table && ctx.conf.page_table_pointer_mask == 0 ? page : ctx.reg_alloc.ScratchGpr(code);
|
||||
const Xbyak::Reg32 tmp = ctx.conf.absolute_offset_page_table ? page.cvt32() : ctx.reg_alloc.ScratchGpr(code).cvt32();
|
||||
|
||||
EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp);
|
||||
EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp.cvt64());
|
||||
|
||||
code.mov(tmp, vaddr);
|
||||
// TODO: This code assumes vaddr has been zext from 32-bits to 64-bits.
|
||||
|
||||
code.mov(tmp, vaddr.cvt32());
|
||||
code.shr(tmp, int(page_table_const_bits));
|
||||
code.shl(tmp, int(ctx.conf.page_table_log2_stride));
|
||||
code.mov(page, qword[r14 + tmp.cvt64()]);
|
||||
|
||||
// check for marked bit, use as unmapped if marked
|
||||
if (ctx.conf.page_table_marked_bit) {
|
||||
code.bt(page, *ctx.conf.page_table_marked_bit);
|
||||
code.jc(abort, code.T_NEAR);
|
||||
}
|
||||
// mask away attributes
|
||||
if (ctx.conf.page_table_pointer_mask == 0) {
|
||||
if (ctx.conf.page_table_pointer_mask_bits == 0) {
|
||||
code.test(page, page);
|
||||
} else if (std::in_range<s32>(ctx.conf.page_table_pointer_mask)) {
|
||||
code.and_(page, ctx.conf.page_table_pointer_mask);
|
||||
} else {
|
||||
code.mov(tmp, ctx.conf.page_table_pointer_mask);
|
||||
code.and_(page, tmp);
|
||||
code.and_(page, ~u32(0) << ctx.conf.page_table_pointer_mask_bits);
|
||||
}
|
||||
if (ctx.conf.page_table_sign_extension) {
|
||||
code.shl(page, *ctx.conf.page_table_sign_extension);
|
||||
code.sar(page, *ctx.conf.page_table_sign_extension);
|
||||
}
|
||||
|
||||
code.jz(abort, code.T_NEAR);
|
||||
if (ctx.conf.absolute_offset_page_table) {
|
||||
return page + vaddr;
|
||||
}
|
||||
code.mov(tmp, vaddr);
|
||||
code.and_(tmp, u32(page_table_const_mask));
|
||||
code.mov(tmp, vaddr.cvt32());
|
||||
code.and_(tmp, static_cast<u32>(page_table_const_mask));
|
||||
return page + tmp.cvt64();
|
||||
}
|
||||
|
||||
@@ -122,7 +108,7 @@ template<>
|
||||
const size_t unused_top_bits = 64 - ctx.conf.page_table_address_space_bits;
|
||||
|
||||
const Xbyak::Reg64 page = ctx.reg_alloc.ScratchGpr(code);
|
||||
const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table && ctx.conf.page_table_pointer_mask == 0 ? page : ctx.reg_alloc.ScratchGpr(code);
|
||||
const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table ? page : ctx.reg_alloc.ScratchGpr(code);
|
||||
|
||||
EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp);
|
||||
|
||||
@@ -157,26 +143,11 @@ template<>
|
||||
|
||||
code.shl(tmp, int(ctx.conf.page_table_log2_stride));
|
||||
code.mov(page, qword[r14 + tmp]);
|
||||
|
||||
// check for marked bit, use as unmapped if marked
|
||||
if (ctx.conf.page_table_marked_bit) {
|
||||
code.bt(page, *ctx.conf.page_table_marked_bit);
|
||||
code.jc(abort, code.T_NEAR);
|
||||
}
|
||||
// mask away attributes
|
||||
if (ctx.conf.page_table_pointer_mask == 0) {
|
||||
if (ctx.conf.page_table_pointer_mask_bits == 0) {
|
||||
code.test(page, page);
|
||||
} else if (std::in_range<s32>(ctx.conf.page_table_pointer_mask)) {
|
||||
code.and_(page, ctx.conf.page_table_pointer_mask);
|
||||
} else {
|
||||
code.mov(tmp, ctx.conf.page_table_pointer_mask);
|
||||
code.and_(page, tmp);
|
||||
code.and_(page, ~u32(0) << ctx.conf.page_table_pointer_mask_bits);
|
||||
}
|
||||
if (ctx.conf.page_table_sign_extension) {
|
||||
code.shl(page, *ctx.conf.page_table_sign_extension);
|
||||
code.sar(page, *ctx.conf.page_table_sign_extension);
|
||||
}
|
||||
|
||||
code.jz(abort, code.T_NEAR);
|
||||
if (ctx.conf.absolute_offset_page_table) {
|
||||
return page + vaddr;
|
||||
|
||||
@@ -159,23 +159,14 @@ struct UserConfig {
|
||||
/// Maximum size is limited by the maximum length of a x86_64 / arm64 jump.
|
||||
std::uint32_t code_cache_size = 128 * 1024 * 1024; // bytes
|
||||
|
||||
/// Applies a bit mask to the bits in host pointers from the page table.
|
||||
/// Masks out the first N bits in host pointers from the page table.
|
||||
/// The intention behind this is to allow users of Dynarmic to pack attributes in the
|
||||
/// same integer and update the pointer attribute pair atomically.
|
||||
/// If the configured value is ~(0b111ULL), all pointers will be forcefully aligned to 8 bytes.
|
||||
std::uint64_t page_table_pointer_mask = 0;
|
||||
/// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes.
|
||||
std::int32_t page_table_pointer_mask_bits = 0;
|
||||
|
||||
/// Log2 of the size per page entry, value should be either 3 or 4
|
||||
std::uint32_t page_table_log2_stride = 3;
|
||||
|
||||
/// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table.
|
||||
/// If the bit is set to 1, Dynarmic will treat it as unmapped.
|
||||
/// This bit should be included as part of `page_table_pointer_mask_bits`.
|
||||
std::optional<std::uint8_t> page_table_marked_bit = std::nullopt;
|
||||
|
||||
/// If this value is set, Dynarmic will sign extend the page table pointer by this bit.
|
||||
/// Useful for compacting bits into the page table and should be used as part of `page_table_pointer_mask`.
|
||||
std::optional<std::uint8_t> page_table_sign_extension = std::nullopt;
|
||||
// Log2 of the size per page entry, value should be either 3 or 4
|
||||
std::size_t page_table_log2_stride = 3;
|
||||
|
||||
/// Select the architecture version to use.
|
||||
/// There are minor behavioural differences between versions.
|
||||
|
||||
@@ -173,23 +173,14 @@ struct UserConfig {
|
||||
/// This is only used if page_table is not nullptr.
|
||||
std::uint32_t page_table_address_space_bits = 36;
|
||||
|
||||
/// Applies a bit mask to the bits in host pointers from the page table.
|
||||
/// Masks out the first N bits in host pointers from the page table.
|
||||
/// The intention behind this is to allow users of Dynarmic to pack attributes in the
|
||||
/// same integer and update the pointer attribute pair atomically.
|
||||
/// If the configured value is ~(0b111ULL), all pointers will be forcefully aligned to 8 bytes.
|
||||
std::uint64_t page_table_pointer_mask = 0;
|
||||
/// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes.
|
||||
std::int32_t page_table_pointer_mask_bits = 0;
|
||||
|
||||
/// Log2 of the size per page entry, value should be either 3 or 4
|
||||
std::uint32_t page_table_log2_stride = 3;
|
||||
|
||||
/// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table.
|
||||
/// If the bit is set to 1, Dynarmic will treat it as unmapped.
|
||||
/// This bit should be included as part of `page_table_pointer_mask`.
|
||||
std::optional<std::uint8_t> page_table_marked_bit = std::nullopt;
|
||||
|
||||
/// If this value is set, Dynarmic will sign extend the page table pointer by this bit.
|
||||
/// Useful for compacting bits into the page table and should be used as part of `page_table_pointer_mask`.
|
||||
std::optional<std::uint8_t> page_table_sign_extension = std::nullopt;
|
||||
// Log2 of the size per page entry, value should be either 3 or 4
|
||||
std::size_t page_table_log2_stride = 3;
|
||||
|
||||
/// Counter-timer frequency register. The value of the register is not interpreted by
|
||||
/// dynarmic.
|
||||
|
||||
@@ -46,8 +46,8 @@ MemoryManager::MemoryManager(Core::System& system_, MaxwellDeviceMemoryManager&
|
||||
page_table_mask = page_table_size - 1;
|
||||
big_page_table_mask = big_page_table_size - 1;
|
||||
|
||||
big_page_table_dev.ResizeAndClear(big_page_table_size);
|
||||
big_entries.resize(big_page_table_size / 32, 0);
|
||||
big_page_table_dev.resize(big_page_table_size);
|
||||
big_page_continuous.resize(big_page_table_size / continuous_bits, 0);
|
||||
entries.resize(page_table_size / 32, 0);
|
||||
}
|
||||
@@ -143,7 +143,7 @@ GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr
|
||||
const DAddr current_dev_addr = dev_addr + offset;
|
||||
const auto index = PageEntryIndex(current_gpu_addr, true);
|
||||
const u32 sub_value = static_cast<u32>(current_dev_addr >> cpu_page_bits);
|
||||
big_page_table_dev.Set(index, sub_value);
|
||||
big_page_table_dev[index] = sub_value;
|
||||
const bool is_continuous = ([&] {
|
||||
uintptr_t base_ptr{
|
||||
reinterpret_cast<uintptr_t>(memory.GetPointer<u8>(current_dev_addr))};
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
#include "common/multi_level_page_table.h"
|
||||
#include "common/range_map.h"
|
||||
#include "common/scratch_buffer.h"
|
||||
#include "common/sparse_large_vector.h"
|
||||
#include "common/virtual_buffer.h"
|
||||
#include "video_core/invalidation_accumulator.h"
|
||||
#include "video_core/cache_types.h"
|
||||
#include "video_core/host1x/gpu_device_memory_manager.h"
|
||||
@@ -214,7 +214,7 @@ private:
|
||||
|
||||
Common::MultiLevelPageTable<u32> page_table;
|
||||
Common::RangeMap<GPUVAddr, PTEKind> kind_map;
|
||||
Common::SparseLargeVector<u32> big_page_table_dev;
|
||||
Common::VirtualBuffer<u32> big_page_table_dev;
|
||||
|
||||
std::vector<u64> big_page_continuous;
|
||||
boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash{};
|
||||
|
||||
Reference in New Issue
Block a user