Compare commits

..

6 Commits

Author SHA1 Message Date
lizzie 1439a8041b fx 2026-05-19 07:53:59 +00:00
lizzie 6c3ea34293 fixup the stupid boost map 2026-05-19 07:08:37 +00:00
lizzie bca75e8c21 Trigger Build 2026-05-19 07:08:37 +00:00
lizzie 7572632afa fuckery2 2026-05-19 07:08:37 +00:00
lizzie 641a430463 no move constructor? 2026-05-19 07:08:36 +00:00
lizzie 60328b0bd9 [core/hle/service/nvdrv] fix Nvmap storage being pointer-unstable due to ankerl maps
Signed-off-by: lizzie <lizzie@eden-emu.dev>
2026-05-19 07:08:36 +00:00
17 changed files with 471 additions and 393 deletions
@@ -1,6 +1,3 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -10,30 +7,28 @@
namespace Core {
DynarmicExclusiveMonitor::DynarmicExclusiveMonitor(Memory::Memory& memory_, std::size_t core_count_)
: monitor{}
, memory{memory_}
{}
: monitor{core_count_}, memory{memory_} {}
DynarmicExclusiveMonitor::~DynarmicExclusiveMonitor() = default;
u8 DynarmicExclusiveMonitor::ExclusiveRead8(std::size_t core_index, VAddr addr) {
return monitor.ReadAndMark<u8>(core_index, addr, [=]() -> u8 { return memory.Read8(addr); });
return monitor.ReadAndMark<u8>(core_index, addr, [&]() -> u8 { return memory.Read8(addr); });
}
u16 DynarmicExclusiveMonitor::ExclusiveRead16(std::size_t core_index, VAddr addr) {
return monitor.ReadAndMark<u16>(core_index, addr, [=]() -> u16 { return memory.Read16(addr); });
return monitor.ReadAndMark<u16>(core_index, addr, [&]() -> u16 { return memory.Read16(addr); });
}
u32 DynarmicExclusiveMonitor::ExclusiveRead32(std::size_t core_index, VAddr addr) {
return monitor.ReadAndMark<u32>(core_index, addr, [=]() -> u32 { return memory.Read32(addr); });
return monitor.ReadAndMark<u32>(core_index, addr, [&]() -> u32 { return memory.Read32(addr); });
}
u64 DynarmicExclusiveMonitor::ExclusiveRead64(std::size_t core_index, VAddr addr) {
return monitor.ReadAndMark<u64>(core_index, addr, [=]() -> u64 { return memory.Read64(addr); });
return monitor.ReadAndMark<u64>(core_index, addr, [&]() -> u64 { return memory.Read64(addr); });
}
u128 DynarmicExclusiveMonitor::ExclusiveRead128(std::size_t core_index, VAddr addr) {
return monitor.ReadAndMark<u128>(core_index, addr, [=]() -> u128 {
return monitor.ReadAndMark<u128>(core_index, addr, [&]() -> u128 {
u128 result;
result[0] = memory.Read64(addr);
result[1] = memory.Read64(addr + 8);
@@ -46,31 +41,31 @@ void DynarmicExclusiveMonitor::ClearExclusive(std::size_t core_index) {
}
bool DynarmicExclusiveMonitor::ExclusiveWrite8(std::size_t core_index, VAddr vaddr, u8 value) {
return monitor.DoExclusiveOperation<u8>(core_index, vaddr, [=](u8 expected) -> bool {
return monitor.DoExclusiveOperation<u8>(core_index, vaddr, [&](u8 expected) -> bool {
return memory.WriteExclusive8(vaddr, value, expected);
});
}
bool DynarmicExclusiveMonitor::ExclusiveWrite16(std::size_t core_index, VAddr vaddr, u16 value) {
return monitor.DoExclusiveOperation<u16>(core_index, vaddr, [=](u16 expected) -> bool {
return monitor.DoExclusiveOperation<u16>(core_index, vaddr, [&](u16 expected) -> bool {
return memory.WriteExclusive16(vaddr, value, expected);
});
}
bool DynarmicExclusiveMonitor::ExclusiveWrite32(std::size_t core_index, VAddr vaddr, u32 value) {
return monitor.DoExclusiveOperation<u32>(core_index, vaddr, [=](u32 expected) -> bool {
return monitor.DoExclusiveOperation<u32>(core_index, vaddr, [&](u32 expected) -> bool {
return memory.WriteExclusive32(vaddr, value, expected);
});
}
bool DynarmicExclusiveMonitor::ExclusiveWrite64(std::size_t core_index, VAddr vaddr, u64 value) {
return monitor.DoExclusiveOperation<u64>(core_index, vaddr, [=](u64 expected) -> bool {
return monitor.DoExclusiveOperation<u64>(core_index, vaddr, [&](u64 expected) -> bool {
return memory.WriteExclusive64(vaddr, value, expected);
});
}
bool DynarmicExclusiveMonitor::ExclusiveWrite128(std::size_t core_index, VAddr vaddr, u128 value) {
return monitor.DoExclusiveOperation<u128>(core_index, vaddr, [=](u128 expected) -> bool {
return monitor.DoExclusiveOperation<u128>(core_index, vaddr, [&](u128 expected) -> bool {
return memory.WriteExclusive128(vaddr, value, expected);
});
}
+61 -106
View File
@@ -25,42 +25,33 @@ NvMap::Handle::Handle(u64 size_, Id id_)
flags.raw = 0;
}
NvResult NvMap::Handle::Alloc(Flags pFlags, u32 pAlign, u8 pKind, u64 pAddress,
NvCore::SessionId pSessionId) {
std::scoped_lock lock(mutex);
NvResult NvMap::Handle::Alloc(Flags pFlags, u32 pAlign, u8 pKind, u64 pAddress, NvCore::SessionId pSessionId) {
// Handles cannot be allocated twice
if (allocated) {
return NvResult::AccessDenied;
}
flags = pFlags;
kind = pKind;
align = pAlign < YUZU_PAGESIZE ? YUZU_PAGESIZE : pAlign;
session_id = pSessionId;
// This flag is only applicable for handles with an address passed
if (pAddress) {
flags.keep_uncached_after_free.Assign(0);
} else {
LOG_CRITICAL(Service_NVDRV,
"Mapping nvmap handles without a CPU side address is unimplemented!");
LOG_CRITICAL(Service_NVDRV, "Mapping nvmap handles without a CPU side address is unimplemented!");
}
size = Common::AlignUp(size, YUZU_PAGESIZE);
aligned_size = Common::AlignUp(size, align);
address = pAddress;
allocated = true;
return NvResult::Success;
}
NvResult NvMap::Handle::Duplicate(bool internal_session) {
std::scoped_lock lock(mutex);
// Unallocated handles cannot be duplicated as duplication requires memory accounting (in HOS)
if (!allocated) [[unlikely]] {
return NvResult::BadValue;
}
// If we internally use FromId the duplication tracking of handles won't work accurately due to
// us not implementing per-process handle refs.
if (internal_session) {
@@ -68,16 +59,14 @@ NvResult NvMap::Handle::Duplicate(bool internal_session) {
} else {
dupes++;
}
return NvResult::Success;
}
NvMap::NvMap(Container& core_, Tegra::Host1x::Host1x& host1x_) : host1x{host1x_}, core{core_} {}
void NvMap::AddHandle(std::shared_ptr<Handle> handle_description) {
std::scoped_lock lock(handles_lock);
handles.emplace(handle_description->id, std::move(handle_description));
void NvMap::AddHandle(Handle&& handle_description) {
std::scoped_lock l(handles_lock);
handles.insert_or_assign(handle_description.id, std::move(handle_description));
}
void NvMap::UnmapHandle(Handle& handle_description) {
@@ -116,57 +105,48 @@ void NvMap::UnmapHandle(Handle& handle_description) {
bool NvMap::TryRemoveHandle(const Handle& handle_description) {
// No dupes left, we can remove from handle map
if (handle_description.dupes == 0 && handle_description.internal_dupes == 0) {
std::scoped_lock lock(handles_lock);
auto it{handles.find(handle_description.id)};
std::scoped_lock l(handles_lock);
auto it = handles.find(handle_description.id);
if (it != handles.end()) {
handles.erase(it);
}
return true;
} else {
return false;
}
}
NvResult NvMap::CreateHandle(u64 size, std::shared_ptr<NvMap::Handle>& result_out) {
if (!size) [[unlikely]] {
NvResult NvMap::CreateHandle(u64 size, Handle::Id& out_handle) {
if (!Common::AlignUp(size, YUZU_PAGESIZE)) {
return NvResult::BadValue;
}
u32 id{next_handle_id.fetch_add(HandleIdIncrement, std::memory_order_relaxed)};
auto handle_description{std::make_shared<Handle>(size, id)};
AddHandle(handle_description);
result_out = handle_description;
u32 id = next_handle_id.fetch_add(HandleIdIncrement, std::memory_order_relaxed);
AddHandle(Handle(size, id));
out_handle = id;
return NvResult::Success;
}
std::shared_ptr<NvMap::Handle> NvMap::GetHandle(Handle::Id handle) {
std::scoped_lock lock(handles_lock);
try {
return handles.at(handle);
} catch (std::out_of_range&) {
return nullptr;
}
std::optional<std::reference_wrapper<NvMap::Handle>> NvMap::GetHandle(Handle::Id handle) {
if (auto const it = handles.find(handle); it != handles.end())
return {it->second};
return std::nullopt;
}
DAddr NvMap::GetHandleAddress(Handle::Id handle) {
std::scoped_lock lock(handles_lock);
try {
return handles.at(handle)->d_address;
} catch (std::out_of_range&) {
return 0;
}
if (auto const it = handles.find(handle); it != handles.end())
return it->second.d_address;
return 0;
}
DAddr NvMap::PinHandle(NvMap::Handle::Id handle, bool low_area_pin) {
auto handle_description{GetHandle(handle)};
if (!handle_description) [[unlikely]] {
std::scoped_lock lock(handles_lock);
auto o = GetHandle(handle);
if (!o) [[unlikely]] {
return 0;
}
std::scoped_lock lock(handle_description->mutex);
auto handle_description = &o->get();
const auto map_low_area = [&] {
if (handle_description->pin_virt_address == 0) {
u32 address = host1x.Allocator().Allocate(u32(handle_description->aligned_size));
@@ -180,17 +160,15 @@ DAddr NvMap::PinHandle(NvMap::Handle::Id handle, bool low_area_pin) {
{
// Lock now to prevent our queue entry from being removed for allocation in-between the
// following check and erase
std::scoped_lock queueLock(unmap_queue_lock);
std::scoped_lock ql(unmap_queue_lock);
if (handle_description->unmap_queue_entry) {
unmap_queue.erase(*handle_description->unmap_queue_entry);
handle_description->unmap_queue_entry.reset();
if (low_area_pin) {
map_low_area();
handle_description->pins++;
return static_cast<DAddr>(handle_description->pin_virt_address);
return DAddr(handle_description->pin_virt_address);
}
handle_description->pins++;
return handle_description->d_address;
}
@@ -211,12 +189,11 @@ DAddr NvMap::PinHandle(NvMap::Handle::Id handle, bool low_area_pin) {
while ((address = smmu.Allocate(aligned_up)) == 0) {
// Free handles until the allocation succeeds
std::scoped_lock queueLock(unmap_queue_lock);
if (auto freeHandleDesc{unmap_queue.front()}) {
if (auto free_handle = handles.find(unmap_queue.front()); free_handle != handles.end()) {
// Handles in the unmap queue are guaranteed not to be pinned so don't bother
// checking if they are before unmapping
std::scoped_lock freeLock(freeHandleDesc->mutex);
if (handle_description->d_address)
UnmapHandle(*freeHandleDesc);
UnmapHandle(free_handle->second);
} else {
LOG_CRITICAL(Service_NVDRV, "Ran out of SMMU address space!");
}
@@ -234,51 +211,44 @@ DAddr NvMap::PinHandle(NvMap::Handle::Id handle, bool low_area_pin) {
handle_description->pins++;
if (low_area_pin) {
return static_cast<DAddr>(handle_description->pin_virt_address);
return DAddr(handle_description->pin_virt_address);
}
return handle_description->d_address;
}
void NvMap::UnpinHandle(Handle::Id handle) {
auto handle_description{GetHandle(handle)};
if (!handle_description) {
return;
}
std::scoped_lock lock(handle_description->mutex);
if (--handle_description->pins < 0) {
LOG_WARNING(Service_NVDRV, "Pin count imbalance detected!");
} else if (!handle_description->pins) {
std::scoped_lock queueLock(unmap_queue_lock);
// Add to the unmap queue allowing this handle's memory to be freed if needed
unmap_queue.push_back(handle_description);
handle_description->unmap_queue_entry = std::prev(unmap_queue.end());
std::scoped_lock lock(handles_lock);
if (auto o = GetHandle(handle); o) {
auto handle_description = &o->get();
if (--handle_description->pins < 0) {
LOG_WARNING(Service_NVDRV, "Pin count imbalance detected!");
} else if (!handle_description->pins) {
std::scoped_lock ql(unmap_queue_lock);
// Add to the unmap queue allowing this handle's memory to be freed if needed
unmap_queue.push_back(handle);
handle_description->unmap_queue_entry = std::prev(unmap_queue.end());
}
}
}
void NvMap::DuplicateHandle(Handle::Id handle, bool internal_session) {
auto handle_description{GetHandle(handle)};
if (!handle_description) {
std::scoped_lock lock(handles_lock);
auto o = GetHandle(handle);
if (!o) {
LOG_CRITICAL(Service_NVDRV, "Unregistered handle!");
return;
}
auto result = handle_description->Duplicate(internal_session);
auto result = o->get().Duplicate(internal_session);
if (result != NvResult::Success) {
LOG_CRITICAL(Service_NVDRV, "Could not duplicate handle!");
}
}
std::optional<NvMap::FreeInfo> NvMap::FreeHandle(Handle::Id handle, bool internal_session) {
std::weak_ptr<Handle> hWeak{GetHandle(handle)};
FreeInfo freeInfo;
// We use a weak ptr here so we can tell when the handle has been freed and report that back to
// guest
if (auto handle_description = hWeak.lock()) {
std::scoped_lock lock(handle_description->mutex);
// We use a weak ptr here so we can tell when the handle has been freed and report that back to guest
std::scoped_lock lock(handles_lock);
if (auto o = GetHandle(handle); o) {
auto handle_description = &o->get();
if (internal_session) {
if (--handle_description->internal_dupes < 0)
LOG_WARNING(Service_NVDRV, "Internal duplicate count imbalance detected!");
@@ -288,25 +258,25 @@ std::optional<NvMap::FreeInfo> NvMap::FreeHandle(Handle::Id handle, bool interna
} else if (handle_description->dupes == 0) {
// Force unmap the handle
if (handle_description->d_address) {
std::scoped_lock queueLock(unmap_queue_lock);
std::scoped_lock ql(unmap_queue_lock);
UnmapHandle(*handle_description);
}
handle_description->pins = 0;
}
}
// Try to remove the shared ptr to the handle from the map, if nothing else is using the
// handle then it will now be freed when `handle_description` goes out of scope
if (TryRemoveHandle(*handle_description)) {
LOG_DEBUG(Service_NVDRV, "Removed nvmap handle: {}", handle);
} else {
LOG_DEBUG(Service_NVDRV,
"Tried to free nvmap handle: {} but didn't as it still has duplicates",
handle);
LOG_DEBUG(Service_NVDRV, "Tried to free nvmap handle: {} but didn't as it still has duplicates", handle);
}
freeInfo = {
// // If the handle hasn't been freed from memory, mark that
// if (!hWeak.expired()) {
// LOG_DEBUG(Service_NVDRV, "nvmap handle: {} wasn't freed as it is still in use", handle);
// freeInfo.can_unlock = false;
// }
return FreeInfo{
.address = handle_description->address,
.size = handle_description->size,
.was_uncached = handle_description->flags.map_uncached.Value() != 0,
@@ -315,30 +285,15 @@ std::optional<NvMap::FreeInfo> NvMap::FreeHandle(Handle::Id handle, bool interna
} else {
return std::nullopt;
}
// If the handle hasn't been freed from memory, mark that
if (!hWeak.expired()) {
LOG_DEBUG(Service_NVDRV, "nvmap handle: {} wasn't freed as it is still in use", handle);
freeInfo.can_unlock = false;
}
return freeInfo;
}
void NvMap::UnmapAllHandles(NvCore::SessionId session_id) {
auto handles_copy = [&] {
std::scoped_lock lk{handles_lock};
return handles;
}();
for (auto& [id, handle] : handles_copy) {
{
std::scoped_lock lk{handle->mutex};
if (handle->session_id.id != session_id.id || handle->dupes <= 0) {
continue;
}
std::scoped_lock lk{handles_lock};
for (auto it = handles.begin(); it != handles.end(); ++it) {
if (it->second.session_id.id != session_id.id || it->second.dupes <= 0) {
continue;
}
FreeHandle(id, false);
FreeHandle(it->first, false);
}
}
+30 -44
View File
@@ -12,6 +12,11 @@
#include <memory>
#include <mutex>
#include <optional>
#if BOOST_VERSION >= 109000
#include <boost/unordered/unordered_node_map.hpp>
#else
#include <unordered_map>
#endif
#include <ankerl/unordered_dense.h>
#include <assert.h>
@@ -31,54 +36,36 @@ class Host1x;
namespace Service::Nvidia::NvCore {
class Container;
/**
* @brief The nvmap core class holds the global state for nvmap and provides methods to manage
* handles
*/
/// @brief The nvmap core class holds the global state for nvmap and provides methods to manage handles
class NvMap {
public:
/**
* @brief A handle to a contiguous block of memory in an application's address space
*/
/// @brief A handle to a contiguous block of memory in an application's address space
struct Handle {
std::mutex mutex;
using Id = u32;
std::optional<typename std::list<Handle::Id>::iterator> unmap_queue_entry{};
u64 align{}; //!< The alignment to use when pinning the handle onto the SMMU
u64 size; //!< Page-aligned size of the memory the handle refers to
u64 aligned_size; //!< `align`-aligned size of the memory the handle refers to
u64 orig_size; //!< Original unaligned size of the memory this handle refers to
DAddr d_address{}; //!< The memory location in the device's AS that this handle corresponds to, this can also be in the nvdrv tmem
VAddr address{}; //!< The memory location in the guest's AS that this handle corresponds to, this can also be in the nvdrv tmem
s64 pins{};
s32 dupes{1}; //!< How many guest references there are to this handle
s32 internal_dupes{0}; //!< How many emulator-internal references there are to this handle
using Id = u32;
Id id; //!< A globally unique identifier for this handle
s64 pins{};
u32 pin_virt_address{};
std::optional<typename std::list<std::shared_ptr<Handle>>::iterator> unmap_queue_entry{};
union Flags {
u32 raw;
BitField<0, 1, u32> map_uncached; //!< If the handle should be mapped as uncached
BitField<2, 1, u32> keep_uncached_after_free; //!< Only applicable when the handle was
//!< allocated with a fixed address
BitField<4, 1, u32> _unk0_; //!< Passed to IOVMM for pins
BitField<2, 1, u32> keep_uncached_after_free; //!< Only applicable when the handle was allocated with a fixed address
BitField<4, 1, u32> _unk0_; //!< Passed to IOVMM for pins
} flags{};
static_assert(sizeof(Flags) == sizeof(u32));
VAddr address{}; //!< The memory location in the guest's AS that this handle corresponds to,
//!< this can also be in the nvdrv tmem
bool is_shared_mem_mapped{}; //!< If this nvmap has been mapped with the MapSharedMem IPC
//!< call
u8 kind{}; //!< Used for memory compression
bool allocated{}; //!< If the handle has been allocated with `Alloc`
bool in_heap{};
NvCore::SessionId session_id{};
DAddr d_address{}; //!< The memory location in the device's AS that this handle corresponds
//!< to, this can also be in the nvdrv tmem
u8 kind{}; //!< Used for memory compression
bool allocated : 1 = false; //!< If the handle has been allocated with `Alloc`
bool in_heap : 1 = false;
bool is_shared_mem_mapped : 1 = false; //!< If this nvmap has been mapped with the MapSharedMem IPC < call
Handle(u64 size, Id id);
@@ -123,9 +110,9 @@ public:
/**
* @brief Creates an unallocated handle of the given size
*/
[[nodiscard]] NvResult CreateHandle(u64 size, std::shared_ptr<NvMap::Handle>& result_out);
[[nodiscard]] NvResult CreateHandle(u64 size, Handle::Id& out_handle);
std::shared_ptr<Handle> GetHandle(Handle::Id handle);
std::optional<std::reference_wrapper<Handle>> GetHandle(Handle::Id handle);
DAddr GetHandleAddress(Handle::Id handle);
@@ -157,20 +144,21 @@ public:
void UnmapAllHandles(NvCore::SessionId session_id);
private:
std::list<std::shared_ptr<Handle>> unmap_queue{};
std::list<Handle::Id> unmap_queue{};
/// Main owning map of handles
#if BOOST_VERSION >= 109000
boost::unordered_node_map<Handle::Id, Handle> handles{};
#else
std::unordered_map<Handle::Id, Handle> handles{};
#endif
std::mutex unmap_queue_lock{}; //!< Protects access to `unmap_queue`
ankerl::unordered_dense::map<Handle::Id, std::shared_ptr<Handle>>
handles{}; //!< Main owning map of handles
std::mutex handles_lock; //!< Protects access to `handles`
static constexpr u32 HandleIdIncrement{
4}; //!< Each new handle ID is an increment of 4 from the previous
static constexpr u32 HandleIdIncrement{4}; //!< Each new handle ID is an increment of 4 from the previous
std::atomic<u32> next_handle_id{HandleIdIncrement};
Tegra::Host1x::Host1x& host1x;
Container& core;
void AddHandle(std::shared_ptr<Handle> handle);
void AddHandle(Handle&& handle);
/**
* @brief Unmaps and frees the SMMU memory region a handle is mapped to
@@ -184,7 +172,5 @@ private:
* @return If the handle was removed from the map
*/
bool TryRemoveHandle(const Handle& handle_description);
Container& core;
};
} // namespace Service::Nvidia::NvCore
@@ -329,10 +329,11 @@ NvResult nvhost_as_gpu::MapBufferEx(IoctlMapBufferEx& params) {
}
}
auto handle{nvmap.GetHandle(params.handle)};
if (!handle) {
auto o = nvmap.GetHandle(params.handle);
if (!o) {
return NvResult::BadValue;
}
auto handle = &o->get();
DAddr device_address = DAddr(nvmap.PinHandle(params.handle, false) + params.buffer_offset);
u64 size{params.mapping_size ? params.mapping_size : handle->orig_size};
@@ -103,17 +103,13 @@ NvResult nvhost_nvdec_common::Submit(IoctlSubmit& params, std::span<u8> data, De
for (std::size_t i = 0; i < syncpt_increments.size(); i++) {
const SyncptIncr& syncpt_incr = syncpt_increments[i];
fence_thresholds[i] =
syncpoint_manager.IncrementSyncpointMaxExt(syncpt_incr.id, syncpt_incr.increments);
fence_thresholds[i] = syncpoint_manager.IncrementSyncpointMaxExt(syncpt_incr.id, syncpt_incr.increments);
}
for (const auto& cmd_buffer : command_buffers) {
const auto object = nvmap.GetHandle(cmd_buffer.memory_id);
ASSERT_OR_EXECUTE(object, return NvResult::InvalidState;);
Core::Memory::CpuGuestMemory<Tegra::ChCommandHeader,
Core::Memory::GuestMemoryFlags::SafeRead>
cmdlist(session->process->GetMemory(), object->address + cmd_buffer.offset,
cmd_buffer.word_count);
Core::Memory::CpuGuestMemory<Tegra::ChCommandHeader, Core::Memory::GuestMemoryFlags::SafeRead> cmdlist(session->process->GetMemory(), object->get().address + cmd_buffer.offset, cmd_buffer.word_count);
host1x.PushEntries(fd, std::move(cmdlist));
}
+23 -25
View File
@@ -83,17 +83,14 @@ void nvmap::OnClose(DeviceFD fd) {
NvResult nvmap::IocCreate(IocCreateParams& params) {
LOG_DEBUG(Service_NVDRV, "called, size=0x{:08X}", params.size);
std::shared_ptr<NvCore::NvMap::Handle> handle_description{};
auto result =
file.CreateHandle(Common::AlignUp(params.size, YUZU_PAGESIZE), handle_description);
NvCore::NvMap::Handle handle_description(0, 0);
// Orig size is the unaligned size, set the handle to that
auto result = file.CreateHandle(params.size, params.handle);
if (result != NvResult::Success) {
LOG_CRITICAL(Service_NVDRV, "Failed to create Object");
return result;
}
handle_description->orig_size = params.size; // Orig size is the unaligned size
params.handle = handle_description->id;
LOG_DEBUG(Service_NVDRV, "handle: {}, size: {:#X}", handle_description->id, params.size);
LOG_DEBUG(Service_NVDRV, "handle: {}, size: {:#X}", params.handle, params.size);
return NvResult::Success;
}
@@ -115,30 +112,27 @@ NvResult nvmap::IocAlloc(IocAllocParams& params, DeviceFD fd) {
params.align = YUZU_PAGESIZE;
}
auto handle_description{file.GetHandle(params.handle)};
if (!handle_description) {
std::scoped_lock lock(file.handles_lock);
auto o = file.GetHandle(params.handle);
if (!o) {
LOG_CRITICAL(Service_NVDRV, "Object does not exist, handle={:08X}", params.handle);
return NvResult::BadValue;
}
auto handle_description = &o->get();
if (handle_description->allocated) {
LOG_CRITICAL(Service_NVDRV, "Object is already allocated, handle={:08X}", params.handle);
return NvResult::InsufficientMemory;
}
const auto result = handle_description->Alloc(params.flags, params.align, params.kind,
params.address, sessions[fd]);
const auto result = handle_description->Alloc(params.flags, params.align, params.kind, params.address, sessions[fd]);
if (result != NvResult::Success) {
LOG_CRITICAL(Service_NVDRV, "Object failed to allocate, handle={:08X}", params.handle);
return result;
}
bool is_out_io{};
auto process = container.GetSession(sessions[fd])->process;
ASSERT(process->GetPageTable()
.LockForMapDeviceAddressSpace(&is_out_io, handle_description->address,
handle_description->size,
Kernel::KMemoryPermission::None, true, false)
.IsSuccess());
ASSERT(process->GetPageTable().LockForMapDeviceAddressSpace(&is_out_io, handle_description->address, handle_description->size, Kernel::KMemoryPermission::None, true, false).IsSuccess());
return result;
}
@@ -151,13 +145,13 @@ NvResult nvmap::IocGetId(IocGetIdParams& params) {
return NvResult::BadValue;
}
auto handle_description{file.GetHandle(params.handle)};
if (!handle_description) {
std::scoped_lock lock(file.handles_lock);
auto o = file.GetHandle(params.handle);
if (!o) {
LOG_CRITICAL(Service_NVDRV, "Error!");
return NvResult::AccessDenied; // This will always return EPERM irrespective of if the
// handle exists or not
return NvResult::AccessDenied; // This will always return EPERM irrespective of if the handle exists or not
}
auto handle_description = &o->get();
params.id = handle_description->id;
return NvResult::Success;
}
@@ -174,12 +168,14 @@ NvResult nvmap::IocFromId(IocFromIdParams& params) {
return NvResult::BadValue;
}
auto handle_description{file.GetHandle(params.id)};
if (!handle_description) {
std::scoped_lock lock(file.handles_lock);
auto o = file.GetHandle(params.id);
if (!o) {
LOG_CRITICAL(Service_NVDRV, "Unregistered handle!");
return NvResult::BadValue;
}
auto handle_description = &o->get();
auto result = handle_description->Duplicate(false);
if (result != NvResult::Success) {
LOG_CRITICAL(Service_NVDRV, "Could not duplicate handle!");
@@ -199,12 +195,14 @@ NvResult nvmap::IocParam(IocParamParams& params) {
return NvResult::BadValue;
}
auto handle_description{file.GetHandle(params.handle)};
if (!handle_description) {
std::scoped_lock lock(file.handles_lock);
auto o = file.GetHandle(params.handle);
if (!o) {
LOG_CRITICAL(Service_NVDRV, "Not registered handle!");
return NvResult::BadValue;
}
auto handle_description = &o->get();
switch (params.param) {
case HandleParameterType::Size:
params.result = static_cast<u32_le>(handle_description->orig_size);
+4
View File
@@ -169,6 +169,8 @@ if ("x86_64" IN_LIST ARCHITECTURE)
backend/x64/emit_x64_vector.cpp
backend/x64/emit_x64_vector_floating_point.cpp
backend/x64/emit_x64_vector_saturation.cpp
backend/x64/exclusive_monitor.cpp
backend/x64/exclusive_monitor_friend.h
backend/x64/host_feature.h
backend/x64/hostloc.h
backend/x64/jitstate_info.h
@@ -229,6 +231,7 @@ if ("arm64" IN_LIST ARCHITECTURE)
backend/arm64/emit_arm64_vector_floating_point.cpp
backend/arm64/emit_arm64_vector_saturation.cpp
backend/arm64/emit_context.h
backend/arm64/exclusive_monitor.cpp
backend/arm64/fastmem.h
backend/arm64/fpsr_manager.cpp
backend/arm64/fpsr_manager.h
@@ -275,6 +278,7 @@ if ("riscv64" IN_LIST ARCHITECTURE)
backend/riscv64/emit_riscv64_vector.cpp
backend/riscv64/emit_riscv64.cpp
backend/riscv64/emit_riscv64.h
backend/riscv64/exclusive_monitor.cpp
backend/riscv64/reg_alloc.cpp
backend/riscv64/reg_alloc.h
backend/riscv64/stack_layout.h
@@ -135,9 +135,12 @@ static void* EmitExclusiveWriteCallTrampoline(oaknut::CodeGenerator& code, const
oaknut::Label l_addr, l_this;
auto fn = [](const A64::UserConfig& conf, A64::VAddr vaddr, T value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<T>(conf.processor_id, vaddr, [&](T expected) -> bool {
return (conf.callbacks->*callback)(vaddr, value, expected);
}) ? 0 : 1;
return conf.global_monitor->DoExclusiveOperation<T>(conf.processor_id, vaddr,
[&](T expected) -> bool {
return (conf.callbacks->*callback)(vaddr, value, expected);
})
? 0
: 1;
};
void* target = code.xptr<void*>();
@@ -297,9 +300,12 @@ static void* EmitExclusiveWrite128CallTrampoline(oaknut::CodeGenerator& code, co
oaknut::Label l_addr, l_this;
auto fn = [](const A64::UserConfig& conf, A64::VAddr vaddr, Vector value) -> u32 {
return conf.global_monitor->DoExclusiveOperation<Vector>(conf.processor_id, vaddr, [&](Vector expected) -> bool {
return conf.callbacks->MemoryWriteExclusive128(vaddr, value, expected);
}) ? 0 : 1;
return conf.global_monitor->DoExclusiveOperation<Vector>(conf.processor_id, vaddr,
[&](Vector expected) -> bool {
return conf.callbacks->MemoryWriteExclusive128(vaddr, value, expected);
})
? 0
: 1;
};
void* target = code.xptr<void*>();
@@ -0,0 +1,61 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
/* This file is part of the dynarmic project.
* Copyright (c) 2022 MerryMage
* SPDX-License-Identifier: 0BSD
*/
#include "dynarmic/interface/exclusive_monitor.h"
#include <algorithm>
#include "common/assert.h"
namespace Dynarmic {
ExclusiveMonitor::ExclusiveMonitor(std::size_t processor_count)
: exclusive_addresses(processor_count, INVALID_EXCLUSIVE_ADDRESS), exclusive_values(processor_count) {}
size_t ExclusiveMonitor::GetProcessorCount() const {
return exclusive_addresses.size();
}
void ExclusiveMonitor::Lock() {
lock.Lock();
}
void ExclusiveMonitor::Unlock() {
lock.Unlock();
}
bool ExclusiveMonitor::CheckAndClear(std::size_t processor_id, VAddr address) {
const VAddr masked_address = address & RESERVATION_GRANULE_MASK;
Lock();
if (exclusive_addresses[processor_id] != masked_address) {
Unlock();
return false;
}
for (VAddr& other_address : exclusive_addresses) {
if (other_address == masked_address) {
other_address = INVALID_EXCLUSIVE_ADDRESS;
}
}
return true;
}
void ExclusiveMonitor::Clear() {
Lock();
std::fill(exclusive_addresses.begin(), exclusive_addresses.end(), INVALID_EXCLUSIVE_ADDRESS);
Unlock();
}
void ExclusiveMonitor::ClearProcessor(std::size_t processor_id) {
Lock();
exclusive_addresses[processor_id] = INVALID_EXCLUSIVE_ADDRESS;
Unlock();
}
} // namespace Dynarmic
@@ -0,0 +1,54 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#include "dynarmic/interface/exclusive_monitor.h"
#include <algorithm>
namespace Dynarmic {
ExclusiveMonitor::ExclusiveMonitor(std::size_t processor_count)
: exclusive_addresses(processor_count, INVALID_EXCLUSIVE_ADDRESS), exclusive_values(processor_count) {}
size_t ExclusiveMonitor::GetProcessorCount() const {
return exclusive_addresses.size();
}
void ExclusiveMonitor::Lock() {
lock.Lock();
}
void ExclusiveMonitor::Unlock() {
lock.Unlock();
}
bool ExclusiveMonitor::CheckAndClear(size_t processor_id, VAddr address) {
const VAddr masked_address = address & RESERVATION_GRANULE_MASK;
Lock();
if (exclusive_addresses[processor_id] != masked_address) {
Unlock();
return false;
}
for (VAddr& other_address : exclusive_addresses) {
if (other_address == masked_address) {
other_address = INVALID_EXCLUSIVE_ADDRESS;
}
}
return true;
}
void ExclusiveMonitor::Clear() {
Lock();
std::fill(exclusive_addresses.begin(), exclusive_addresses.end(), INVALID_EXCLUSIVE_ADDRESS);
Unlock();
}
void ExclusiveMonitor::ClearProcessor(size_t processor_id) {
Lock();
exclusive_addresses[processor_id] = INVALID_EXCLUSIVE_ADDRESS;
Unlock();
}
} // namespace Dynarmic
@@ -20,7 +20,7 @@
#include "dynarmic/backend/x64/abi.h"
#include "dynarmic/backend/x64/devirtualize.h"
#include "dynarmic/backend/x64/emit_x64_memory.h"
#include "dynarmic/interface/exclusive_monitor.h"
#include "dynarmic/backend/x64/exclusive_monitor_friend.h"
#include "dynarmic/backend/x64/perf_map.h"
#include "dynarmic/interface/exclusive_monitor.h"
@@ -174,35 +174,67 @@ void A32EmitX64::EmitA32ClearExclusive(A32EmitContext&, IR::Inst*) {
}
void A32EmitX64::EmitA32ExclusiveReadMemory8(A32EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveReadMemoryInline<8, &A32::UserCallbacks::MemoryRead8>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveReadMemoryInline<8, &A32::UserCallbacks::MemoryRead8>(ctx, inst);
} else {
EmitExclusiveReadMemory<8, &A32::UserCallbacks::MemoryRead8>(ctx, inst);
}
}
void A32EmitX64::EmitA32ExclusiveReadMemory16(A32EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveReadMemoryInline<16, &A32::UserCallbacks::MemoryRead16>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveReadMemoryInline<16, &A32::UserCallbacks::MemoryRead16>(ctx, inst);
} else {
EmitExclusiveReadMemory<16, &A32::UserCallbacks::MemoryRead16>(ctx, inst);
}
}
void A32EmitX64::EmitA32ExclusiveReadMemory32(A32EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveReadMemoryInline<32, &A32::UserCallbacks::MemoryRead32>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveReadMemoryInline<32, &A32::UserCallbacks::MemoryRead32>(ctx, inst);
} else {
EmitExclusiveReadMemory<32, &A32::UserCallbacks::MemoryRead32>(ctx, inst);
}
}
void A32EmitX64::EmitA32ExclusiveReadMemory64(A32EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveReadMemoryInline<64, &A32::UserCallbacks::MemoryRead64>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveReadMemoryInline<64, &A32::UserCallbacks::MemoryRead64>(ctx, inst);
} else {
EmitExclusiveReadMemory<64, &A32::UserCallbacks::MemoryRead64>(ctx, inst);
}
}
void A32EmitX64::EmitA32ExclusiveWriteMemory8(A32EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveWriteMemoryInline<8, &A32::UserCallbacks::MemoryWriteExclusive8>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveWriteMemoryInline<8, &A32::UserCallbacks::MemoryWriteExclusive8>(ctx, inst);
} else {
EmitExclusiveWriteMemory<8, &A32::UserCallbacks::MemoryWriteExclusive8>(ctx, inst);
}
}
void A32EmitX64::EmitA32ExclusiveWriteMemory16(A32EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveWriteMemoryInline<16, &A32::UserCallbacks::MemoryWriteExclusive16>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveWriteMemoryInline<16, &A32::UserCallbacks::MemoryWriteExclusive16>(ctx, inst);
} else {
EmitExclusiveWriteMemory<16, &A32::UserCallbacks::MemoryWriteExclusive16>(ctx, inst);
}
}
void A32EmitX64::EmitA32ExclusiveWriteMemory32(A32EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveWriteMemoryInline<32, &A32::UserCallbacks::MemoryWriteExclusive32>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveWriteMemoryInline<32, &A32::UserCallbacks::MemoryWriteExclusive32>(ctx, inst);
} else {
EmitExclusiveWriteMemory<32, &A32::UserCallbacks::MemoryWriteExclusive32>(ctx, inst);
}
}
void A32EmitX64::EmitA32ExclusiveWriteMemory64(A32EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveWriteMemoryInline<64, &A32::UserCallbacks::MemoryWriteExclusive64>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveWriteMemoryInline<64, &A32::UserCallbacks::MemoryWriteExclusive64>(ctx, inst);
} else {
EmitExclusiveWriteMemory<64, &A32::UserCallbacks::MemoryWriteExclusive64>(ctx, inst);
}
}
void A32EmitX64::EmitCheckMemoryAbort(A32EmitContext& ctx, IR::Inst* inst, Xbyak::Label* end) {
@@ -20,7 +20,7 @@
#include "dynarmic/backend/x64/abi.h"
#include "dynarmic/backend/x64/devirtualize.h"
#include "dynarmic/backend/x64/emit_x64_memory.h"
#include "dynarmic/interface/exclusive_monitor.h"
#include "dynarmic/backend/x64/exclusive_monitor_friend.h"
#include "dynarmic/backend/x64/perf_map.h"
#include "dynarmic/common/spin_lock_x64.h"
#include "dynarmic/interface/exclusive_monitor.h"
@@ -330,43 +330,83 @@ void A64EmitX64::EmitA64ClearExclusive(A64EmitContext&, IR::Inst*) {
}
void A64EmitX64::EmitA64ExclusiveReadMemory8(A64EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveReadMemoryInline<8, &A64::UserCallbacks::MemoryRead8>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveReadMemoryInline<8, &A64::UserCallbacks::MemoryRead8>(ctx, inst);
} else {
EmitExclusiveReadMemory<8, &A64::UserCallbacks::MemoryRead8>(ctx, inst);
}
}
void A64EmitX64::EmitA64ExclusiveReadMemory16(A64EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveReadMemoryInline<16, &A64::UserCallbacks::MemoryRead16>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveReadMemoryInline<16, &A64::UserCallbacks::MemoryRead16>(ctx, inst);
} else {
EmitExclusiveReadMemory<16, &A64::UserCallbacks::MemoryRead16>(ctx, inst);
}
}
void A64EmitX64::EmitA64ExclusiveReadMemory32(A64EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveReadMemoryInline<32, &A64::UserCallbacks::MemoryRead32>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveReadMemoryInline<32, &A64::UserCallbacks::MemoryRead32>(ctx, inst);
} else {
EmitExclusiveReadMemory<32, &A64::UserCallbacks::MemoryRead32>(ctx, inst);
}
}
void A64EmitX64::EmitA64ExclusiveReadMemory64(A64EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveReadMemoryInline<64, &A64::UserCallbacks::MemoryRead64>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveReadMemoryInline<64, &A64::UserCallbacks::MemoryRead64>(ctx, inst);
} else {
EmitExclusiveReadMemory<64, &A64::UserCallbacks::MemoryRead64>(ctx, inst);
}
}
void A64EmitX64::EmitA64ExclusiveReadMemory128(A64EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveReadMemoryInline<128, &A64::UserCallbacks::MemoryRead128>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveReadMemoryInline<128, &A64::UserCallbacks::MemoryRead128>(ctx, inst);
} else {
EmitExclusiveReadMemory<128, &A64::UserCallbacks::MemoryRead128>(ctx, inst);
}
}
void A64EmitX64::EmitA64ExclusiveWriteMemory8(A64EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveWriteMemoryInline<8, &A64::UserCallbacks::MemoryWriteExclusive8>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveWriteMemoryInline<8, &A64::UserCallbacks::MemoryWriteExclusive8>(ctx, inst);
} else {
EmitExclusiveWriteMemory<8, &A64::UserCallbacks::MemoryWriteExclusive8>(ctx, inst);
}
}
void A64EmitX64::EmitA64ExclusiveWriteMemory16(A64EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveWriteMemoryInline<16, &A64::UserCallbacks::MemoryWriteExclusive16>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveWriteMemoryInline<16, &A64::UserCallbacks::MemoryWriteExclusive16>(ctx, inst);
} else {
EmitExclusiveWriteMemory<16, &A64::UserCallbacks::MemoryWriteExclusive16>(ctx, inst);
}
}
void A64EmitX64::EmitA64ExclusiveWriteMemory32(A64EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveWriteMemoryInline<32, &A64::UserCallbacks::MemoryWriteExclusive32>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveWriteMemoryInline<32, &A64::UserCallbacks::MemoryWriteExclusive32>(ctx, inst);
} else {
EmitExclusiveWriteMemory<32, &A64::UserCallbacks::MemoryWriteExclusive32>(ctx, inst);
}
}
void A64EmitX64::EmitA64ExclusiveWriteMemory64(A64EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveWriteMemoryInline<64, &A64::UserCallbacks::MemoryWriteExclusive64>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveWriteMemoryInline<64, &A64::UserCallbacks::MemoryWriteExclusive64>(ctx, inst);
} else {
EmitExclusiveWriteMemory<64, &A64::UserCallbacks::MemoryWriteExclusive64>(ctx, inst);
}
}
void A64EmitX64::EmitA64ExclusiveWriteMemory128(A64EmitContext& ctx, IR::Inst* inst) {
EmitExclusiveWriteMemoryInline<128, &A64::UserCallbacks::MemoryWriteExclusive128>(ctx, inst);
if (conf.fastmem_exclusive_access) {
EmitExclusiveWriteMemoryInline<128, &A64::UserCallbacks::MemoryWriteExclusive128>(ctx, inst);
} else {
EmitExclusiveWriteMemory<128, &A64::UserCallbacks::MemoryWriteExclusive128>(ctx, inst);
}
}
void A64EmitX64::EmitCheckMemoryAbort(A64EmitContext&, IR::Inst* inst, Xbyak::Label* end) {
@@ -230,11 +230,12 @@ void AxxEmitX64::EmitExclusiveReadMemory(AxxEmitContext& ctx, IR::Inst* inst) {
if (ordered) {
code.mfence();
}
code.CallLambda([](AxxUserConfig& conf, Axx::VAddr vaddr) -> T {
return conf.global_monitor->ReadAndMark<T>(conf.processor_id, vaddr, [&]() -> T {
return (conf.callbacks->*callback)(vaddr);
code.CallLambda(
[](AxxUserConfig& conf, Axx::VAddr vaddr) -> T {
return conf.global_monitor->ReadAndMark<T>(conf.processor_id, vaddr, [&]() -> T {
return (conf.callbacks->*callback)(vaddr);
});
});
});
code.ZeroExtendFrom(bitsize, code.ABI_RETURN);
} else {
const Xbyak::Xmm result = ctx.reg_alloc.ScratchXmm(code);
@@ -249,11 +250,12 @@ void AxxEmitX64::EmitExclusiveReadMemory(AxxEmitContext& ctx, IR::Inst* inst) {
if (ordered) {
code.mfence();
}
code.CallLambda([](AxxUserConfig& conf, Axx::VAddr vaddr, Vector& ret) {
ret = conf.global_monitor->ReadAndMark<Vector>(conf.processor_id, vaddr, [&]() -> Vector {
return (conf.callbacks->*callback)(vaddr);
code.CallLambda(
[](AxxUserConfig& conf, Axx::VAddr vaddr, Vector& ret) {
ret = conf.global_monitor->ReadAndMark<Vector>(conf.processor_id, vaddr, [&]() -> Vector {
return (conf.callbacks->*callback)(vaddr);
});
});
});
code.movups(result, xword[rsp + ABI_SHADOW_SPACE]);
ctx.reg_alloc.ReleaseStackSpace(code, 16 + ABI_SHADOW_SPACE);
@@ -318,12 +320,11 @@ void AxxEmitX64::EmitExclusiveWriteMemory(AxxEmitContext& ctx, IR::Inst* inst) {
template<std::size_t bitsize, auto callback>
void AxxEmitX64::EmitExclusiveReadMemoryInline(AxxEmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor);
if (!conf.fastmem_exclusive_access || !exception_handler.SupportsFastmem()) {
ASSERT(conf.global_monitor && conf.fastmem_pointer);
if (!exception_handler.SupportsFastmem()) {
EmitExclusiveReadMemory<bitsize, callback>(ctx, inst);
return;
}
ASSERT(conf.fastmem_pointer);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
constexpr bool ordered = true;
@@ -343,10 +344,10 @@ void AxxEmitX64::EmitExclusiveReadMemoryInline(AxxEmitContext& ctx, IR::Inst* in
const auto wrapped_fn = read_fallbacks[std::make_tuple(ordered, bitsize, vaddr.getIdx(), value_idx)];
EmitExclusiveLock(code, conf, tmp2.cvt32());
EmitExclusiveLock(code, conf, tmp, tmp2.cvt32());
code.mov(code.byte[code.ABI_JIT_PTR + offsetof(AxxJitState, exclusive_state)], u8(1));
code.mov(tmp, std::bit_cast<u64>(conf.global_monitor->exclusive_addresses.data() + conf.processor_id));
code.mov(tmp, std::bit_cast<u64>(GetExclusiveMonitorAddressPointer(conf.global_monitor, conf.processor_id)));
code.mov(qword[tmp], vaddr);
const auto fastmem_marker = ShouldFastmem(ctx, inst);
@@ -380,10 +381,10 @@ void AxxEmitX64::EmitExclusiveReadMemoryInline(AxxEmitContext& ctx, IR::Inst* in
code.call(wrapped_fn);
}
code.mov(tmp, std::bit_cast<u64>(conf.global_monitor->exclusive_values.data() + conf.processor_id));
code.mov(tmp, std::bit_cast<u64>(GetExclusiveMonitorValuePointer(conf.global_monitor, conf.processor_id)));
EmitWriteMemoryMov<bitsize>(code, tmp, value_idx, false);
EmitExclusiveUnlock(code, conf, tmp2.cvt32());
EmitExclusiveUnlock(code, conf, tmp, tmp2.cvt32());
if constexpr (bitsize == 128) {
ctx.reg_alloc.DefineValue(code, inst, Xbyak::Xmm{value_idx});
@@ -396,12 +397,11 @@ void AxxEmitX64::EmitExclusiveReadMemoryInline(AxxEmitContext& ctx, IR::Inst* in
template<std::size_t bitsize, auto callback>
void AxxEmitX64::EmitExclusiveWriteMemoryInline(AxxEmitContext& ctx, IR::Inst* inst) {
ASSERT(conf.global_monitor);
if (!conf.fastmem_exclusive_access || !exception_handler.SupportsFastmem()) {
ASSERT(conf.global_monitor && conf.fastmem_pointer);
if (!exception_handler.SupportsFastmem()) {
EmitExclusiveWriteMemory<bitsize, callback>(ctx, inst);
return;
}
ASSERT(conf.fastmem_pointer);
auto args = ctx.reg_alloc.GetArgumentInfo(inst);
constexpr bool ordered = true;
@@ -425,7 +425,7 @@ void AxxEmitX64::EmitExclusiveWriteMemoryInline(AxxEmitContext& ctx, IR::Inst* i
const auto wrapped_fn = exclusive_write_fallbacks[std::make_tuple(ordered, bitsize, vaddr.getIdx(), value.getIdx())];
EmitExclusiveLock(code, conf, tmp2.cvt32());
EmitExclusiveLock(code, conf, tmp, tmp2.cvt32());
SharedLabel end = ctx.GenSharedLabel();
@@ -433,14 +433,14 @@ void AxxEmitX64::EmitExclusiveWriteMemoryInline(AxxEmitContext& ctx, IR::Inst* i
code.movzx(tmp.cvt32(), code.byte[code.ABI_JIT_PTR + offsetof(AxxJitState, exclusive_state)]);
code.test(tmp.cvt8(), tmp.cvt8());
code.je(*end, code.T_NEAR);
code.mov(tmp, std::bit_cast<u64>(conf.global_monitor->exclusive_addresses.data() + conf.processor_id));
code.mov(tmp, std::bit_cast<u64>(GetExclusiveMonitorAddressPointer(conf.global_monitor, conf.processor_id)));
code.cmp(qword[tmp], vaddr);
code.jne(*end, code.T_NEAR);
EmitExclusiveTestAndClear(code, conf, vaddr, tmp, rax);
code.mov(code.byte[code.ABI_JIT_PTR + offsetof(AxxJitState, exclusive_state)], u8(0));
code.mov(tmp, std::bit_cast<u64>(conf.global_monitor->exclusive_values.data() + conf.processor_id));
code.mov(tmp, std::bit_cast<u64>(GetExclusiveMonitorValuePointer(conf.global_monitor, conf.processor_id)));
if constexpr (bitsize == 128) {
code.mov(rax, qword[tmp + 0]);
@@ -519,7 +519,7 @@ void AxxEmitX64::EmitExclusiveWriteMemoryInline(AxxEmitContext& ctx, IR::Inst* i
}
code.L(*end);
EmitExclusiveUnlock(code, conf, eax);
EmitExclusiveUnlock(code, conf, tmp, eax);
ctx.reg_alloc.DefineValue(code, inst, status);
EmitCheckMemoryAbort(ctx, inst);
}
@@ -13,7 +13,7 @@
#include "dynarmic/backend/x64/a32_emit_x64.h"
#include "dynarmic/backend/x64/a64_emit_x64.h"
#include "dynarmic/interface/exclusive_monitor.h"
#include "dynarmic/backend/x64/exclusive_monitor_friend.h"
#include "dynarmic/common/spin_lock_x64.h"
#include "dynarmic/interface/exclusive_monitor.h"
#include "dynarmic/ir/acc_type.h"
@@ -344,36 +344,43 @@ const void* EmitWriteMemoryMov(BlockOfCode& code, const Xbyak::RegExp& addr, int
}
template<typename UserConfig>
void EmitExclusiveLock(BlockOfCode& code, const UserConfig& conf, Xbyak::Reg32 tmp) {
if (!conf.HasOptimization(OptimizationFlag::Unsafe_IgnoreGlobalMonitor)) {
u64 const slp = std::bit_cast<u64>(std::addressof(conf.global_monitor->lock.storage));
EmitSpinLockLock(code, dword[slp], tmp, code.HasHostFeature(HostFeature::WAITPKG));
void EmitExclusiveLock(BlockOfCode& code, const UserConfig& conf, Xbyak::Reg64 pointer, Xbyak::Reg32 tmp) {
if (conf.HasOptimization(OptimizationFlag::Unsafe_IgnoreGlobalMonitor)) {
return;
}
code.mov(pointer, std::bit_cast<u64>(GetExclusiveMonitorLockPointer(conf.global_monitor)));
EmitSpinLockLock(code, pointer, tmp, code.HasHostFeature(HostFeature::WAITPKG));
}
template<typename UserConfig>
void EmitExclusiveUnlock(BlockOfCode& code, const UserConfig& conf, Xbyak::Reg32 tmp) {
if (!conf.HasOptimization(OptimizationFlag::Unsafe_IgnoreGlobalMonitor)) {
u64 const slp = std::bit_cast<u64>(std::addressof(conf.global_monitor->lock.storage));
EmitSpinLockUnlock(code, dword[slp], tmp);
void EmitExclusiveUnlock(BlockOfCode& code, const UserConfig& conf, Xbyak::Reg64 pointer, Xbyak::Reg32 tmp) {
if (conf.HasOptimization(OptimizationFlag::Unsafe_IgnoreGlobalMonitor)) {
return;
}
code.mov(pointer, std::bit_cast<u64>(GetExclusiveMonitorLockPointer(conf.global_monitor)));
EmitSpinLockUnlock(code, pointer, tmp);
}
template<typename UserConfig>
void EmitExclusiveTestAndClear(BlockOfCode& code, const UserConfig& conf, Xbyak::Reg64 vaddr, Xbyak::Reg64 pointer, Xbyak::Reg64 tmp) {
if (!conf.HasOptimization(OptimizationFlag::Unsafe_IgnoreGlobalMonitor)) {
code.mov(tmp, 0xDEAD'DEAD'DEAD'DEAD);
static_assert(ExclusiveMonitor::MAX_NUM_CPU_CORES == 4);
for (size_t i = 0; i < ExclusiveMonitor::MAX_NUM_CPU_CORES; i++) {
if (i != conf.processor_id) {
Xbyak::Label ok;
code.mov(pointer, std::bit_cast<u64>(conf.global_monitor->exclusive_addresses.data() + i));
code.cmp(qword[pointer], vaddr);
code.jne(ok, code.T_NEAR);
code.mov(qword[pointer], tmp);
code.L(ok);
}
if (conf.HasOptimization(OptimizationFlag::Unsafe_IgnoreGlobalMonitor)) {
return;
}
code.mov(tmp, 0xDEAD'DEAD'DEAD'DEAD);
const size_t processor_count = GetExclusiveMonitorProcessorCount(conf.global_monitor);
for (size_t processor_index = 0; processor_index < processor_count; processor_index++) {
if (processor_index == conf.processor_id) {
continue;
}
Xbyak::Label ok;
code.mov(pointer, std::bit_cast<u64>(GetExclusiveMonitorAddressPointer(conf.global_monitor, processor_index)));
code.cmp(qword[pointer], vaddr);
code.jne(ok, code.T_NEAR);
code.mov(qword[pointer], tmp);
code.L(ok);
}
}
@@ -22,33 +22,25 @@ static const auto default_cg_mode = nullptr; //Allow RWE
namespace Dynarmic {
/// @brief Emits a lock path for a given spinlock
/// @arg ptr Operand must be a dword[ptr]
/// @arg waitpkg Whetever or not the "UMWAIT" instruction can be used
void EmitSpinLockLock(Xbyak::CodeGenerator& code, Xbyak::Address ptr, Xbyak::Reg32 tmp, bool waitpkg) {
// TODO: this is because we lack regalloc - so better to be safe :(
// TODO: really involve regalloc when we require a 64 bit disp temporal... aside from the one we got handed of course
void EmitSpinLockLock(Xbyak::CodeGenerator& code, Xbyak::Reg64 ptr, Xbyak::Reg32 tmp, bool waitpkg) {
Xbyak::Label start, loop;
code.jmp(start, code.T_NEAR);
code.L(loop);
if (waitpkg) {
Xbyak::Label start, loop;
code.jmp(start, code.T_NEAR);
code.L(loop);
code.push(Xbyak::util::eax);
code.push(Xbyak::util::ebx);
code.push(Xbyak::util::edx);
// TODO: this is because we lack regalloc - so better to be safe :(
code.push(Xbyak::util::rax);
code.push(Xbyak::util::rbx);
code.push(Xbyak::util::rdx);
// TODO: This clobbers EAX and EDX did we tell the regalloc?
// ARM ptr for address-monitoring
// XBYAK BUG: code.umonitor(ptr); see issue #255
// replace once xbyak has been fixed
if (ptr.isREG()) {
code.db(0xF3);
if (ptr.getIdx() >= 8) code.db(0x41);
code.db(0x0F); code.db(0xAE);
code.db(uint8_t((3 << 6) | ((6 & 7) << 3) | (ptr.getIdx() & 7)));
} else {
code.mov(Xbyak::util::rax, ptr);
code.umonitor(Xbyak::util::rax);
}
code.db(0xF3);
if (ptr.getIdx() >= 8) code.db(0x41);
code.db(0x0F); code.db(0xAE);
code.db(uint8_t((3 << 6) | ((6 & 7) << 3) | (ptr.getIdx() & 7)));
// tmp.bit[0] = 0: C0.1 | Slow Wakup | Better Savings
// tmp.bit[0] = 1: C0.2 | Fast Wakup | Lesser Savings
@@ -63,64 +55,22 @@ void EmitSpinLockLock(Xbyak::CodeGenerator& code, Xbyak::Address ptr, Xbyak::Reg
code.umwait(Xbyak::util::ebx);
// CF == 1 if we hit the OS-timeout in IA32_UMWAIT_CONTROL without a write
// CF == 0 if we exited the wait for any other reason
code.pop(Xbyak::util::edx);
code.pop(Xbyak::util::ebx);
code.pop(Xbyak::util::eax);
code.L(start);
code.mov(tmp, 1);
if (ptr.is64bitDisp()) {
// if tmp is on eax, use ebx, otherwise use eax!
auto const other_tmp = tmp.cvt32() == Xbyak::util::eax
? Xbyak::util::rbx
: Xbyak::util::rax;
code.push(other_tmp);
code.mov(other_tmp, ptr.getDisp());
/*code.lock();*/ code.xchg(code.dword[other_tmp], tmp);
code.pop(other_tmp);
} else {
/*code.lock();*/ code.xchg(ptr, tmp);
}
code.test(tmp, tmp);
code.jnz(loop, code.T_NEAR);
code.pop(Xbyak::util::rdx);
code.pop(Xbyak::util::rbx);
code.pop(Xbyak::util::rax);
} else {
Xbyak::Label start, loop;
code.jmp(start, code.T_NEAR);
code.L(loop);
code.pause();
code.L(start);
code.mov(tmp, 1);
if (ptr.is64bitDisp()) {
// if tmp is on eax, use ebx, otherwise use eax!
auto const other_tmp = tmp.cvt32() == Xbyak::util::eax
? Xbyak::util::rbx
: Xbyak::util::rax;
code.push(other_tmp);
code.mov(other_tmp, ptr.getDisp());
/*code.lock();*/ code.xchg(code.dword[other_tmp], tmp);
code.pop(other_tmp);
} else {
/*code.lock();*/ code.xchg(ptr, tmp);
}
code.test(tmp, tmp);
code.jnz(loop, code.T_NEAR);
}
code.L(start);
code.mov(tmp, 1);
/*code.lock();*/ code.xchg(code.dword[ptr], tmp);
code.test(tmp, tmp);
code.jnz(loop, code.T_NEAR);
}
// ptr operand must be a dword[ptr]
void EmitSpinLockUnlock(Xbyak::CodeGenerator& code, Xbyak::Address ptr, Xbyak::Reg32 tmp) {
void EmitSpinLockUnlock(Xbyak::CodeGenerator& code, Xbyak::Reg64 ptr, Xbyak::Reg32 tmp) {
code.xor_(tmp, tmp);
if (ptr.is64bitDisp()) {
// if tmp is on eax, use ebx, otherwise use eax!
auto const other_tmp = tmp.cvt32() == Xbyak::util::eax
? Xbyak::util::rbx
: Xbyak::util::rax;
code.push(other_tmp);
code.mov(other_tmp, ptr.getDisp());
/*code.lock();*/ code.xchg(code.dword[other_tmp], tmp);
code.pop(other_tmp);
} else {
/*code.lock();*/ code.xchg(ptr, tmp);
}
code.xchg(code.dword[ptr], tmp);
code.mfence();
}
@@ -143,12 +93,11 @@ void SpinLockImpl::Initialize() noexcept {
Xbyak::Reg64 const ABI_PARAM1 = Backend::X64::HostLocToReg64(Backend::X64::ABI_PARAM1);
code.align();
lock = code.getCurr<void (*)(volatile int*)>();
EmitSpinLockLock(code, code.dword[ABI_PARAM1], code.eax, false);
EmitSpinLockLock(code, ABI_PARAM1, code.eax, false);
code.ret();
code.align();
unlock = code.getCurr<void (*)(volatile int*)>();
EmitSpinLockUnlock(code, code.dword[ABI_PARAM1], code.eax);
EmitSpinLockUnlock(code, ABI_PARAM1, code.eax);
code.ret();
}
@@ -12,7 +12,7 @@
namespace Dynarmic {
void EmitSpinLockLock(Xbyak::CodeGenerator& code, Xbyak::Address ptr, Xbyak::Reg32 tmp, bool waitpkg);
void EmitSpinLockUnlock(Xbyak::CodeGenerator& code, Xbyak::Address ptr, Xbyak::Reg32 tmp);
void EmitSpinLockLock(Xbyak::CodeGenerator& code, Xbyak::Reg64 ptr, Xbyak::Reg32 tmp, bool waitpkg);
void EmitSpinLockUnlock(Xbyak::CodeGenerator& code, Xbyak::Reg64 ptr, Xbyak::Reg32 tmp);
} // namespace Dynarmic
@@ -1,6 +1,3 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
/* This file is part of the dynarmic project.
* Copyright (c) 2018 MerryMage
* SPDX-License-Identifier: 0BSD
@@ -23,71 +20,68 @@ using Vector = std::array<std::uint64_t, 2>;
class ExclusiveMonitor {
public:
explicit ExclusiveMonitor() noexcept {
std::fill(exclusive_addresses.begin(), exclusive_addresses.end(), INVALID_EXCLUSIVE_ADDRESS);
}
/// @param processor_count Maximum number of processors using this global
/// exclusive monitor. Each processor must have a
/// unique id.
explicit ExclusiveMonitor(size_t processor_count);
size_t GetProcessorCount() const;
/// Marks a region containing [address, address+size) to be exclusive to
/// processor index.
template<typename T, typename F>
[[nodiscard]] inline T ReadAndMark(std::size_t index, VAddr address, F f) {
/// processor processor_id.
template<typename T, typename Function>
T ReadAndMark(size_t processor_id, VAddr address, Function op) {
static_assert(std::is_trivially_copyable_v<T>);
const VAddr masked_address = address & RESERVATION_GRANULE_MASK;
lock.Lock();
exclusive_addresses[index] = masked_address;
T const value = f();
std::memcpy(exclusive_values[index].data(), std::addressof(value), sizeof(T));
lock.Unlock();
Lock();
exclusive_addresses[processor_id] = masked_address;
const T value = op();
std::memcpy(exclusive_values[processor_id].data(), &value, sizeof(T));
Unlock();
return value;
}
[[nodiscard]] inline bool CheckAndClear(std::size_t index, VAddr address) {
const VAddr masked_address = address & RESERVATION_GRANULE_MASK;
if (exclusive_addresses[index] != masked_address)
return false;
for (VAddr& other_address : exclusive_addresses)
if (other_address == masked_address)
other_address = INVALID_EXCLUSIVE_ADDRESS;
return true;
}
/// Checks to see if processor index has exclusive access to the
/// Checks to see if processor processor_id has exclusive access to the
/// specified region. If it does, executes the operation then clears
/// the exclusive state for processors if their exclusive region(s)
/// contain [address, address+size).
template<typename T, typename F>
[[nodiscard]] inline bool DoExclusiveOperation(std::size_t index, VAddr address, F&& f) {
template<typename T, typename Function>
bool DoExclusiveOperation(size_t processor_id, VAddr address, Function op) {
static_assert(std::is_trivially_copyable_v<T>);
bool result = false;
lock.Lock();
if (CheckAndClear(index, address)) {
T saved_value{};
std::memcpy(std::addressof(saved_value), exclusive_values[index].data(), sizeof(T));
result = f(saved_value);
if (!CheckAndClear(processor_id, address)) {
return false;
}
lock.Unlock();
T saved_value;
std::memcpy(&saved_value, exclusive_values[processor_id].data(), sizeof(T));
const bool result = op(saved_value);
Unlock();
return result;
}
/// Unmark everything.
inline void Clear() {
lock.Lock();
std::fill(exclusive_addresses.begin(), exclusive_addresses.end(), INVALID_EXCLUSIVE_ADDRESS);
lock.Unlock();
}
void Clear();
/// Unmark processor id
inline void ClearProcessor(size_t index) {
lock.Lock();
exclusive_addresses[index] = INVALID_EXCLUSIVE_ADDRESS;
lock.Unlock();
}
void ClearProcessor(size_t processor_id);
private:
bool CheckAndClear(size_t processor_id, VAddr address);
void Lock();
void Unlock();
friend volatile int* GetExclusiveMonitorLockPointer(ExclusiveMonitor*);
friend size_t GetExclusiveMonitorProcessorCount(ExclusiveMonitor*);
friend VAddr* GetExclusiveMonitorAddressPointer(ExclusiveMonitor*, size_t index);
friend Vector* GetExclusiveMonitorValuePointer(ExclusiveMonitor*, size_t index);
static constexpr VAddr RESERVATION_GRANULE_MASK = 0xFFFF'FFFF'FFFF'FFFFull;
static constexpr VAddr INVALID_EXCLUSIVE_ADDRESS = 0xDEAD'DEAD'DEAD'DEADull;
static constexpr size_t MAX_NUM_CPU_CORES = 4; // Sync with src/core/hardware_properties
std::array<VAddr, MAX_NUM_CPU_CORES> exclusive_addresses;
std::array<Vector, MAX_NUM_CPU_CORES> exclusive_values;
boost::container::static_vector<VAddr, MAX_NUM_CPU_CORES> exclusive_addresses;
boost::container::static_vector<Vector, MAX_NUM_CPU_CORES> exclusive_values;
SpinLock lock;
};