Initial implementation on unified memory access

src/common/host_memory.h

@@ -62,6 +62,10 @@ public:
         return backing_base;
     }
 
+    [[nodiscard]] size_t BackingSize() const noexcept {
+        return backing_size;
+    }
+
     [[nodiscard]] u8* VirtualBasePointer() noexcept {
         return virtual_base;
     }

src/common/settings.h

@@ -587,6 +587,9 @@ struct Values {
     SwitchableSetting<bool> use_asynchronous_shaders{linkage, false, "use_asynchronous_shaders",
                                                      Category::RendererHacks};
 
+    SwitchableSetting<bool> use_unified_memory{linkage, false, "use_unified_memory",
+                                               Category::RendererHacks};
+
     SwitchableSetting<GpuUnswizzleSize> gpu_unswizzle_texture_size{linkage,
                                                   GpuUnswizzleSize::Large,
                                                   "gpu_unswizzle_texture_size",

src/core/device_memory_manager.h

@@ -95,6 +95,18 @@ public:
         ApplyOpOnPAddr(address, buffer, operation);
     }
 
+    u8* GetPhysicalBase() noexcept {
+        return reinterpret_cast<u8*>(physical_base);
+    }
+
+    const u8* GetPhysicalBase() const noexcept {
+        return reinterpret_cast<const u8*>(physical_base);
+    }
+
+    size_t GetPhysicalSize() const noexcept {
+        return physical_size;
+    }
+
     PAddr GetPhysicalRawAddressFromDAddr(DAddr address) const {
         PAddr subbits = PAddr(address & page_mask);
         auto paddr = tracked_entries[(address >> page_bits)].compressed_physical_ptr;
@@ -175,6 +187,7 @@ private:
     std::unique_ptr<DeviceMemoryManagerAllocator<Traits>> impl;
 
     const uintptr_t physical_base;
+    const size_t physical_size;
     DeviceInterface* device_inter;
 
     struct TrackedEntry {

src/core/device_memory_manager.inc

@@ -171,6 +171,7 @@ struct DeviceMemoryManagerAllocator {
 template <typename Traits>
 DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memory_)
     : physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())}
+    , physical_size{device_memory_.buffer.BackingSize()}
     , device_inter{nullptr}
     , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS))
     , tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS)

src/qt_common/config/shared_translation.cpp

@@ -230,6 +230,10 @@ std::unique_ptr<TranslationMap> InitializeTranslations(QObject* parent) {
            tr("Preserves GPU-modified data by reading it back before uploading.\nSome games require this to render certain effects properly."));
     INSERT(Settings, use_asynchronous_shaders, tr("Enable asynchronous shader compilation"),
            tr("May reduce shader stutter."));
+    INSERT(Settings, use_unified_memory, tr("Enable unified memory access (UMA)"),
+           tr("Lets the GPU read guest memory directly for buffer uploads, skipping the CPU "
+              "staging copy.\nRequires driver support for host memory import and may cause "
+              "issues in some games."));
     INSERT(Settings, pipeline_worker_count, tr("Pipeline Worker Threads"),
            tr("Number of threads used to build Vulkan pipelines.\n"
               "Higher values speed up compilation at the cost of heat and power."));

src/video_core/buffer_cache/buffer_cache.h

@@ -1814,11 +1814,70 @@ void BufferCache<P>::ImmediateUploadMemory([[maybe_unused]] Buffer& buffer,
     }
 }
 
+template <class P>
+bool BufferCache<P>::TryUnifiedUploadMemory([[maybe_unused]] Buffer& buffer,
+                                            [[maybe_unused]] std::span<BufferCopy> copies) {
+    if constexpr (USE_UNIFIED_MEMORY) {
+        boost::container::small_vector<BufferCopy, 16> host_copies;
+        const u8* const physical_base = device_memory.GetPhysicalBase();
+        const u64 unified_size = runtime.UnifiedMemorySize();
+        for (const BufferCopy& copy : copies) {
+            const DAddr device_addr = buffer.CpuAddr() + copy.dst_offset;
+            u64 uploaded = 0;
+            while (uploaded < copy.size) {
+                const DAddr page_addr = device_addr + uploaded;
+                const u8* const ptr = device_memory.GetPointer<u8>(page_addr);
+                if (ptr == nullptr) {
+                    return false;
+                }
+                const u64 page_offset = page_addr & Core::DEVICE_PAGEMASK;
+                const u64 chunk = (std::min)(copy.size - uploaded,
+                                             static_cast<u64>(Core::DEVICE_PAGESIZE) - page_offset);
+                const u64 src_offset = static_cast<u64>(ptr - physical_base);
+                if (src_offset + chunk > unified_size) {
+                    return false;
+                }
+                if (!host_copies.empty()) {
+                    BufferCopy& last = host_copies.back();
+                    if (last.src_offset + last.size == src_offset &&
+                        last.dst_offset + last.size == copy.dst_offset + uploaded) {
+                        last.size += chunk;
+                        uploaded += chunk;
+                        continue;
+                    }
+                }
+                host_copies.push_back(BufferCopy{
+                    .src_offset = src_offset,
+                    .dst_offset = copy.dst_offset + uploaded,
+                    .size = chunk,
+                });
+                uploaded += chunk;
+            }
+        }
+        for (const BufferCopy& copy : copies) {
+            if (Settings::values.enable_gpu_buffer_readback.GetValue()) {
+                DownloadBufferMemory(buffer, buffer.CpuAddr() + copy.dst_offset, copy.size);
+            }
+        }
+        const std::span<BufferCopy> host_span(host_copies.data(), host_copies.size());
+        const bool can_reorder = runtime.CanReorderUpload(buffer, host_span);
+        runtime.CopyBuffer(buffer, runtime.UnifiedMemoryBuffer(), host_span, true, can_reorder);
+        return true;
+    } else {
+        return false;
+    }
+}
+
 template <class P>
 void BufferCache<P>::MappedUploadMemory([[maybe_unused]] Buffer& buffer,
                                         [[maybe_unused]] u64 total_size_bytes,
                                         [[maybe_unused]] std::span<BufferCopy> copies) {
     if constexpr (USE_MEMORY_MAPS) {
+        if constexpr (USE_UNIFIED_MEMORY) {
+            if (runtime.HasUnifiedMemory() && TryUnifiedUploadMemory(buffer, copies)) {
+                return;
+            }
+        }
         auto upload_staging = runtime.UploadStagingBuffer(total_size_bytes);
         const std::span<u8> staging_pointer = upload_staging.mapped_span;
         for (BufferCopy& copy : copies) {

src/video_core/buffer_cache/buffer_cache_base.h

@@ -181,6 +181,7 @@ class BufferCache : public VideoCommon::ChannelSetupCaches<BufferCacheChannelInf
     static constexpr bool USE_MEMORY_MAPS = P::USE_MEMORY_MAPS;
     static constexpr bool SEPARATE_IMAGE_BUFFERS_BINDINGS = P::SEPARATE_IMAGE_BUFFER_BINDINGS;
     static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = P::USE_MEMORY_MAPS_FOR_UPLOADS;
+    static constexpr bool USE_UNIFIED_MEMORY = P::USE_UNIFIED_MEMORY;
 
 #ifdef YUZU_LEGACY
     static constexpr u64 RECLAIM_HEADROOM = 384_MiB;
@@ -450,6 +451,8 @@ private:
 
     void MappedUploadMemory(Buffer& buffer, u64 total_size_bytes, std::span<BufferCopy> copies);
 
+    bool TryUnifiedUploadMemory(Buffer& buffer, std::span<BufferCopy> copies);
+
     void DownloadBufferMemory(Buffer& buffer_id);
 
     void DownloadBufferMemory(Buffer& buffer_id, DAddr device_addr, u64 size);

src/video_core/renderer_opengl/gl_buffer_cache.h

@@ -279,6 +279,7 @@ struct BufferCacheParams {
 
     // TODO: Investigate why OpenGL seems to perform worse with persistently mapped buffer uploads
     static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = false;
+    static constexpr bool USE_UNIFIED_MEMORY = false;
 };
 
 using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;

src/video_core/renderer_vulkan/vk_buffer_cache.cpp

@@ -364,6 +364,13 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m
                                                                      scheduler_, staging_pool_);
 }
 
+void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size) {
+    unified_memory = std::make_unique<HostMemoryImport>(device, base, size);
+    if (!unified_memory->IsValid()) {
+        unified_memory.reset();
+    }
+}
+
 StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) {
     return staging_pool.Request(size, MemoryUsage::Upload);
 }

src/video_core/renderer_vulkan/vk_buffer_cache.h

@@ -7,6 +7,7 @@
 #pragma once
 
 #include <limits>
+#include <memory>
 
 #include "video_core/buffer_cache/buffer_cache_base.h"
 #include "video_core/buffer_cache/memory_tracker_base.h"
@@ -97,6 +98,20 @@ public:
 
     void TickFrame(Common::SlotVector<Buffer>& slot_buffers) noexcept;
 
+    void TryEnableUnifiedMemory(void* base, size_t size);
+
+    [[nodiscard]] bool HasUnifiedMemory() const noexcept {
+        return unified_memory != nullptr && unified_memory->IsValid();
+    }
+
+    [[nodiscard]] VkBuffer UnifiedMemoryBuffer() const noexcept {
+        return unified_memory ? unified_memory->GetBuffer() : VK_NULL_HANDLE;
+    }
+
+    [[nodiscard]] u64 UnifiedMemorySize() const noexcept {
+        return unified_memory ? unified_memory->GetSize() : 0;
+    }
+
     u64 CurrentTick();
 
     u64 KnownGpuTick();
@@ -210,6 +225,7 @@ private:
     std::shared_ptr<QuadStripIndexBuffer> quad_strip_index_buffer;
 
     vk::Buffer null_buffer;
+    std::unique_ptr<HostMemoryImport> unified_memory;
 
     std::unique_ptr<Uint8Pass> uint8_pass;
     QuadIndexedPass quad_index_pass;
@@ -232,6 +248,7 @@ struct BufferCacheParams {
     static constexpr bool USE_MEMORY_MAPS = true;
     static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = false;
     static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = true;
+    static constexpr bool USE_UNIFIED_MEMORY = true;
 };
 
 using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;

src/video_core/renderer_vulkan/vk_rasterizer.cpp

@@ -239,6 +239,10 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra
       fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler),
       wfi_event(device.GetLogical().CreateEvent()) {
     scheduler.SetQueryCache(query_cache);
+    if (Settings::values.use_unified_memory.GetValue()) {
+        buffer_cache_runtime.TryEnableUnifiedMemory(device_memory.GetPhysicalBase(),
+                                                    device_memory.GetPhysicalSize());
+    }
     memory_allocator.SetReclaimCallback([this](u64 bytes) -> u64 {
         u64 freed = staging_pool.ReclaimMemory(bytes);
         if (freed < bytes) {

src/video_core/vulkan_common/vulkan_device.cpp

@@ -1239,6 +1239,16 @@ bool Device::GetSuitability(bool requires_swapchain) {
             VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_5_PROPERTIES_KHR;
         SetNext(next, properties.maintenance5);
     }
+    if (extensions.external_memory_host) {
+        properties.external_memory_host.sType =
+            VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT;
+        SetNext(next, properties.external_memory_host);
+    }
+    if (extensions.maintenance4 || features.maintenance4.maintenance4) {
+        properties.maintenance4.sType =
+            VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_4_PROPERTIES;
+        SetNext(next, properties.maintenance4);
+    }
     if (instance_version >= VK_API_VERSION_1_2) {
         properties.depth_stencil_resolve.sType =
             VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_STENCIL_RESOLVE_PROPERTIES;

src/video_core/vulkan_common/vulkan_device.h

@@ -87,6 +87,7 @@ VK_DEFINE_HANDLE(VmaAllocator)
     EXTENSION(EXT, CONDITIONAL_RENDERING, conditional_rendering)                                   \
     EXTENSION(EXT, CONSERVATIVE_RASTERIZATION, conservative_rasterization)                         \
     EXTENSION(EXT, DEPTH_RANGE_UNRESTRICTED, depth_range_unrestricted)                             \
+    EXTENSION(EXT, EXTERNAL_MEMORY_HOST, external_memory_host)                                     \
     EXTENSION(EXT, MEMORY_BUDGET, memory_budget)                                                   \
     EXTENSION(EXT, PIPELINE_CREATION_FEEDBACK, pipeline_creation_feedback)                         \
     EXTENSION(EXT, ROBUSTNESS_2, robustness_2)                                                     \
@@ -900,6 +901,18 @@ FN_MAX_LIMIT_LIST
         return extensions.conditional_rendering;
     }
 
+    bool IsExtExternalMemoryHostSupported() const {
+        return extensions.external_memory_host;
+    }
+
+    u64 GetMinImportedHostPointerAlignment() const {
+        return properties.external_memory_host.minImportedHostPointerAlignment;
+    }
+
+    u64 GetMaxBufferSize() const {
+        return properties.maintenance4.maxBufferSize;
+    }
+
     bool HasTimelineSemaphore() const;
 
     /// Returns true if the device supports VK_KHR_synchronization2.
@@ -1247,8 +1260,10 @@ private:
         VkPhysicalDeviceDescriptorBufferPropertiesEXT descriptor_buffer{};
         VkPhysicalDeviceSubgroupSizeControlProperties subgroup_size_control{};
         VkPhysicalDeviceTransformFeedbackPropertiesEXT transform_feedback{};
+        VkPhysicalDeviceMaintenance4Properties maintenance4{};
         VkPhysicalDeviceMaintenance5PropertiesKHR maintenance5{};
         VkPhysicalDeviceDepthStencilResolveProperties depth_stencil_resolve{};
+        VkPhysicalDeviceExternalMemoryHostPropertiesEXT external_memory_host{};
 
         VkPhysicalDeviceProperties properties{};
     };

src/video_core/vulkan_common/vulkan_memory_allocator.cpp

@@ -68,6 +68,118 @@ namespace Vulkan {
 
     } // namespace
 
+    HostMemoryImport::HostMemoryImport(const Device &device_, void *base, size_t size)
+            : device{device_} {
+        if (!device.IsExtExternalMemoryHostSupported()) {
+            return;
+        }
+        const u64 alignment = device.GetMinImportedHostPointerAlignment();
+        if (alignment == 0 || !Common::IsAligned(reinterpret_cast<uintptr_t>(base), alignment) ||
+            !Common::IsAligned(size, alignment)) {
+            LOG_INFO(Render_Vulkan,
+                     "Unified memory disabled, host allocation does not satisfy alignment {}",
+                     alignment);
+            return;
+        }
+        const u64 max_buffer_size = device.GetMaxBufferSize();
+        if (max_buffer_size != 0 && max_buffer_size < size) {
+            size = static_cast<size_t>(Common::AlignDown(max_buffer_size, alignment));
+            if (size == 0) {
+                return;
+            }
+        }
+        const auto &logical = device.GetLogical();
+        VkMemoryHostPointerPropertiesEXT host_props{
+                .sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT,
+                .pNext = nullptr,
+                .memoryTypeBits = 0,
+        };
+        if (logical.GetMemoryHostPointerPropertiesEXT(
+                    VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, base, &host_props) !=
+                    VK_SUCCESS ||
+            host_props.memoryTypeBits == 0) {
+            return;
+        }
+        const VkExternalMemoryBufferCreateInfo external_info{
+                .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
+                .pNext = nullptr,
+                .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
+        };
+        const VkBufferCreateInfo buffer_ci{
+                .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
+                .pNext = &external_info,
+                .flags = 0,
+                .size = size,
+                .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT,
+                .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
+                .queueFamilyIndexCount = 0,
+                .pQueueFamilyIndices = nullptr,
+        };
+        VkBuffer new_buffer{};
+        if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) {
+            return;
+        }
+        const VkMemoryRequirements requirements = logical.GetBufferMemoryRequirements(new_buffer);
+        const u32 type_mask = requirements.memoryTypeBits & host_props.memoryTypeBits;
+        if (type_mask == 0 || requirements.size > size) {
+            logical.DestroyBufferRaw(new_buffer);
+            return;
+        }
+        const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties;
+        const auto find_type = [&](VkMemoryPropertyFlags wanted) -> std::optional<u32> {
+            for (u32 i = 0; i < memory_props.memoryTypeCount; ++i) {
+                if (((type_mask >> i) & 1u) != 0 &&
+                    (memory_props.memoryTypes[i].propertyFlags & wanted) == wanted) {
+                    return i;
+                }
+            }
+            return std::nullopt;
+        };
+        auto type_index = find_type(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
+                                    VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
+                                    VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
+        if (!type_index) {
+            type_index = find_type(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
+                                   VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
+        }
+        if (!type_index) {
+            logical.DestroyBufferRaw(new_buffer);
+            return;
+        }
+        const VkImportMemoryHostPointerInfoEXT import_info{
+                .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT,
+                .pNext = nullptr,
+                .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
+                .pHostPointer = base,
+        };
+        const VkMemoryAllocateInfo alloc_info{
+                .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
+                .pNext = &import_info,
+                .allocationSize = size,
+                .memoryTypeIndex = *type_index,
+        };
+        memory = logical.TryAllocateMemory(alloc_info);
+        if (!memory) {
+            logical.DestroyBufferRaw(new_buffer);
+            return;
+        }
+        if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) {
+            logical.DestroyBufferRaw(new_buffer);
+            memory = vk::DeviceMemory{};
+            return;
+        }
+        buffer = new_buffer;
+        imported_size = size;
+        LOG_INFO(Render_Vulkan, "Imported {} MiB of guest memory for unified memory access",
+                 size >> 20);
+    }
+
+    HostMemoryImport::~HostMemoryImport() {
+        if (buffer != VK_NULL_HANDLE) {
+            device.GetLogical().DestroyBufferRaw(buffer);
+        }
+    }
+
     MemoryAllocator::MemoryAllocator(const Device &device_)
             : device{device_}, allocator{device.GetAllocator()},
               properties{device_.GetPhysical().GetMemoryProperties().memoryProperties} {

src/video_core/vulkan_common/vulkan_memory_allocator.h

@@ -40,6 +40,35 @@ namespace Vulkan {
         }
     }
 
+    class HostMemoryImport {
+    public:
+        explicit HostMemoryImport(const Device &device_, void *base, size_t size);
+
+        ~HostMemoryImport();
+
+        HostMemoryImport(const HostMemoryImport &) = delete;
+
+        HostMemoryImport &operator=(const HostMemoryImport &) = delete;
+
+        [[nodiscard]] bool IsValid() const noexcept {
+            return buffer != VK_NULL_HANDLE;
+        }
+
+        [[nodiscard]] VkBuffer GetBuffer() const noexcept {
+            return buffer;
+        }
+
+        [[nodiscard]] size_t GetSize() const noexcept {
+            return imported_size;
+        }
+
+    private:
+        const Device &device;
+        vk::DeviceMemory memory;
+        VkBuffer buffer{};
+        size_t imported_size{};
+    };
+
 /// Memory allocator container.
 /// Allocates and releases memory allocations on demand.
     class MemoryAllocator {

src/video_core/vulkan_common/vulkan_wrapper.cpp

@@ -277,6 +277,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept {
     X(vkGetBufferMemoryRequirements2);
     X(vkGetDeviceQueue);
     X(vkGetEventStatus);
+    X(vkGetMemoryHostPointerPropertiesEXT);
     X(vkGetFenceStatus);
     X(vkGetImageMemoryRequirements);
     X(vkGetPipelineCacheData);

src/video_core/vulkan_common/vulkan_wrapper.h

@@ -344,6 +344,7 @@ struct DeviceDispatch : InstanceDispatch {
     PFN_vkGetBufferMemoryRequirements2 vkGetBufferMemoryRequirements2{};
     PFN_vkGetDeviceQueue vkGetDeviceQueue{};
     PFN_vkGetEventStatus vkGetEventStatus{};
+    PFN_vkGetMemoryHostPointerPropertiesEXT vkGetMemoryHostPointerPropertiesEXT{};
     PFN_vkGetFenceStatus vkGetFenceStatus{};
     PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements{};
     PFN_vkGetPipelineCacheData vkGetPipelineCacheData{};
@@ -1094,6 +1095,26 @@ public:
     VkMemoryRequirements GetBufferMemoryRequirements(VkBuffer buffer,
                                                      void* pnext = nullptr) const noexcept;
 
+    VkResult GetMemoryHostPointerPropertiesEXT(
+        VkExternalMemoryHandleTypeFlagBits handle_type, const void* host_pointer,
+        VkMemoryHostPointerPropertiesEXT* out_properties) const noexcept {
+        return dld->vkGetMemoryHostPointerPropertiesEXT(handle, handle_type, host_pointer,
+                                                        out_properties);
+    }
+
+    VkResult CreateBufferRaw(const VkBufferCreateInfo& ci, VkBuffer* out_buffer) const noexcept {
+        return dld->vkCreateBuffer(handle, &ci, nullptr, out_buffer);
+    }
+
+    void DestroyBufferRaw(VkBuffer buffer) const noexcept {
+        dld->vkDestroyBuffer(handle, buffer, nullptr);
+    }
+
+    VkResult BindBufferMemory(VkBuffer buffer, VkDeviceMemory memory,
+                              VkDeviceSize offset) const noexcept {
+        return dld->vkBindBufferMemory(handle, buffer, memory, offset);
+    }
+
     VkMemoryRequirements GetImageMemoryRequirements(VkImage image) const noexcept;
 
     std::vector<VkPipelineExecutablePropertiesKHR> GetPipelineExecutablePropertiesKHR(