diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 57ba8fa769..1ba1887d4a 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1085,11 +1085,25 @@ void BufferCache

::BindHostGraphicsStorageBuffers(size_t stage) { Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); const u32 size = binding.size; + const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; + + if constexpr (USE_UNIFIED_MEMORY) { + const auto window = TryResolveUnifiedRange(binding.device_addr, size); + if (window && runtime.IsUnifiedStorageRange(size, window->offset)) { + if (is_written) { + memory_tracker.MarkRegionAsCpuModified(binding.device_addr, size); + } + runtime.BindStorageBuffer(runtime.UnifiedWindowBuffer(window->window), + runtime.UnifiedWindowAddress(window->window), + static_cast(window->offset), size, is_written); + return; + } + } + SynchronizeBuffer(buffer, binding.device_addr, size); const u32 offset = buffer.Offset(binding.device_addr); buffer.MarkUsage(offset, size); - const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; if (is_written) { MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); @@ -1842,6 +1856,57 @@ bool BufferCache

::ResolveUnifiedWindows( } } +template +std::optional::UnifiedWindowRange> +BufferCache

::TryResolveUnifiedRange([[maybe_unused]] DAddr device_addr, + [[maybe_unused]] u64 size) { + if constexpr (USE_UNIFIED_MEMORY) { + if (size == 0 || !runtime.IsUnifiedMemoryBindable()) { + return std::nullopt; + } + const u64 window_size = runtime.UnifiedMemoryWindowSize(); + if (window_size == 0) { + return std::nullopt; + } + const u8* const first = device_memory.GetPointer(device_addr); + if (first == nullptr) { + return std::nullopt; + } + const u64 phys_offset = static_cast(first - device_memory.GetPhysicalBase()); + const u64 unified_base = runtime.UnifiedMemoryBase(); + if (phys_offset < unified_base) { + return std::nullopt; + } + const u64 relative = phys_offset - unified_base; + const u64 unified_size = runtime.UnifiedMemorySize(); + if (relative >= unified_size || unified_size - relative < size) { + return std::nullopt; + } + const u64 local_offset = relative % window_size; + if (window_size - local_offset < size) { + return std::nullopt; + } + if (memory_tracker.IsRegionGpuModified(device_addr, size) || + IsRegionGpuModified(device_addr, size)) { + return std::nullopt; + } + u64 walked = Core::DEVICE_PAGESIZE - (device_addr & Core::DEVICE_PAGEMASK); + while (walked < size) { + const u8* const next = device_memory.GetPointer(device_addr + walked); + if (next != first + walked) { + return std::nullopt; + } + walked += Core::DEVICE_PAGESIZE; + } + return UnifiedWindowRange{ + .window = static_cast(relative / window_size), + .offset = local_offset, + }; + } else { + return std::nullopt; + } +} + template bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, [[maybe_unused]] std::span copies) { diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 9d66fe9b40..b84cd2709c 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -446,6 +447,13 @@ private: bool TryUnifiedDownloadMemory(Buffer& buffer, std::span copies); + struct UnifiedWindowRange { + size_t window; + u64 offset; + }; + + std::optional TryResolveUnifiedRange(DAddr device_addr, u64 size); + using UnifiedWindowGroups = boost::container::small_vector, 4>; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 490cb59fab..73649bfeb8 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -195,6 +195,29 @@ public: BindBuffer(buffer, offset, size); } + void BindStorageBuffer(VkBuffer buffer, VkDeviceAddress address, u32 offset, u32 size, + [[maybe_unused]] bool is_written) { + guest_descriptor_queue.AddBuffer(buffer, address, offset, size); + } + + [[nodiscard]] bool IsUnifiedMemoryBindable() const noexcept { + return unified_memory != nullptr && unified_memory->IsValid() && + unified_memory->IsBindable(); + } + + [[nodiscard]] VkBuffer UnifiedWindowBuffer(size_t index) const noexcept { + return unified_memory->GetWindowBuffer(index); + } + + [[nodiscard]] VkDeviceAddress UnifiedWindowAddress(size_t index) const noexcept { + return unified_memory->GetWindowAddress(index); + } + + [[nodiscard]] bool IsUnifiedStorageRange(u32 size, u64 offset) const { + return size <= device.GetMaxStorageBufferRange() && + (offset % device.GetStorageBufferAlignment()) == 0; + } + void BindTextureBuffer(Buffer& buffer, u32 offset, u32 size, VideoCore::Surface::PixelFormat format) { guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format), diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index f3a33abed4..47a9849cef 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -341,7 +341,11 @@ public: return properties.properties.limits.minUniformBufferOffsetAlignment; } - /// Returns storage alignment requirement. + /// Returns max storage buffer range requirement. + u32 GetMaxStorageBufferRange() const { + return properties.properties.limits.maxStorageBufferRange; + } + VkDeviceSize GetStorageBufferAlignment() const { return properties.properties.limits.minStorageBufferOffsetAlignment; } diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 4f205938c9..3589f0afc6 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -379,88 +379,125 @@ namespace Vulkan { const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties; window_size = hardware_buffer_window; base_offset = hardware_buffer_base; - for (size_t i = 0; i < hardware_buffers.size(); ++i) { - const size_t offset = hardware_buffer_base + i * hardware_buffer_window; - if (offset >= size) { - break; + + const auto import_all = [&](VkBufferUsageFlags usage, bool want_address) { + for (size_t i = 0; i < hardware_buffers.size(); ++i) { + const size_t offset = hardware_buffer_base + i * hardware_buffer_window; + if (offset >= size) { + break; + } + const VkDeviceSize window_len = (std::min)( + static_cast(size - offset), + static_cast(hardware_buffer_window)); + VkAndroidHardwareBufferPropertiesANDROID ahb_props{ + .sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID, + .pNext = nullptr, + .allocationSize = 0, + .memoryTypeBits = 0, + }; + if (logical.GetAndroidHardwareBufferPropertiesANDROID(hardware_buffers[i], + &ahb_props) != VK_SUCCESS || + ahb_props.memoryTypeBits == 0 || ahb_props.allocationSize < window_len) { + break; + } + const VkExternalMemoryBufferCreateInfo external_info{ + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, + .pNext = nullptr, + .handleTypes = + VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID, + }; + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = &external_info, + .flags = 0, + .size = window_len, + .usage = usage, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + VkBuffer new_buffer{}; + if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) { + break; + } + const VkMemoryRequirements requirements = + logical.GetBufferMemoryRequirements(new_buffer); + const u32 type_mask = requirements.memoryTypeBits & ahb_props.memoryTypeBits; + if (type_mask == 0 || requirements.size > ahb_props.allocationSize) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const auto type_index = FindImportMemoryType(memory_props, type_mask); + if (!type_index) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const VkImportAndroidHardwareBufferInfoANDROID import_info{ + .sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID, + .pNext = nullptr, + .buffer = hardware_buffers[i], + }; + const VkMemoryDedicatedAllocateInfo dedicated_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO, + .pNext = &import_info, + .image = VK_NULL_HANDLE, + .buffer = new_buffer, + }; + const VkMemoryAllocateFlagsInfo flags_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, + .pNext = &dedicated_info, + .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, + .deviceMask = 0, + }; + const void *alloc_next = &dedicated_info; + if (want_address) { + alloc_next = &flags_info; + } + const VkMemoryAllocateInfo alloc_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .pNext = alloc_next, + .allocationSize = ahb_props.allocationSize, + .memoryTypeIndex = *type_index, + }; + vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info); + if (!memory) { + logical.DestroyBufferRaw(new_buffer); + break; + } + if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) { + logical.DestroyBufferRaw(new_buffer); + break; + } + VkDeviceAddress address = 0; + if (want_address) { + address = logical.GetBufferDeviceAddress(new_buffer); + } + windows.push_back(Window{ + .memory = std::move(memory), + .buffer = new_buffer, + .address = address, + }); + imported_size += static_cast(window_len); } - const VkDeviceSize window_len = (std::min)( - static_cast(size - offset), - static_cast(hardware_buffer_window)); - VkAndroidHardwareBufferPropertiesANDROID ahb_props{ - .sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID, - .pNext = nullptr, - .allocationSize = 0, - .memoryTypeBits = 0, - }; - if (logical.GetAndroidHardwareBufferPropertiesANDROID(hardware_buffers[i], - &ahb_props) != VK_SUCCESS || - ahb_props.memoryTypeBits == 0 || ahb_props.allocationSize < window_len) { - break; - } - const VkExternalMemoryBufferCreateInfo external_info{ - .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, - .pNext = nullptr, - .handleTypes = - VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID, - }; - const VkBufferCreateInfo buffer_ci{ - .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, - .pNext = &external_info, - .flags = 0, - .size = window_len, - .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, - .sharingMode = VK_SHARING_MODE_EXCLUSIVE, - .queueFamilyIndexCount = 0, - .pQueueFamilyIndices = nullptr, - }; - VkBuffer new_buffer{}; - if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) { - break; - } - const VkMemoryRequirements requirements = - logical.GetBufferMemoryRequirements(new_buffer); - const u32 type_mask = requirements.memoryTypeBits & ahb_props.memoryTypeBits; - if (type_mask == 0 || requirements.size > ahb_props.allocationSize) { - logical.DestroyBufferRaw(new_buffer); - break; - } - const auto type_index = FindImportMemoryType(memory_props, type_mask); - if (!type_index) { - logical.DestroyBufferRaw(new_buffer); - break; - } - const VkImportAndroidHardwareBufferInfoANDROID import_info{ - .sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID, - .pNext = nullptr, - .buffer = hardware_buffers[i], - }; - const VkMemoryDedicatedAllocateInfo dedicated_info{ - .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO, - .pNext = &import_info, - .image = VK_NULL_HANDLE, - .buffer = new_buffer, - }; - const VkMemoryAllocateInfo alloc_info{ - .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, - .pNext = &dedicated_info, - .allocationSize = ahb_props.allocationSize, - .memoryTypeIndex = *type_index, - }; - vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info); - if (!memory) { - logical.DestroyBufferRaw(new_buffer); - break; - } - if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) { - logical.DestroyBufferRaw(new_buffer); - break; - } - windows.push_back(Window{ - .memory = std::move(memory), - .buffer = new_buffer, - }); - imported_size += static_cast(window_len); + return !windows.empty(); + }; + + constexpr VkBufferUsageFlags TransferUsage = + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + VkBufferUsageFlags shader_usage = TransferUsage | + VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | + VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | + VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; + const bool want_address = device.IsBufferDeviceAddressSupported(); + if (want_address) { + shader_usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + } + + bindable = import_all(shader_usage, want_address); + if (!bindable) { + imported_size = 0; + import_all(TransferUsage, false); } if (windows.empty()) { window_size = 0; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index 9ece2bbfe4..8cff6b844d 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -122,14 +122,23 @@ namespace Vulkan { return windows[index].buffer; } + [[nodiscard]] VkDeviceAddress GetWindowAddress(size_t index) const noexcept { + return windows[index].address; + } + [[nodiscard]] size_t GetWindowCount() const noexcept { return windows.size(); } + [[nodiscard]] bool IsBindable() const noexcept { + return bindable; + } + private: struct Window { vk::DeviceMemory memory; VkBuffer buffer{}; + VkDeviceAddress address{}; }; bool ImportHostPointer(void *base, size_t size); @@ -144,6 +153,7 @@ namespace Vulkan { size_t imported_size{}; size_t base_offset{}; bool foreign_ownership{}; + bool bindable{}; }; /// Memory allocator container.