diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 4f979a162a..e1689390c0 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1004,37 +1004,66 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 } template -bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_written) { +void BufferCache

::ResolveMultiRangeStorage(Binding& binding, bool is_written, + std::vector& pool) { + binding.segment_first = 0; + binding.segment_count = 0; if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}); }) { if (binding.gpu_addr == 0 || binding.size == 0) { - return false; + return; } if (is_written && !runtime.PrefersSparseSources()) { - return false; + return; } - const VirtualSegments* segments = + const VirtualSegments* found = virtual_ranges.Query(*gpu_memory, binding.gpu_addr, binding.size); - if (!segments || segments->size() < 2) { - return false; + if (!found || found->size() < 2) { + return; } - const u64 key = (static_cast(gpu_memory->GetID()) << 48) ^ binding.gpu_addr; + const VirtualSegments segments = *found; + const u32 first = static_cast(pool.size()); const bool prefer_sparse = runtime.PrefersSparseSources(); - runtime.ResetMultiRange(); - for (const VirtualSegment& segment : *segments) { + for (const VirtualSegment& segment : segments) { const BufferId buffer_id = FindBuffer(segment.device_addr, segment.size, prefer_sparse); if (!buffer_id) { - return false; + pool.resize(first); + return; } - Buffer& buffer = slot_buffers[buffer_id]; - TouchBuffer(buffer, buffer_id); + pool.push_back(MultiRangeSegment{ + .buffer_id = buffer_id, + .device_addr = segment.device_addr, + .size = segment.size, + }); + } + binding.segment_first = first; + binding.segment_count = static_cast(segments.size()); + } +} + +template +bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_written, + std::span pool) { + if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}); }) { + if (binding.segment_count < 2) { + return false; + } + if (binding.segment_first + binding.segment_count > pool.size()) { + return false; + } + const u64 key = (static_cast(gpu_memory->GetID()) << 48) ^ binding.gpu_addr; + runtime.ResetMultiRange(); + for (u32 index = 0; index < binding.segment_count; ++index) { + const MultiRangeSegment& segment = pool[binding.segment_first + index]; + Buffer& buffer = slot_buffers[segment.buffer_id]; + TouchBuffer(buffer, segment.buffer_id); if (SynchronizeBuffer(buffer, segment.device_addr, segment.size)) { runtime.InvalidateMultiRange(key); } const u32 offset = buffer.Offset(segment.device_addr); buffer.MarkUsage(offset, segment.size); if (is_written) { - MarkWrittenBuffer(buffer_id, segment.device_addr, segment.size); + MarkWrittenBuffer(segment.buffer_id, segment.device_addr, segment.size); } runtime.PushMultiRangeSource(buffer, offset, segment.size); } @@ -1050,7 +1079,7 @@ void BufferCache

::BindHostGraphicsStorageBuffers(size_t stage) { ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { const Binding& binding = channel_state->storage_buffers[stage][index]; const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; - if (BindMultiRangeStorage(binding, is_written)) { + if (BindMultiRangeStorage(binding, is_written, graphics_segments)) { return; } Buffer& buffer = slot_buffers[binding.buffer_id]; @@ -1190,7 +1219,7 @@ void BufferCache

::BindHostComputeStorageBuffers() { const Binding& binding = channel_state->compute_storage_buffers[index]; const bool is_written = ((channel_state->written_compute_storage_buffers >> index) & 1) != 0; - if (BindMultiRangeStorage(binding, is_written)) { + if (BindMultiRangeStorage(binding, is_written, compute_segments)) { return; } Buffer& buffer = slot_buffers[binding.buffer_id]; @@ -1245,6 +1274,7 @@ void BufferCache

::BindHostComputeTextureBuffers() { template void BufferCache

::DoUpdateGraphicsBuffers(bool is_indexed) { + graphics_segments.clear(); BufferOperations([&]() { if (is_indexed) { UpdateIndexBuffer(); @@ -1264,6 +1294,7 @@ void BufferCache

::DoUpdateGraphicsBuffers(bool is_indexed) { template void BufferCache

::DoUpdateComputeBuffers() { + compute_segments.clear(); BufferOperations([&]() { UpdateComputeUniformBuffers(); UpdateComputeStorageBuffers(); @@ -1406,6 +1437,8 @@ void BufferCache

::UpdateStorageBuffers(size_t stage) { Binding& binding = channel_state->storage_buffers[stage][index]; const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size); binding.buffer_id = buffer_id; + const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; + ResolveMultiRangeStorage(binding, is_written, graphics_segments); }); } @@ -1469,6 +1502,9 @@ void BufferCache

::UpdateComputeStorageBuffers() { // Resolve buffer Binding& binding = channel_state->compute_storage_buffers[index]; binding.buffer_id = FindBuffer(binding.device_addr, binding.size); + const bool is_written = + ((channel_state->written_compute_storage_buffers >> index) & 1) != 0; + ResolveMultiRangeStorage(binding, is_written, compute_segments); }); } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index c90ccb1cc7..18a8d19fe9 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -85,6 +85,14 @@ struct Binding { u32 size{}; BufferId buffer_id; GPUVAddr gpu_addr{}; + u32 segment_first{}; + u32 segment_count{}; +}; + +struct MultiRangeSegment { + BufferId buffer_id; + DAddr device_addr{}; + u32 size{}; }; struct TextureBufferBinding : Binding { @@ -217,7 +225,11 @@ public: void TickFrame(); - bool BindMultiRangeStorage(const Binding& binding, bool is_written); + bool BindMultiRangeStorage(const Binding& binding, bool is_written, + std::span pool); + + void ResolveMultiRangeStorage(Binding& binding, bool is_written, + std::vector& pool); void UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size); @@ -523,6 +535,8 @@ private: Common::LeastRecentlyUsedCache lru_cache; u64 frame_tick = 0; VirtualRangeCache virtual_ranges; + std::vector graphics_segments; + std::vector compute_segments; u64 total_used_memory = 0; u64 minimum_memory = 0; u64 critical_memory = 0; diff --git a/src/video_core/buffer_cache/virtual_range_cache.h b/src/video_core/buffer_cache/virtual_range_cache.h index 7c279ed0da..3682acf5bc 100644 --- a/src/video_core/buffer_cache/virtual_range_cache.h +++ b/src/video_core/buffer_cache/virtual_range_cache.h @@ -43,20 +43,31 @@ public: entry.gpu_addr = gpu_addr; entry.size = size; const auto ranges = memory.GetSubmappedRange(gpu_addr, size); + GPUVAddr expected = gpu_addr; + bool contiguous = true; for (const auto& [range_addr, range_size] : ranges) { - const std::optional device_addr = memory.GpuToCpuAddress(range_addr); - if (!device_addr) { + if (range_addr != expected || range_size == 0) { + contiguous = false; break; } - u32 segment_size = (std::numeric_limits::max)(); - if (range_size < static_cast(segment_size)) { - segment_size = static_cast(range_size); + const std::optional device_addr = memory.GpuToCpuAddress(range_addr); + if (!device_addr || *device_addr == 0) { + contiguous = false; + break; + } + if (range_size > static_cast((std::numeric_limits::max)())) { + contiguous = false; + break; } entry.segments.push_back(VirtualSegment{ .gpu_addr = range_addr, .device_addr = *device_addr, - .size = segment_size, + .size = static_cast(range_size), }); + expected += range_size; + } + if (!contiguous || expected != gpu_addr + size) { + entry.segments.clear(); } const auto result = entries.insert_or_assign(key, std::move(entry)); return &result.first->second.segments; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 4e0b45cf0a..e9b162fb0f 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -192,6 +192,7 @@ public: .memory_offset = location.offset, .offset = offset, .size = size, + .memory_type = location.memory_type, }); multi_range_total += size; } diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp index 8b0158924d..79086c4244 100644 --- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp +++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp @@ -21,11 +21,13 @@ MultiRangeBufferCache::MultiRangeBufferCache(const Device& device_, if (!device.IsSparseBindingSupported()) { return; } - const VkDeviceSize queried = QueryBlockSize(); - if (queried == 0) { + u32 memory_type_bits = 0; + const VkDeviceSize queried = QueryBlockSize(memory_type_bits); + if (queried == 0 || memory_type_bits == 0) { return; } block_size = queried; + sparse_memory_type_bits = memory_type_bits; use_sparse = true; } @@ -44,7 +46,7 @@ MultiRangeBufferCache::~MultiRangeBufferCache() { retired.clear(); } -VkDeviceSize MultiRangeBufferCache::QueryBlockSize() const { +VkDeviceSize MultiRangeBufferCache::QueryBlockSize(u32& memory_type_bits) const { const VkDevice logical = *device.GetLogical(); const auto& dld = device.GetDispatchLoader(); const VkBufferCreateInfo probe_ci{ @@ -73,6 +75,7 @@ VkDeviceSize MultiRangeBufferCache::QueryBlockSize() const { }; dld.vkGetBufferMemoryRequirements2(logical, &reqs_info, &reqs2); dld.vkDestroyBuffer(logical, probe, nullptr); + memory_type_bits = reqs2.memoryRequirements.memoryTypeBits; return reqs2.memoryRequirements.alignment; } @@ -98,6 +101,12 @@ bool MultiRangeBufferCache::CanBindSparse(std::span sour if (source.memory == VK_NULL_HANDLE) { return false; } + if (source.memory_type >= 32) { + return false; + } + if (((sparse_memory_type_bits >> source.memory_type) & 1) == 0) { + return false; + } const VkDeviceSize memory_offset = source.memory_offset + source.offset; if ((memory_offset % block_size) != 0) { return false; diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h index e809897bc1..6c9b738876 100644 --- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h +++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h @@ -22,6 +22,7 @@ struct MultiRangeSource { VkDeviceSize memory_offset{}; VkDeviceSize offset{}; VkDeviceSize size{}; + u32 memory_type{}; }; struct MultiRangeRef { @@ -81,7 +82,7 @@ private: [[nodiscard]] VkBuffer CreateSparse(std::span sources, VkDeviceSize total); - [[nodiscard]] VkDeviceSize QueryBlockSize() const; + [[nodiscard]] VkDeviceSize QueryBlockSize(u32& memory_type_bits) const; void DestroySparse(VkBuffer handle); @@ -92,6 +93,7 @@ private: Scheduler& scheduler; bool use_sparse{}; VkDeviceSize block_size{DEFAULT_BLOCK_SIZE}; + u32 sparse_memory_type_bits{}; VkBufferUsageFlags sparse_usage{}; std::unordered_map entries; std::vector retired; diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index 4f070c6f88..ab8c8b238d 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -549,6 +549,7 @@ MemoryLocation Buffer::Location() const noexcept { return MemoryLocation{ .memory = info.deviceMemory, .offset = info.offset, + .memory_type = info.memoryType, }; } diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index 3d7c3b9fe8..1535b91688 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -744,6 +744,7 @@ private: struct MemoryLocation { VkDeviceMemory memory{}; VkDeviceSize offset{}; + u32 memory_type{}; }; class Buffer {