diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 4ee87d4d7d..4c345dd213 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -114,7 +114,7 @@ void BufferCache

::TickFrame() { template void BufferCache

::UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size) { - if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}); }) { + if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) { virtual_ranges.Unmap(as_id, gpu_addr, size); } } @@ -215,8 +215,8 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am BufferId buffer_b; do { channel_state->has_deleted_buffers = false; - buffer_a = FindBuffer(*cpu_src_address, static_cast(amount)); - buffer_b = FindBuffer(*cpu_dest_address, static_cast(amount)); + buffer_a = FindBuffer(*cpu_src_address, static_cast(amount), false); + buffer_b = FindBuffer(*cpu_dest_address, static_cast(amount), false); } while (channel_state->has_deleted_buffers); auto& src_buffer = slot_buffers[buffer_a]; auto& dest_buffer = slot_buffers[buffer_b]; @@ -272,7 +272,7 @@ bool BufferCache

::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) { ClearDownload(*cpu_dst_address, size); gpu_modified_ranges.Subtract(*cpu_dst_address, size); - const BufferId buffer = FindBuffer(*cpu_dst_address, static_cast(size)); + const BufferId buffer = FindBuffer(*cpu_dst_address, static_cast(size), false); Buffer& dest_buffer = slot_buffers[buffer]; const u32 offset = dest_buffer.Offset(*cpu_dst_address); runtime.ClearBuffer(dest_buffer, offset, size, value); @@ -294,7 +294,7 @@ std::pair BufferCache

::ObtainBuffer(GPUVAddr gpu_ad template std::pair BufferCache

::ObtainCPUBuffer( DAddr device_addr, u32 size, ObtainBufferSynchronize sync_info, ObtainBufferOperation post_op) { - const BufferId buffer_id = FindBuffer(device_addr, size); + const BufferId buffer_id = FindBuffer(device_addr, size, false); Buffer& buffer = slot_buffers[buffer_id]; // synchronize op @@ -1010,7 +1010,7 @@ void BufferCache

::ResolveMultiRangeStorage(Binding& binding, bool is_written, std::vector& pool) { binding.segment_first = 0; binding.segment_count = 0; - if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}); }) { + if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) { if (binding.gpu_addr == 0 || binding.size == 0) { return; } @@ -1046,7 +1046,7 @@ void BufferCache

::ResolveMultiRangeStorage(Binding& binding, bool is_written, template bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_written, std::span pool) { - if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}); }) { + if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) { if (binding.segment_count < 2) { return false; } @@ -1069,7 +1069,7 @@ bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_writt } runtime.PushMultiRangeSource(buffer, offset, segment.size); } - return runtime.BindMultiRangeStorageBuffer(key); + return runtime.BindMultiRangeStorageBuffer(key, is_written); } else { return false; } @@ -1319,11 +1319,11 @@ void BufferCache

::UpdateIndexBuffer() { auto inline_index_size = static_cast(draw_state.inline_index_draw_indexes.size()); u32 buffer_size = Common::AlignUp(inline_index_size, CACHING_PAGESIZE); if (inline_buffer_id == NULL_BUFFER_ID) [[unlikely]] { - inline_buffer_id = CreateBuffer(0, buffer_size); + inline_buffer_id = CreateBuffer(0, buffer_size, false); } if (slot_buffers[inline_buffer_id].SizeBytes() < buffer_size) [[unlikely]] { slot_buffers.erase(inline_buffer_id); - inline_buffer_id = CreateBuffer(0, buffer_size); + inline_buffer_id = CreateBuffer(0, buffer_size, false); } channel_state->index_buffer = Binding{ .device_addr = 0, @@ -1346,7 +1346,7 @@ void BufferCache

::UpdateIndexBuffer() { channel_state->index_buffer = Binding{ .device_addr = *device_addr, .size = size, - .buffer_id = FindBuffer(*device_addr, size), + .buffer_id = FindBuffer(*device_addr, size, false), }; } @@ -1383,7 +1383,7 @@ void BufferCache

::UpdateVertexBuffer(u32 index) { if (!gpu_memory->IsWithinGPUAddressRange(gpu_addr_end) || size >= 64_MiB) { size = static_cast(gpu_memory->MaxContinuousRange(gpu_addr_begin, size)); } - const BufferId buffer_id = FindBuffer(*device_addr, size); + const BufferId buffer_id = FindBuffer(*device_addr, size, false); const Binding binding{ .device_addr = *device_addr, .size = size, @@ -1404,7 +1404,7 @@ void BufferCache

::UpdateDrawIndirect() { binding = Binding{ .device_addr = *device_addr, .size = static_cast(size), - .buffer_id = FindBuffer(*device_addr, static_cast(size)), + .buffer_id = FindBuffer(*device_addr, static_cast(size), false), }; }; if (current_draw_indirect->include_count) { @@ -1428,7 +1428,7 @@ void BufferCache

::UpdateUniformBuffers(size_t stage) { channel_state->dirty_uniform_buffers[stage] |= 1U << index; } // Resolve buffer - binding.buffer_id = FindBuffer(binding.device_addr, binding.size); + binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); }); } @@ -1437,7 +1437,7 @@ void BufferCache

::UpdateStorageBuffers(size_t stage) { ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { // Resolve buffer Binding& binding = channel_state->storage_buffers[stage][index]; - const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size); + const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size, false); binding.buffer_id = buffer_id; const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; ResolveMultiRangeStorage(binding, is_written, graphics_segments); @@ -1448,7 +1448,7 @@ template void BufferCache

::UpdateTextureBuffers(size_t stage) { ForEachEnabledBit(channel_state->enabled_texture_buffers[stage], [&](u32 index) { Binding& binding = channel_state->texture_buffers[stage][index]; - binding.buffer_id = FindBuffer(binding.device_addr, binding.size); + binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); }); } @@ -1472,7 +1472,7 @@ void BufferCache

::UpdateTransformFeedbackBuffer(u32 index) { channel_state->transform_feedback_buffers[index] = NULL_BINDING; return; } - const BufferId buffer_id = FindBuffer(*device_addr, size); + const BufferId buffer_id = FindBuffer(*device_addr, size, false); channel_state->transform_feedback_buffers[index] = Binding{ .device_addr = *device_addr, .size = size, @@ -1494,7 +1494,7 @@ void BufferCache

::UpdateComputeUniformBuffers() { binding.size = cbuf.size; } } - binding.buffer_id = FindBuffer(binding.device_addr, binding.size); + binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); }); } @@ -1503,7 +1503,7 @@ void BufferCache

::UpdateComputeStorageBuffers() { ForEachEnabledBit(channel_state->enabled_compute_storage_buffers, [&](u32 index) { // Resolve buffer Binding& binding = channel_state->compute_storage_buffers[index]; - binding.buffer_id = FindBuffer(binding.device_addr, binding.size); + binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); const bool is_written = ((channel_state->written_compute_storage_buffers >> index) & 1) != 0; ResolveMultiRangeStorage(binding, is_written, compute_segments); @@ -1514,7 +1514,7 @@ template void BufferCache

::UpdateComputeTextureBuffers() { ForEachEnabledBit(channel_state->enabled_compute_texture_buffers, [&](u32 index) { Binding& binding = channel_state->compute_texture_buffers[index]; - binding.buffer_id = FindBuffer(binding.device_addr, binding.size); + binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); }); } @@ -1845,7 +1845,7 @@ void BufferCache

::InlineMemoryImplementation(DAddr dest_address, size_t copy_ ClearDownload(dest_address, copy_size); gpu_modified_ranges.Subtract(dest_address, copy_size); - BufferId buffer_id = FindBuffer(dest_address, static_cast(copy_size)); + BufferId buffer_id = FindBuffer(dest_address, static_cast(copy_size), false); auto& buffer = slot_buffers[buffer_id]; SynchronizeBuffer(buffer, dest_address, static_cast(copy_size)); @@ -2043,9 +2043,9 @@ Binding BufferCache

::StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, } const Binding binding{ .device_addr = *aligned_device_addr, + .gpu_addr = aligned_gpu_addr, .size = binding_size, .buffer_id = BufferId{}, - .gpu_addr = aligned_gpu_addr, }; return binding; } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 18a8d19fe9..0dbb368c47 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -82,9 +82,9 @@ static constexpr u32 DEFAULT_SKIP_CACHE_SIZE = static_cast(4_KiB); struct Binding { DAddr device_addr{}; + GPUVAddr gpu_addr{}; u32 size{}; BufferId buffer_id; - GPUVAddr gpu_addr{}; u32 segment_first{}; u32 segment_count{}; }; @@ -432,8 +432,7 @@ private: void MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size); - [[nodiscard]] BufferId FindBuffer(DAddr device_addr, u32 size, - bool sparse_compatible = false); + [[nodiscard]] BufferId FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible); void WaitForGpuFenceIfNeeded(Buffer& buffer); @@ -442,7 +441,7 @@ private: void JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, bool accumulate_stream_score); [[nodiscard]] BufferId CreateBuffer(DAddr device_addr, u32 wanted_size, - bool sparse_compatible = false); + bool sparse_compatible); void Register(BufferId buffer_id); @@ -533,10 +532,10 @@ private: using TickType = u64; }; Common::LeastRecentlyUsedCache lru_cache; - u64 frame_tick = 0; VirtualRangeCache virtual_ranges; std::vector graphics_segments; std::vector compute_segments; + u64 frame_tick = 0; u64 total_used_memory = 0; u64 minimum_memory = 0; u64 critical_memory = 0; diff --git a/src/video_core/buffer_cache/virtual_range_cache.h b/src/video_core/buffer_cache/virtual_range_cache.h index 3682acf5bc..f4284d6abb 100644 --- a/src/video_core/buffer_cache/virtual_range_cache.h +++ b/src/video_core/buffer_cache/virtual_range_cache.h @@ -7,12 +7,12 @@ #include #include #include -#include #include #include #include "common/common_types.h" +#include "common/container/unordered_map.h" #include "video_core/memory_manager.h" namespace VideoCommon { @@ -27,10 +27,16 @@ using VirtualSegments = boost::container::small_vector; class VirtualRangeCache { public: + static constexpr size_t MAX_ENTRIES = 8192; + static constexpr size_t MAX_DEFERRED = 4096; + const VirtualSegments* Query(Tegra::MemoryManager& memory, GPUVAddr gpu_addr, u32 size) { if (has_deferred.load(std::memory_order_acquire)) { ApplyDeferred(); } + if (entries.size() > MAX_ENTRIES) { + entries.clear(); + } const size_t as_id = memory.GetID(); const u64 key = MakeKey(as_id, gpu_addr); const auto it = entries.find(key); @@ -38,7 +44,7 @@ public: it->second.gpu_addr == gpu_addr && it->second.size == size) { return &it->second.segments; } - Entry entry; + Entry entry{}; entry.as_id = as_id; entry.gpu_addr = gpu_addr; entry.size = size; @@ -87,30 +93,26 @@ public: return; } } - deferred.push_back(DeferredUnmap{ - .as_id = as_id, - .gpu_addr = gpu_addr, - .size = size, - }); + if (deferred.size() >= MAX_DEFERRED) { + deferred.clear(); + deferred_overflow = true; + } else { + deferred.push_back(DeferredUnmap{ + .as_id = as_id, + .gpu_addr = gpu_addr, + .size = size, + }); + } } has_deferred.store(true, std::memory_order_release); } - void Clear() { - { - std::scoped_lock lock{deferred_mutex}; - deferred.clear(); - } - has_deferred.store(false, std::memory_order_release); - entries.clear(); - } - private: struct Entry { + VirtualSegments segments; size_t as_id{}; GPUVAddr gpu_addr{}; u32 size{}; - VirtualSegments segments; }; struct DeferredUnmap { @@ -125,10 +127,17 @@ private: void ApplyDeferred() { std::vector pending; + bool overflow = false; { std::scoped_lock lock{deferred_mutex}; has_deferred.store(false, std::memory_order_release); pending.swap(deferred); + overflow = deferred_overflow; + deferred_overflow = false; + } + if (overflow) { + entries.clear(); + return; } if (pending.empty() || entries.empty()) { return; @@ -154,10 +163,11 @@ private: } } - std::unordered_map entries; + ::Common::unordered_map entries; std::vector deferred; std::mutex deferred_mutex; std::atomic has_deferred{false}; + bool deferred_overflow{}; }; } // namespace VideoCommon diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.h b/src/video_core/renderer_opengl/gl_buffer_cache.h index ef1bc47448..6fee58a0eb 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.h +++ b/src/video_core/renderer_opengl/gl_buffer_cache.h @@ -24,7 +24,7 @@ class BufferCacheRuntime; class Buffer : public VideoCommon::BufferBase { public: explicit Buffer(BufferCacheRuntime&, DAddr cpu_addr, u64 size_bytes, - bool sparse_compatible = false); + bool sparse_compatible); explicit Buffer(BufferCacheRuntime&, VideoCommon::NullBufferParams); void ImmediateUpload(size_t offset, std::span data) noexcept; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 351dcdb9da..0256306214 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -357,7 +357,7 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m staging_pool{staging_pool_}, guest_descriptor_queue{guest_descriptor_queue_}, quad_index_pass(device, scheduler, descriptor_pool, staging_pool, compute_pass_descriptor_queue), - multi_range_buffers(device_, memory_allocator_, scheduler_) { + multi_range_buffers(device_) { const VkDriverIdKHR driver_id = device.GetDriverID(); limit_dynamic_storage_buffers = driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || driver_id == VK_DRIVER_ID_ARM_PROPRIETARY; @@ -411,6 +411,7 @@ u32 BufferCacheRuntime::GetStorageBufferAlignment() const { } void BufferCacheRuntime::TickFrame(Common::SlotVector& slot_buffers) noexcept { + multi_range_buffers.TickFrame(scheduler); for (auto it = slot_buffers.begin(); it != slot_buffers.end(); it++) { if (scheduler.IsFree(it->LastUsageTick())) { it->ResetUsageTracking(); @@ -545,22 +546,26 @@ void BufferCacheRuntime::ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t si }); } -bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key) { +bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key, bool is_written) { if (multi_range_sources.empty() || multi_range_total == 0) { return false; } - const MultiRangeRef ref = multi_range_buffers.Get(key, multi_range_sources, multi_range_total); + const MultiRangeRef ref = multi_range_buffers.Get(device, scheduler, memory_allocator, key, + multi_range_sources, multi_range_total); if (ref.handle == VK_NULL_HANDLE) { return false; } + if (is_written && !ref.sparse) { + return false; + } if (ref.needs_gather) { PreCopyBarrier(); VkDeviceSize dst_offset = 0; for (const MultiRangeSource& source : multi_range_sources) { const std::array copy{VideoCommon::BufferCopy{ - .src_offset = static_cast(source.offset), - .dst_offset = static_cast(dst_offset), - .size = static_cast(source.size), + .src_offset = u64(source.offset), + .dst_offset = u64(dst_offset), + .size = size_t(source.size), }}; CopyBuffer(ref.handle, source.handle, copy, false); dst_offset += source.size; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 5cb445ec74..fbd8c2e753 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -35,7 +35,7 @@ class Buffer : public VideoCommon::BufferBase { public: explicit Buffer(BufferCacheRuntime&, VideoCommon::NullBufferParams null_params); explicit Buffer(BufferCacheRuntime& runtime, VAddr cpu_addr_, u64 size_bytes_, - bool sparse_compatible_ = false); + bool sparse_compatible_); [[nodiscard]] VkBufferView View(u32 offset, u32 size, VideoCore::Surface::PixelFormat format); @@ -169,14 +169,14 @@ public: } [[nodiscard]] VkDeviceSize SparseAlignmentFor(bool sparse_compatible) const noexcept { - if (!sparse_compatible || !multi_range_buffers.UsesSparse()) { + if (!sparse_compatible || !multi_range_buffers.use_sparse) { return 0; } - return multi_range_buffers.BlockSize(); + return multi_range_buffers.block_size; } [[nodiscard]] bool PrefersSparseSources() const noexcept { - return multi_range_buffers.UsesSparse(); + return multi_range_buffers.use_sparse; } void ResetMultiRange() noexcept { @@ -192,19 +192,20 @@ public: .memory_offset = location.offset, .offset = offset, .size = size, + .write_tick = buffer.getWriteTick(), .memory_type = location.memory_type, }); multi_range_total += size; } - bool BindMultiRangeStorageBuffer(u64 key); + bool BindMultiRangeStorageBuffer(u64 key, bool is_written); void InvalidateMultiRange(u64 key) { multi_range_buffers.Invalidate(key); } void OnBufferDeleted(const Buffer& buffer) { - multi_range_buffers.DropOwner(buffer.Handle()); + multi_range_buffers.DropOwner(scheduler, buffer.Handle()); } void BindUniformBuffer(const Buffer& buffer, u32 offset, u32 size) { diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp index 4d07081317..edfc678992 100644 --- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp +++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include #include +#include #include "video_core/renderer_vulkan/vk_multi_range_buffer.h" #include "video_core/renderer_vulkan/vk_scheduler.h" @@ -9,10 +11,7 @@ namespace Vulkan { -MultiRangeBufferCache::MultiRangeBufferCache(const Device& device_, - MemoryAllocator& memory_allocator_, - Scheduler& scheduler_) - : device{device_}, memory_allocator{memory_allocator_}, scheduler{scheduler_} { +MultiRangeBufferCache::MultiRangeBufferCache(const Device& device) { sparse_usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; if (device.IsBufferDeviceAddressSupported()) { @@ -22,7 +21,7 @@ MultiRangeBufferCache::MultiRangeBufferCache(const Device& device_, return; } u32 memory_type_bits = 0; - const VkDeviceSize queried = QueryBlockSize(memory_type_bits); + const VkDeviceSize queried = QueryBlockSize(device, memory_type_bits); if (queried == 0 || memory_type_bits == 0) { return; } @@ -31,22 +30,8 @@ MultiRangeBufferCache::MultiRangeBufferCache(const Device& device_, use_sparse = true; } -MultiRangeBufferCache::~MultiRangeBufferCache() { - const VkDevice logical = *device.GetLogical(); - const auto& dld = device.GetDispatchLoader(); - for (auto& [key, entry] : entries) { - if (entry.sparse_handle != VK_NULL_HANDLE) { - dld.vkDestroyBuffer(logical, entry.sparse_handle, nullptr); - } - } - entries.clear(); - for (const Retired& item : retired) { - dld.vkDestroyBuffer(logical, item.handle, nullptr); - } - retired.clear(); -} - -VkDeviceSize MultiRangeBufferCache::QueryBlockSize(u32& memory_type_bits) const { +VkDeviceSize MultiRangeBufferCache::QueryBlockSize(const Device& device, + u32& memory_type_bits) const { const VkDevice logical = *device.GetLogical(); const auto& dld = device.GetDispatchLoader(); const VkBufferCreateInfo probe_ci{ @@ -63,6 +48,7 @@ VkDeviceSize MultiRangeBufferCache::QueryBlockSize(u32& memory_type_bits) const if (dld.vkCreateBuffer(logical, &probe_ci, nullptr, &probe) != VK_SUCCESS) { return 0; } + const SparseBuffer owned{probe, logical, dld}; const VkBufferMemoryRequirementsInfo2 reqs_info{ .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2, .pNext = nullptr, @@ -74,7 +60,6 @@ VkDeviceSize MultiRangeBufferCache::QueryBlockSize(u32& memory_type_bits) const .memoryRequirements = {}, }; dld.vkGetBufferMemoryRequirements2(logical, &reqs_info, &reqs2); - dld.vkDestroyBuffer(logical, probe, nullptr); memory_type_bits = reqs2.memoryRequirements.memoryTypeBits; return reqs2.memoryRequirements.alignment; } @@ -86,40 +71,36 @@ u64 MultiRangeBufferCache::HashSources(std::span sources hash *= 0x100000001b3ULL; }; for (const MultiRangeSource& source : sources) { - mix(reinterpret_cast(source.handle)); - mix(static_cast(source.offset)); - mix(static_cast(source.size)); + mix(u64(source.handle)); + mix(u64(source.offset)); + mix(u64(source.size)); + } + return hash; +} + +u64 MultiRangeBufferCache::HashContent(std::span sources) const { + u64 hash = 0xcbf29ce484222325ULL; + for (const MultiRangeSource& source : sources) { + hash ^= source.write_tick; + hash *= 0x100000001b3ULL; } return hash; } bool MultiRangeBufferCache::CanBindSparse(std::span sources) const { - if (!UsesSparse()) { - return false; - } - for (const MultiRangeSource& source : sources) { - if (source.memory == VK_NULL_HANDLE) { - return false; - } - if (source.memory_type >= 32) { - return false; - } - if (((sparse_memory_type_bits >> source.memory_type) & 1) == 0) { - return false; - } - const VkDeviceSize memory_offset = source.memory_offset + source.offset; - if ((memory_offset % block_size) != 0) { - return false; - } - if ((source.size % block_size) != 0) { - return false; - } - } - return true; + return use_sparse && + std::none_of(sources.begin(), sources.end(), + [block = block_size, bits = sparse_memory_type_bits](auto const& e) { + const VkDeviceSize memory_offset = e.memory_offset + e.offset; + return e.memory == VK_NULL_HANDLE || e.memory_type >= 32 || + ((bits >> e.memory_type) & 1) == 0 || + (memory_offset % block) != 0 || (e.size % block) != 0; + }); } -VkBuffer MultiRangeBufferCache::CreateSparse(std::span sources, - VkDeviceSize total) { +SparseBuffer MultiRangeBufferCache::CreateSparse(const Device& device, Scheduler& scheduler, + std::span sources, + VkDeviceSize total) { const VkDevice logical = *device.GetLogical(); const auto& dld = device.GetDispatchLoader(); const VkBufferCreateInfo buffer_ci{ @@ -132,10 +113,11 @@ VkBuffer MultiRangeBufferCache::CreateSparse(std::span s .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, }; - VkBuffer handle{}; - if (dld.vkCreateBuffer(logical, &buffer_ci, nullptr, &handle) != VK_SUCCESS) { - return VK_NULL_HANDLE; + VkBuffer raw{}; + if (dld.vkCreateBuffer(logical, &buffer_ci, nullptr, &raw) != VK_SUCCESS) { + return SparseBuffer{}; } + SparseBuffer handle{raw, logical, dld}; std::vector binds; binds.reserve(sources.size()); VkDeviceSize resource_offset = 0; @@ -150,7 +132,7 @@ VkBuffer MultiRangeBufferCache::CreateSparse(std::span s resource_offset += source.size; } const VkSparseBufferMemoryBindInfo buffer_bind{ - .buffer = handle, + .buffer = raw, .bindCount = static_cast(binds.size()), .pBinds = binds.data(), }; @@ -180,31 +162,34 @@ VkBuffer MultiRangeBufferCache::CreateSparse(std::span s bind_result = device.GetGraphicsQueue().BindSparse(bind_info, *fence); } if (bind_result != VK_SUCCESS) { - dld.vkDestroyBuffer(logical, handle, nullptr); - return VK_NULL_HANDLE; + return SparseBuffer{}; } fence.Wait(); return handle; } -void MultiRangeBufferCache::DestroySparse(VkBuffer handle) { - if (handle == VK_NULL_HANDLE) { - return; +bool MultiRangeBufferCache::DestroySparse(Scheduler& scheduler, SparseBuffer&& handle) { + if (!handle) { + return true; + } + if (retired.size() == retired.capacity()) { + DrainRetired(scheduler); + } + if (retired.size() == retired.capacity()) { + return false; } retired.push_back(Retired{ - .handle = handle, + .handle = std::move(handle), .tick = scheduler.CurrentTick(), }); + return true; } -void MultiRangeBufferCache::DrainRetired() { - const VkDevice logical = *device.GetLogical(); - const auto& dld = device.GetDispatchLoader(); +void MultiRangeBufferCache::DrainRetired(Scheduler& scheduler) { size_t index = 0; while (index < retired.size()) { if (scheduler.IsFree(retired[index].tick)) { - dld.vkDestroyBuffer(logical, retired[index].handle, nullptr); - retired[index] = retired.back(); + retired[index] = std::move(retired.back()); retired.pop_back(); } else { ++index; @@ -212,48 +197,64 @@ void MultiRangeBufferCache::DrainRetired() { } } -MultiRangeRef MultiRangeBufferCache::Get(u64 key, std::span sources, +MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& scheduler, + MemoryAllocator& memory_allocator, u64 key, + std::span sources, VkDeviceSize total) { if (sources.empty() || total == 0) { return MultiRangeRef{}; } if (!retired.empty()) { - DrainRetired(); + DrainRetired(scheduler); } const u64 geometry = HashSources(sources); + const u64 content = HashContent(sources); const auto it = entries.find(key); if (it != entries.end() && it->second.geometry == geometry && it->second.size == total) { Entry& entry = it->second; + entry.frame = frame_tick; + entry.gpu_tick = scheduler.CurrentTick(); + if (entry.content != content) { + entry.content = content; + entry.dirty = true; + } MultiRangeRef ref{ - .handle = entry.sparse_handle, + .handle = *entry.sparse_handle, .address = entry.address, .size = entry.size, + .sparse = true, .needs_gather = false, }; - if (entry.sparse_handle == VK_NULL_HANDLE) { + if (!entry.sparse_handle) { ref.handle = *entry.gathered; + ref.sparse = false; ref.needs_gather = entry.dirty; } return ref; } if (it != entries.end()) { - DestroySparse(it->second.sparse_handle); + if (!DestroySparse(scheduler, std::move(it->second.sparse_handle))) { + return MultiRangeRef{}; + } entries.erase(it); } - Entry entry; + Entry entry{}; entry.geometry = geometry; + entry.content = content; entry.size = total; + entry.frame = frame_tick; + entry.gpu_tick = scheduler.CurrentTick(); if (CanBindSparse(sources)) { - entry.sparse_handle = CreateSparse(sources, total); - if (entry.sparse_handle != VK_NULL_HANDLE) { + entry.sparse_handle = CreateSparse(device, scheduler, sources, total); + if (entry.sparse_handle) { entry.owners.reserve(sources.size()); for (const MultiRangeSource& source : sources) { entry.owners.push_back(source.handle); } } } - if (entry.sparse_handle == VK_NULL_HANDLE) { + if (!entry.sparse_handle) { VkBufferUsageFlags flags = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; @@ -274,21 +275,23 @@ MultiRangeRef MultiRangeBufferCache::Get(u64 key, std::spansecond.dirty = false; } } -void MultiRangeBufferCache::DropOwner(VkBuffer owner) { +void MultiRangeBufferCache::DropOwner(Scheduler& scheduler, VkBuffer owner) { if (owner == VK_NULL_HANDLE) { return; } @@ -315,8 +317,7 @@ void MultiRangeBufferCache::DropOwner(VkBuffer owner) { break; } } - if (owned) { - DestroySparse(entry.sparse_handle); + if (owned && DestroySparse(scheduler, std::move(entry.sparse_handle))) { it = entries.erase(it); } else { ++it; @@ -325,17 +326,29 @@ void MultiRangeBufferCache::DropOwner(VkBuffer owner) { } void MultiRangeBufferCache::Invalidate(u64 key) { - const auto it = entries.find(key); - if (it != entries.end()) { + if (auto const it = entries.find(key); it != entries.end()) { it->second.dirty = true; } } -void MultiRangeBufferCache::Clear() { - for (auto& [key, entry] : entries) { - DestroySparse(entry.sparse_handle); +void MultiRangeBufferCache::TickFrame(Scheduler& scheduler) { + ++frame_tick; + if (!retired.empty()) { + DrainRetired(scheduler); + } + if (entries.empty()) { + return; + } + for (auto it = entries.begin(); it != entries.end();) { + Entry& entry = it->second; + const bool expired = + frame_tick - entry.frame > FRAMES_TO_LIVE && scheduler.IsFree(entry.gpu_tick); + if (expired && DestroySparse(scheduler, std::move(entry.sparse_handle))) { + it = entries.erase(it); + } else { + ++it; + } } - entries.clear(); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h index 9d91822159..e4f9591e7d 100644 --- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h +++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h @@ -4,15 +4,20 @@ #pragma once #include -#include #include +#include + +#include "common/common_funcs.h" #include "common/common_types.h" +#include "common/container/unordered_map.h" #include "video_core/vulkan_common/vulkan_memory_allocator.h" #include "video_core/vulkan_common/vulkan_wrapper.h" namespace Vulkan { +using SparseBuffer = vk::Handle; + class Device; class Scheduler; @@ -22,6 +27,7 @@ struct MultiRangeSource { VkDeviceSize memory_offset{}; VkDeviceSize offset{}; VkDeviceSize size{}; + u64 write_tick{}; u32 memory_type{}; }; @@ -29,77 +35,76 @@ struct MultiRangeRef { VkBuffer handle{}; VkDeviceAddress address{}; VkDeviceSize size{}; + bool sparse{}; bool needs_gather{}; }; class MultiRangeBufferCache final { public: static constexpr VkDeviceSize DEFAULT_BLOCK_SIZE = 64 * 1024; + static constexpr u64 FRAMES_TO_LIVE = 120; + static constexpr size_t MAX_RETIRED = 256; - explicit MultiRangeBufferCache(const Device& device_, MemoryAllocator& memory_allocator_, - Scheduler& scheduler_); - ~MultiRangeBufferCache(); + explicit MultiRangeBufferCache(const Device& device); - MultiRangeBufferCache(const MultiRangeBufferCache&) = delete; - MultiRangeBufferCache& operator=(const MultiRangeBufferCache&) = delete; + YUZU_NON_COPYABLE(MultiRangeBufferCache); - [[nodiscard]] bool UsesSparse() const noexcept { - return use_sparse; - } - - [[nodiscard]] VkDeviceSize BlockSize() const noexcept { - return block_size; - } - - [[nodiscard]] MultiRangeRef Get(u64 key, std::span sources, + [[nodiscard]] MultiRangeRef Get(const Device& device, Scheduler& scheduler, + MemoryAllocator& memory_allocator, u64 key, + std::span sources, VkDeviceSize total); void MarkGathered(u64 key); void Invalidate(u64 key); - void DropOwner(VkBuffer owner); + void DropOwner(Scheduler& scheduler, VkBuffer owner); - void Clear(); + void TickFrame(Scheduler& scheduler); + + VkDeviceSize block_size{DEFAULT_BLOCK_SIZE}; + bool use_sparse{}; private: struct Retired { - VkBuffer handle{}; + SparseBuffer handle; u64 tick{}; }; struct Entry { vk::Buffer gathered; - VkBuffer sparse_handle{}; + SparseBuffer sparse_handle; + std::vector owners; VkDeviceAddress address{}; VkDeviceSize size{}; u64 geometry{}; + u64 content{}; + u64 frame{}; + u64 gpu_tick{}; bool dirty{true}; - std::vector owners; }; [[nodiscard]] u64 HashSources(std::span sources) const; + [[nodiscard]] u64 HashContent(std::span sources) const; + [[nodiscard]] bool CanBindSparse(std::span sources) const; - [[nodiscard]] VkBuffer CreateSparse(std::span sources, - VkDeviceSize total); + [[nodiscard]] SparseBuffer CreateSparse(const Device& device, Scheduler& scheduler, + std::span sources, + VkDeviceSize total); - [[nodiscard]] VkDeviceSize QueryBlockSize(u32& memory_type_bits) const; + [[nodiscard]] VkDeviceSize QueryBlockSize(const Device& device, u32& memory_type_bits) const; - void DestroySparse(VkBuffer handle); + bool DestroySparse(Scheduler& scheduler, SparseBuffer&& handle); - void DrainRetired(); + void DrainRetired(Scheduler& scheduler); - const Device& device; - MemoryAllocator& memory_allocator; - Scheduler& scheduler; - bool use_sparse{}; - VkDeviceSize block_size{DEFAULT_BLOCK_SIZE}; + ::Common::unordered_map entries; + boost::container::static_vector retired; + u64 frame_tick{}; u32 sparse_memory_type_bits{}; VkBufferUsageFlags sparse_usage{}; - std::unordered_map entries; - std::vector retired; }; } // namespace Vulkan diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index d7e0087c73..077b1c3e23 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -1150,8 +1150,8 @@ private: bool owns_static_pipeline_cache{}; u32 instance_version{}; ///< Vulkan instance version. u32 graphics_family{}; ///< Main graphics queue family index. - bool graphics_family_sparse_binding{}; u32 present_family{}; ///< Main present queue family index. + bool graphics_family_sparse_binding{}; struct Extensions { #define EXTENSION(prefix, macro_name, var_name) bool var_name{}; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index bee7cd6e57..f4721543ae 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -275,9 +275,13 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa const std::span mapped_data = data ? std::span{data, ci.size} : std::span{}; const bool is_coherent = (property_flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0; - return vk::Buffer(handle, *device.GetLogical(), allocator, allocation, mapped_data, - is_coherent, - device.GetDispatchLoader()); + const vk::MemoryLocation location{ + .memory = alloc_info.deviceMemory, + .offset = alloc_info.offset, + .memory_type = alloc_info.memoryType, + }; + return vk::Buffer(handle, *device.GetLogical(), allocator, allocation, mapped_data, is_coherent, + location, device.GetDispatchLoader()); } vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage, @@ -321,8 +325,13 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa } const bool is_coherent = (property_flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0; + const vk::MemoryLocation location{ + .memory = alloc_info.deviceMemory, + .offset = alloc_info.offset, + .memory_type = alloc_info.memoryType, + }; return vk::Buffer(handle, *device.GetLogical(), allocator, allocation, mapped_data, is_coherent, - device.GetDispatchLoader()); + location, device.GetDispatchLoader()); } MemoryCommit MemoryAllocator::Commit(const VkMemoryRequirements &reqs, MemoryUsage usage) diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index ab8c8b238d..f04f137da5 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -540,19 +540,6 @@ void Buffer::SetObjectNameEXT(const char* name) const { SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_BUFFER, name); } -MemoryLocation Buffer::Location() const noexcept { - if (!allocation) { - return MemoryLocation{}; - } - VmaAllocationInfo info{}; - vmaGetAllocationInfo(allocator, allocation, &info); - return MemoryLocation{ - .memory = info.deviceMemory, - .offset = info.offset, - .memory_type = info.memoryType, - }; -} - void Buffer::Release() const noexcept { if (handle) { vmaDestroyBuffer(allocator, handle, allocation); diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index 1535b91688..ffda49dd46 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -751,9 +751,10 @@ class Buffer { public: explicit Buffer(VkBuffer handle_, VkDevice owner_, VmaAllocator allocator_, VmaAllocation allocation_, std::span mapped_, bool is_coherent_, - const DeviceDispatch& dld_) noexcept + MemoryLocation location_, const DeviceDispatch& dld_) noexcept : handle{handle_}, owner{owner_}, allocator{allocator_}, - allocation{allocation_}, mapped{mapped_}, is_coherent{is_coherent_}, dld{&dld_} {} + allocation{allocation_}, mapped{mapped_}, is_coherent{is_coherent_}, + location{location_}, dld{&dld_} {} Buffer() = default; Buffer(const Buffer&) = delete; @@ -762,7 +763,7 @@ public: Buffer(Buffer&& rhs) noexcept : handle{std::exchange(rhs.handle, VkBuffer{})}, owner{rhs.owner}, allocator{rhs.allocator}, allocation{rhs.allocation}, mapped{rhs.mapped}, - is_coherent{rhs.is_coherent}, dld{rhs.dld} {} + is_coherent{rhs.is_coherent}, location{rhs.location}, dld{rhs.dld} {} Buffer& operator=(Buffer&& rhs) noexcept { Release(); @@ -772,6 +773,7 @@ public: allocation = rhs.allocation; mapped = rhs.mapped; is_coherent = rhs.is_coherent; + location = rhs.location; dld = rhs.dld; return *this; } @@ -818,7 +820,9 @@ public: void SetObjectNameEXT(const char* name) const; - MemoryLocation Location() const noexcept; + MemoryLocation Location() const noexcept { + return location; + } private: void Release() const noexcept; @@ -828,6 +832,7 @@ private: VmaAllocator allocator = nullptr; VmaAllocation allocation = nullptr; std::span mapped = {}; + MemoryLocation location{}; bool is_coherent = false; const DeviceDispatch* dld = nullptr; };