diff --git a/src/video_core/buffer_cache/buffer_base.h b/src/video_core/buffer_cache/buffer_base.h index d7eb21612f..6b2a37ea8c 100644 --- a/src/video_core/buffer_cache/buffer_base.h +++ b/src/video_core/buffer_cache/buffer_base.h @@ -129,13 +129,24 @@ public: write_tick = write_tick_; } + u64 ContentSerial() const noexcept { + return content_serial; + } + + void MarkContentModified() noexcept { + content_serial = ++next_content_serial; + } + private: + static inline u64 next_content_serial = 0; + VAddr cpu_addr = 0; BufferFlagBits flags{}; int stream_score = 0; size_t lru_id = SIZE_MAX; size_t size_bytes = 0; u64 write_tick = 0; + u64 content_serial = ++next_content_serial; }; } // namespace VideoCommon diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index eb214af50e..89d0003207 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -266,6 +266,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am const auto& copy = copies[0]; src_buffer.MarkUsage(copy.src_offset, copy.size); dest_buffer.MarkUsage(copy.dst_offset, copy.size); + dest_buffer.MarkContentModified(); runtime.CopyBuffer(dest_buffer, src_buffer, copies, true); if (has_new_downloads) { memory_tracker.MarkRegionAsGpuModified(*cpu_dest_address, amount); @@ -297,6 +298,7 @@ bool BufferCache

::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) { const u32 offset = dest_buffer.Offset(*cpu_dst_address); runtime.ClearBuffer(dest_buffer, offset, size, value); dest_buffer.MarkUsage(offset, size); + dest_buffer.MarkContentModified(); return true; } @@ -773,6 +775,7 @@ void BufferCache

::BindHostIndexBuffer() { if (draw_state.inline_index_draw_indexes.empty()) { SynchronizeBuffer(buffer, channel_state->index_buffer.device_addr, size); } else { + buffer.MarkContentModified(); if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) { auto upload_staging = runtime.UploadStagingBuffer(size); std::array copies{{BufferCopy{.src_offset = upload_staging.offset, .dst_offset = 0, .size = size}}}; @@ -1029,11 +1032,13 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 template void BufferCache

::ResolveMultiRangeStorage(Binding& binding, - std::vector& pool) { + std::vector& pool, + bool is_written) { binding.segment_first = 0; binding.segment_count = 0; if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) { - if (binding.gpu_addr == 0 || binding.size == 0) { + if (binding.gpu_addr == 0 || binding.size == 0 || + (is_written && !runtime.PrefersSparseSources())) { return; } const VirtualSegments* found = @@ -1041,7 +1046,7 @@ void BufferCache

::ResolveMultiRangeStorage(Binding& binding, if (!found || found->size() < 2) { return; } - const VirtualSegments segments = *found; + const VirtualSegments& segments = *found; const u32 first = static_cast(pool.size()); const bool prefer_sparse = runtime.PrefersSparseSources(); for (const VirtualSegment& segment : segments) { @@ -1078,9 +1083,7 @@ bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_writt const MultiRangeSegment& segment = pool[binding.segment_first + index]; Buffer& buffer = slot_buffers[segment.buffer_id]; TouchBuffer(buffer, segment.buffer_id); - if (SynchronizeBuffer(buffer, segment.device_addr, segment.size)) { - runtime.InvalidateMultiRange(key); - } + SynchronizeBuffer(buffer, segment.device_addr, segment.size); const u32 offset = buffer.Offset(segment.device_addr); buffer.MarkUsage(offset, segment.size); if (is_written) { @@ -1456,9 +1459,12 @@ void BufferCache

::UpdateStorageBuffers(size_t stage) { ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { // Resolve buffer Binding& binding = channel_state->storage_buffers[stage][index]; - const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size, false); - binding.buffer_id = buffer_id; - ResolveMultiRangeStorage(binding, graphics_segments); + const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; + ResolveMultiRangeStorage(binding, graphics_segments, is_written); + binding.buffer_id = NULL_BUFFER_ID; + if (binding.segment_count == 0 || is_written) { + binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); + } }); } @@ -1521,8 +1527,12 @@ void BufferCache

::UpdateComputeStorageBuffers() { ForEachEnabledBit(channel_state->enabled_compute_storage_buffers, [&](u32 index) { // Resolve buffer Binding& binding = channel_state->compute_storage_buffers[index]; - binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); - ResolveMultiRangeStorage(binding, compute_segments); + const bool is_written = ((channel_state->written_compute_storage_buffers >> index) & 1) != 0; + ResolveMultiRangeStorage(binding, compute_segments, is_written); + binding.buffer_id = NULL_BUFFER_ID; + if (binding.segment_count == 0 || is_written) { + binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); + } }); } @@ -1539,6 +1549,7 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 if constexpr (!IS_OPENGL) { Buffer& buffer = slot_buffers[buffer_id]; buffer.setWriteTick(runtime.CurrentTick()); + buffer.MarkContentModified(); } memory_tracker.MarkRegionAsGpuModified(device_addr, size); gpu_modified_ranges.Add(device_addr, size); @@ -1784,6 +1795,7 @@ bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 si template void BufferCache

::UploadMemory(Buffer& buffer, u64 total_size_bytes, u64 largest_copy, std::span copies) { + buffer.MarkContentModified(); if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) { MappedUploadMemory(buffer, total_size_bytes, copies); } else { @@ -1869,6 +1881,7 @@ void BufferCache

::InlineMemoryImplementation(DAddr dest_address, size_t copy_ BufferId buffer_id = FindBuffer(dest_address, static_cast(copy_size), false); auto& buffer = slot_buffers[buffer_id]; SynchronizeBuffer(buffer, dest_address, static_cast(copy_size)); + buffer.MarkContentModified(); if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) { auto upload_staging = runtime.UploadStagingBuffer(copy_size); diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index b4af2820d5..80a0b2c7eb 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -235,7 +235,8 @@ public: - void ResolveMultiRangeStorage(Binding& binding, std::vector& pool); + void ResolveMultiRangeStorage(Binding& binding, std::vector& pool, + bool is_written); void UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size); diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index dec26f328d..d3d3d7a437 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -428,6 +428,7 @@ u32 BufferCacheRuntime::GetStorageBufferAlignment() const { } void BufferCacheRuntime::TickFrame(Common::SlotVector& slot_buffers) noexcept { + multi_range_buffers.DrainRetired(scheduler); for (auto it = slot_buffers.begin(); it != slot_buffers.end(); it++) { if (scheduler.IsFree(it->LastUsageTick())) { it->ResetUsageTracking(); diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 582f665b4e..f83696f0de 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -192,7 +192,7 @@ public: .memory_offset = location.offset, .offset = offset, .size = size, - .write_tick = buffer.getWriteTick(), + .content_serial = buffer.ContentSerial(), .memory_type = location.memory_type, }); multi_range_total += size; @@ -200,10 +200,6 @@ public: bool BindMultiRangeStorageBuffer(u64 key, bool is_written); - void InvalidateMultiRange(u64 key) { - multi_range_buffers.Invalidate(key); - } - void OnBufferDeleted(const Buffer& buffer) { multi_range_buffers.DropOwner(scheduler, buffer.Handle()); } diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp index 6d1ae5fc53..68dcab9250 100644 --- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp +++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp @@ -87,7 +87,7 @@ u64 MultiRangeBufferCache::HashSources(std::span sources u64 MultiRangeBufferCache::HashContent(std::span sources) const { u64 hash = 0xcbf29ce484222325ULL; for (const MultiRangeSource& source : sources) { - hash ^= source.write_tick; + hash ^= source.content_serial; hash *= 0x100000001b3ULL; } return hash; @@ -219,9 +219,6 @@ MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& schedu if (sources.empty() || total == 0) { return MultiRangeRef{}; } - if (!retired.empty()) { - DrainRetired(scheduler); - } const u64 geometry = HashSources(sources); const u64 content = HashContent(sources); const auto it = entries.find(key); @@ -335,10 +332,4 @@ void MultiRangeBufferCache::DropOwner(Scheduler& scheduler, VkBuffer owner) { } } -void MultiRangeBufferCache::Invalidate(u64 key) { - if (auto const it = entries.find(key); it != entries.end()) { - it->second.dirty = true; - } -} - } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h index f40d3aa1f8..d9c065df3f 100644 --- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h +++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h @@ -27,7 +27,7 @@ struct MultiRangeSource { VkDeviceSize memory_offset{}; VkDeviceSize offset{}; VkDeviceSize size{}; - u64 write_tick{}; + u64 content_serial{}; u32 memory_type{}; }; @@ -55,10 +55,10 @@ public: void MarkGathered(u64 key); - void Invalidate(u64 key); - void DropOwner(Scheduler& scheduler, VkBuffer owner); + void DrainRetired(Scheduler& scheduler); + VkDeviceSize block_size{DEFAULT_BLOCK_SIZE}; bool use_sparse{}; @@ -94,8 +94,6 @@ private: void RetireEntry(Scheduler& scheduler, Entry& entry); - void DrainRetired(Scheduler& scheduler); - ::Common::unordered_map entries; boost::container::static_vector retired; u32 sparse_memory_type_bits{}; diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index 7bdb5738c8..e14629dfc8 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -1654,6 +1654,7 @@ void QueryCacheRuntime::SyncValues(std::span values, VkBuffer ba const auto post_op = VideoCommon::ObtainBufferOperation::DoNothing; const auto [buffer, offset] = impl->buffer_cache.ObtainCPUBuffer( pair.first, static_cast(pair.second - pair.first), sync_info, post_op); + buffer->MarkContentModified(); impl->buffers_to_upload_to.emplace_back(buffer->Handle(), offset); } });