another take

src/video_core/buffer_cache/buffer_cache.h

@@ -714,6 +714,7 @@ void BufferCache<P>::CommitAsyncFlushesHigh() {
             runtime.CopyToUnifiedMemory(queued.window, slot_buffers[queued.buffer_id], group_span);
         }
         if (!unified_copy_queue.empty()) {
+            runtime.FlushUnifiedMemoryCopies();
             runtime.UnifiedMemoryHostBarrier();
         }
     }
@@ -1861,6 +1862,7 @@ bool BufferCache<P>::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer,
             const std::span<const BufferCopy> group_span(groups[i].data(), groups[i].size());
             runtime.CopyToUnifiedMemory(window_ids[i], buffer, group_span);
         }
+        runtime.FlushUnifiedMemoryCopies();
         runtime.UnifiedMemoryHostBarrier();
         runtime.Finish();
         return true;
@@ -1869,11 +1871,57 @@ bool BufferCache<P>::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer,
     }
 }
 
+template <class P>
+bool BufferCache<P>::TryUnifiedUploadMemory([[maybe_unused]] Buffer& buffer,
+                                            [[maybe_unused]] std::span<const BufferCopy> copies) {
+    if constexpr (USE_UNIFIED_MEMORY) {
+        boost::container::small_vector<u64, 4> window_ids;
+        UnifiedWindowGroups groups;
+        for (const BufferCopy& copy : copies) {
+            if (!ResolveUnifiedWindows(buffer.CpuAddr() + copy.dst_offset, copy.dst_offset,
+                                       copy.size, window_ids, groups)) {
+                return false;
+            }
+        }
+        for (const BufferCopy& copy : copies) {
+            buffer.MarkUsage(copy.dst_offset, copy.size);
+        }
+        runtime.PreCopyBarrier();
+        boost::container::small_vector<BufferCopy, 16> window_source_copies;
+        for (size_t i = 0; i < window_ids.size(); ++i) {
+            window_source_copies.clear();
+            window_source_copies.reserve(groups[i].size());
+            for (const BufferCopy& copy : groups[i]) {
+                window_source_copies.push_back(BufferCopy{
+                    .src_offset = copy.dst_offset,
+                    .dst_offset = copy.src_offset,
+                    .size = copy.size,
+                });
+            }
+            const std::span<const BufferCopy> group_span(window_source_copies.data(),
+                                                         window_source_copies.size());
+            runtime.CopyFromUnifiedMemory(window_ids[i], buffer, group_span);
+        }
+        runtime.FlushUnifiedMemoryCopies();
+        runtime.PostCopyBarrier();
+        return true;
+    } else {
+        return false;
+    }
+}
+
 template <class P>
 void BufferCache<P>::MappedUploadMemory([[maybe_unused]] Buffer& buffer,
                                         [[maybe_unused]] u64 total_size_bytes,
                                         [[maybe_unused]] std::span<BufferCopy> copies) {
     if constexpr (USE_MEMORY_MAPS) {
+        if constexpr (USE_UNIFIED_MEMORY) {
+            if (runtime.HasUnifiedMemory() &&
+                !Settings::values.enable_gpu_buffer_readback.GetValue() &&
+                TryUnifiedUploadMemory(buffer, copies)) {
+                return;
+            }
+        }
         auto upload_staging = runtime.UploadStagingBuffer(total_size_bytes);
         const std::span<u8> staging_pointer = upload_staging.mapped_span;
         for (BufferCopy& copy : copies) {

src/video_core/buffer_cache/buffer_cache_base.h

@@ -446,6 +446,8 @@ private:
 
     bool TryUnifiedDownloadMemory(Buffer& buffer, std::span<BufferCopy> copies);
 
+    bool TryUnifiedUploadMemory(Buffer& buffer, std::span<const BufferCopy> copies);
+
     using UnifiedWindowGroups =
         boost::container::small_vector<boost::container::small_vector<BufferCopy, 16>, 4>;
 

src/video_core/renderer_vulkan/vk_buffer_cache.cpp

@@ -8,6 +8,7 @@
 #include <array>
 #include <cstring>
 #include <span>
+#include <utility>
 #include <vector>
 
 #include "video_core/buffer_cache/buffer_cache_base.h"
@@ -32,6 +33,32 @@ VkBufferCopy MakeBufferCopy(const VideoCommon::BufferCopy& copy) {
     };
 }
 
+constexpr size_t MAX_WINDOW_BARRIER_RANGES = 8;
+
+using WindowRange = std::pair<VkDeviceSize, VkDeviceSize>;
+using WindowRanges = boost::container::small_vector<WindowRange, MAX_WINDOW_BARRIER_RANGES>;
+
+void CoalesceWindowRanges(WindowRanges& ranges) {
+    if (ranges.size() < 2) {
+        return;
+    }
+    std::sort(ranges.begin(), ranges.end());
+    size_t merged = 0;
+    for (size_t index = 1; index < ranges.size(); ++index) {
+        if (ranges[index].first <= ranges[merged].second) {
+            ranges[merged].second = (std::max)(ranges[merged].second, ranges[index].second);
+        } else {
+            ranges[++merged] = ranges[index];
+        }
+    }
+    ranges.resize(merged + 1);
+    if (ranges.size() > MAX_WINDOW_BARRIER_RANGES) {
+        const WindowRange bounding{ranges.front().first, ranges.back().second};
+        ranges.clear();
+        ranges.push_back(bounding);
+    }
+}
+
 VkIndexType IndexTypeFromNumElements(const Device& device, u32 num_elements) {
     if (num_elements <= 0xff && device.IsExtIndexTypeUint8Supported()) {
         return VK_INDEX_TYPE_UINT8_EXT;
@@ -376,66 +403,164 @@ void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size,
     }
 }
 
+void BufferCacheRuntime::QueueUnifiedCopy(size_t window_index, VkBuffer buffer,
+                                          std::span<const VideoCommon::BufferCopy> copies,
+                                          bool reads_window) {
+    if (!unified_memory || buffer == VK_NULL_HANDLE || copies.empty() ||
+        window_index >= unified_memory->GetWindowCount() ||
+        unified_memory->GetWindowBuffer(window_index) == VK_NULL_HANDLE) {
+        return;
+    }
+    PendingUnifiedCopy& pending = pending_unified_copies.emplace_back();
+    pending.window = window_index;
+    pending.buffer = buffer;
+    pending.reads_window = reads_window;
+    pending.copies.resize(copies.size());
+    std::ranges::transform(copies, pending.copies.begin(), MakeBufferCopy);
+}
+
 void BufferCacheRuntime::CopyToUnifiedMemory(
     size_t window_index, VkBuffer src_buffer,
     std::span<const VideoCommon::BufferCopy> copies) {
-    if (!unified_memory || src_buffer == VK_NULL_HANDLE || copies.empty() ||
-        window_index >= unified_memory->GetWindowCount()) {
-        return;
-    }
-    const VkBuffer dst_buffer = unified_memory->GetWindowBuffer(window_index);
-    if (dst_buffer == VK_NULL_HANDLE) {
-        return;
-    }
-    VkDeviceSize covered_begin = std::numeric_limits<VkDeviceSize>::max();
-    VkDeviceSize covered_end = 0;
-    for (const VideoCommon::BufferCopy& copy : copies) {
-        covered_begin = (std::min)(covered_begin, static_cast<VkDeviceSize>(copy.dst_offset));
-        covered_end = (std::max)(covered_end,
-                                 static_cast<VkDeviceSize>(copy.dst_offset + copy.size));
-    }
+    QueueUnifiedCopy(window_index, src_buffer, copies, false);
+}
 
-    boost::container::small_vector<VkBufferCopy, 8> vk_copies(copies.size());
-    std::ranges::transform(copies, vk_copies.begin(), MakeBufferCopy);
+void BufferCacheRuntime::CopyFromUnifiedMemory(
+    size_t window_index, VkBuffer dst_buffer,
+    std::span<const VideoCommon::BufferCopy> copies) {
+    QueueUnifiedCopy(window_index, dst_buffer, copies, true);
+}
+
+void BufferCacheRuntime::FlushUnifiedMemoryCopies() {
+    if (pending_unified_copies.empty()) {
+        return;
+    }
+    struct UnifiedCopyCommand {
+        VkBuffer buffer;
+        bool reads_window;
+        boost::container::small_vector<VkBufferCopy, 8> copies;
+    };
+    static constexpr VkMemoryBarrier WINDOW_TRANSFER_BARRIER{
+        .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+        .pNext = nullptr,
+        .srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
+        .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
+    };
+
+    std::stable_sort(pending_unified_copies.begin(), pending_unified_copies.end(),
+                     [](const PendingUnifiedCopy& lhs, const PendingUnifiedCopy& rhs) {
+                         if (lhs.window != rhs.window) {
+                             return lhs.window < rhs.window;
+                         }
+                         return lhs.reads_window && !rhs.reads_window;
+                     });
 
     const bool foreign = unified_memory->NeedsForeignOwnershipTransfer();
     const u32 queue_family = device.GetGraphicsFamily();
 
-    scheduler.RequestOutsideRenderPassOperationContext();
-    scheduler.Record([src_buffer, dst_buffer, vk_copies, foreign, queue_family, covered_begin,
-                      covered_end](vk::CommandBuffer cmdbuf) {
-        if (foreign) {
-            const VkBufferMemoryBarrier acquire{
-                .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
-                .pNext = nullptr,
-                .srcAccessMask = 0,
-                .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
-                .srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT,
-                .dstQueueFamilyIndex = queue_family,
-                .buffer = dst_buffer,
-                .offset = covered_begin,
-                .size = covered_end - covered_begin,
-            };
-            cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
-                                   VK_PIPELINE_STAGE_TRANSFER_BIT, 0, acquire);
+    size_t group_begin = 0;
+    while (group_begin < pending_unified_copies.size()) {
+        const size_t window = pending_unified_copies[group_begin].window;
+        size_t group_end = group_begin;
+        while (group_end < pending_unified_copies.size() &&
+               pending_unified_copies[group_end].window == window) {
+            ++group_end;
         }
-        cmdbuf.CopyBuffer(src_buffer, dst_buffer, VideoCommon::FixSmallVectorADL(vk_copies));
-        if (foreign) {
-            const VkBufferMemoryBarrier release{
-                .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
-                .pNext = nullptr,
-                .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
-                .dstAccessMask = 0,
-                .srcQueueFamilyIndex = queue_family,
-                .dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT,
-                .buffer = dst_buffer,
-                .offset = covered_begin,
-                .size = covered_end - covered_begin,
-            };
-            cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT,
-                                   VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, release);
+        const VkBuffer window_buffer = unified_memory->GetWindowBuffer(window);
+
+        WindowRanges ranges;
+        VkAccessFlags window_access = 0;
+        size_t reads_in_group = 0;
+        for (size_t index = group_begin; index < group_end; ++index) {
+            const PendingUnifiedCopy& pending = pending_unified_copies[index];
+            if (pending.reads_window) {
+                window_access |= VK_ACCESS_TRANSFER_READ_BIT;
+                ++reads_in_group;
+            } else {
+                window_access |= VK_ACCESS_TRANSFER_WRITE_BIT;
+            }
+            for (const VkBufferCopy& copy : pending.copies) {
+                const VkDeviceSize offset =
+                    pending.reads_window ? copy.srcOffset : copy.dstOffset;
+                ranges.emplace_back(offset, offset + copy.size);
+            }
         }
-    });
+        CoalesceWindowRanges(ranges);
+
+        boost::container::small_vector<VkBufferMemoryBarrier, MAX_WINDOW_BARRIER_RANGES> acquire;
+        boost::container::small_vector<VkBufferMemoryBarrier, MAX_WINDOW_BARRIER_RANGES> release;
+        if (foreign) {
+            for (const WindowRange& range : ranges) {
+                acquire.push_back(VkBufferMemoryBarrier{
+                    .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
+                    .pNext = nullptr,
+                    .srcAccessMask = 0,
+                    .dstAccessMask = window_access,
+                    .srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT,
+                    .dstQueueFamilyIndex = queue_family,
+                    .buffer = window_buffer,
+                    .offset = range.first,
+                    .size = range.second - range.first,
+                });
+                release.push_back(VkBufferMemoryBarrier{
+                    .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
+                    .pNext = nullptr,
+                    .srcAccessMask = window_access,
+                    .dstAccessMask = 0,
+                    .srcQueueFamilyIndex = queue_family,
+                    .dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT,
+                    .buffer = window_buffer,
+                    .offset = range.first,
+                    .size = range.second - range.first,
+                });
+            }
+        }
+
+        boost::container::small_vector<UnifiedCopyCommand, 4> commands;
+        commands.reserve(group_end - group_begin);
+        for (size_t index = group_begin; index < group_end; ++index) {
+            PendingUnifiedCopy& pending = pending_unified_copies[index];
+            commands.push_back(UnifiedCopyCommand{pending.buffer, pending.reads_window,
+                                                  std::move(pending.copies)});
+        }
+        const size_t barrier_before_writes = (reads_in_group > 0 && reads_in_group < commands.size())
+                                                 ? reads_in_group
+                                                 : commands.size();
+
+        scheduler.RequestOutsideRenderPassOperationContext();
+        scheduler.Record([window_buffer, barrier_before_writes, acquire = std::move(acquire),
+                          release = std::move(release),
+                          commands = std::move(commands)](vk::CommandBuffer cmdbuf) {
+            if (!acquire.empty()) {
+                cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
+                                       VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {},
+                                       VideoCommon::FixSmallVectorADL(acquire), {});
+            }
+            for (size_t index = 0; index < commands.size(); ++index) {
+                if (index == barrier_before_writes) {
+                    cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT,
+                                           VK_PIPELINE_STAGE_TRANSFER_BIT, 0,
+                                           WINDOW_TRANSFER_BARRIER);
+                }
+                const UnifiedCopyCommand& command = commands[index];
+                if (command.reads_window) {
+                    cmdbuf.CopyBuffer(window_buffer, command.buffer,
+                                      VideoCommon::FixSmallVectorADL(command.copies));
+                } else {
+                    cmdbuf.CopyBuffer(command.buffer, window_buffer,
+                                      VideoCommon::FixSmallVectorADL(command.copies));
+                }
+            }
+            if (!release.empty()) {
+                cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT,
+                                       VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, {},
+                                       VideoCommon::FixSmallVectorADL(release), {});
+            }
+        });
+
+        group_begin = group_end;
+    }
+    pending_unified_copies.clear();
 }
 
 void BufferCacheRuntime::UnifiedMemoryHostBarrier() {
@@ -489,6 +614,7 @@ u32 BufferCacheRuntime::GetStorageBufferAlignment() const {
 }
 
 void BufferCacheRuntime::TickFrame(Common::SlotVector<Buffer>& slot_buffers) noexcept {
+    FlushUnifiedMemoryCopies();
     for (auto it = slot_buffers.begin(); it != slot_buffers.end(); it++) {
         if (scheduler.IsFree(it->LastUsageTick())) {
             it->ResetUsageTracking();

src/video_core/renderer_vulkan/vk_buffer_cache.h

@@ -10,6 +10,8 @@
 #include <memory>
 #include <span>
 
+#include <boost/container/small_vector.hpp>
+
 #include "video_core/buffer_cache/buffer_cache_base.h"
 #include "video_core/buffer_cache/memory_tracker_base.h"
 #include "video_core/buffer_cache/usage_tracker.h"
@@ -122,6 +124,11 @@ public:
     void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer,
                              std::span<const VideoCommon::BufferCopy> copies);
 
+    void CopyFromUnifiedMemory(size_t window_index, VkBuffer dst_buffer,
+                               std::span<const VideoCommon::BufferCopy> copies);
+
+    void FlushUnifiedMemoryCopies();
+
     void UnifiedMemoryHostBarrier();
 
     u64 CurrentTick();
@@ -207,6 +214,16 @@ public:
     }
 
 private:
+    struct PendingUnifiedCopy {
+        size_t window;
+        VkBuffer buffer;
+        bool reads_window;
+        boost::container::small_vector<VkBufferCopy, 8> copies;
+    };
+
+    void QueueUnifiedCopy(size_t window_index, VkBuffer buffer,
+                          std::span<const VideoCommon::BufferCopy> copies, bool reads_window);
+
     void BindBuffer(const Buffer& buffer, u32 offset, u32 size) {
         const VkBuffer handle = buffer.Handle();
         if (handle == VK_NULL_HANDLE) {
@@ -232,6 +249,7 @@ private:
 
     vk::Buffer null_buffer;
     std::unique_ptr<HostMemoryImport> unified_memory;
+    boost::container::small_vector<PendingUnifiedCopy, 8> pending_unified_copies;
 
     std::unique_ptr<Uint8Pass> uint8_pass;
     QuadIndexedPass quad_index_pass;