Coalescing memory entries on MLP

src/video_core/memory_manager.cpp

@@ -370,8 +370,32 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si
 template <bool is_safe>
 void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size,
                                   [[maybe_unused]] VideoCommon::CacheType which) const {
+    const u8* run_src{nullptr};
+    u8* run_dst{nullptr};
+    std::size_t run_size{0};
+    auto flush_run = [&] {
+        if (run_size == 0) {
+            return;
+        }
+        std::memcpy(run_dst, run_src, run_size);
+        run_src = nullptr;
+        run_dst = nullptr;
+        run_size = 0;
+    };
+    auto append_run = [&](const u8* physical, std::size_t copy_amount) {
+        if (run_size != 0 && run_src + run_size == physical &&
+            run_dst + run_size == static_cast<u8*>(dest_buffer)) {
+            run_size += copy_amount;
+            return;
+        }
+        flush_run();
+        run_src = physical;
+        run_dst = static_cast<u8*>(dest_buffer);
+        run_size = copy_amount;
+    };
     auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index,
                            [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
+        flush_run();
         std::memset(dest_buffer, 0, copy_amount);
         dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
     };
@@ -381,8 +405,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std:
         if constexpr (is_safe) {
             rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
         }
-        u8* physical = memory.GetPointer<u8>(dev_addr_base);
+        append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
-        std::memcpy(dest_buffer, physical, copy_amount);
         dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
     };
     auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
@@ -392,10 +415,10 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std:
             rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
         }
         if (!IsBigPageContinuous(page_index)) [[unlikely]] {
+            flush_run();
             memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount);
         } else {
-            u8* physical = memory.GetPointer<u8>(dev_addr_base);
+            append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
-            std::memcpy(dest_buffer, physical, copy_amount);
         }
         dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
     };
@@ -405,6 +428,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std:
         MemoryOperation<false>(base, copy_amount, mapped_normal, set_to_zero, set_to_zero);
     };
     MemoryOperation<true>(gpu_src_addr, size, mapped_big, set_to_zero, read_short_pages);
+    flush_run();
 }
 
 void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size,
@@ -420,8 +444,32 @@ void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer,
 template <bool is_safe>
 void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size,
                                    [[maybe_unused]] VideoCommon::CacheType which) {
+    const u8* run_src{nullptr};
+    u8* run_dst{nullptr};
+    std::size_t run_size{0};
+    auto flush_run = [&] {
+        if (run_size == 0) {
+            return;
+        }
+        std::memcpy(run_dst, run_src, run_size);
+        run_src = nullptr;
+        run_dst = nullptr;
+        run_size = 0;
+    };
+    auto append_run = [&](u8* physical, std::size_t copy_amount) {
+        if (run_size != 0 && run_dst + run_size == physical &&
+            run_src + run_size == static_cast<const u8*>(src_buffer)) {
+            run_size += copy_amount;
+            return;
+        }
+        flush_run();
+        run_src = static_cast<const u8*>(src_buffer);
+        run_dst = physical;
+        run_size = copy_amount;
+    };
     auto just_advance = [&]([[maybe_unused]] std::size_t page_index,
                             [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
+        flush_run();
         src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
     };
     auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
@@ -430,8 +478,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe
         if constexpr (is_safe) {
             rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
         }
-        u8* physical = memory.GetPointer<u8>(dev_addr_base);
+        append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
-        std::memcpy(physical, src_buffer, copy_amount);
         src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
     };
     auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
@@ -441,10 +488,10 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe
             rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
         }
         if (!IsBigPageContinuous(page_index)) [[unlikely]] {
+            flush_run();
             memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount);
         } else {
-            u8* physical = memory.GetPointer<u8>(dev_addr_base);
+            append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
-            std::memcpy(physical, src_buffer, copy_amount);
         }
         src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
     };
@@ -454,6 +501,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe
         MemoryOperation<false>(base, copy_amount, mapped_normal, just_advance, just_advance);
     };
     MemoryOperation<true>(gpu_dest_addr, size, mapped_big, just_advance, write_short_pages);
+    flush_run();
 }
 
 void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size,
@@ -605,7 +653,7 @@ bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const {
     if (GetEntry<true>(gpu_addr) == EntryType::Mapped) [[likely]] {
         size_t page_index = gpu_addr >> big_page_bits;
         if (IsBigPageContinuous(page_index)) [[likely]] {
-            const std::size_t page{(page_index & big_page_mask) + size};
+            const std::size_t page{(gpu_addr & big_page_mask) + size};
             return page <= big_page_size;
         }
         const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size};

src/video_core/renderer_vulkan/vk_compute_pass.cpp

@@ -1248,11 +1248,6 @@ constexpr DescriptorBankInfo BL3DB_BANK_INFO{
 };
 
 constexpr bool BL3DB_VERIFY_AGAINST_CPU = false;
-
-constexpr u32 MipSize(u32 size, u32 level) {
-    const u32 shifted = size >> level;
-    return shifted != 0 ? shifted : 1u;
-}
 } // Anonymous namespace
 
 BlockLinearUnswizzle3DBufferPass::BlockLinearUnswizzle3DBufferPass(
@@ -1273,7 +1268,7 @@ bool BlockLinearUnswizzle3DBufferPass::IsSupported(const Device& device,
     if (info.type != VideoCommon::ImageType::e3D) {
         return false;
     }
-    if (info.resources.layers != 1) {
+    if (info.resources.levels != 1 || info.resources.layers != 1) {
         return false;
     }
     if (info.num_samples > 1) {
@@ -1299,107 +1294,78 @@ void BlockLinearUnswizzle3DBufferPass::Unswizzle(
         return;
     }
 
+    const VideoCommon::SwizzleParameters& sw = swizzles.front();
+    const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info);
+
+    const u32 blocks_x = sw.num_tiles.width;
+    const u32 blocks_y = sw.num_tiles.height;
+    const u32 blocks_z = sw.num_tiles.depth;
+    const u32 bytes_per_block = 1u << params.bytes_per_block_log2;
+    const VkDeviceSize output_size =
+        static_cast<VkDeviceSize>(blocks_x) * blocks_y * blocks_z * bytes_per_block;
+
+    const StagingBufferRef output =
+        staging_buffer_pool.Request(static_cast<size_t>(output_size), MemoryUsage::DeviceLocal);
+
+    BlockLinearUnswizzle3DBufferPushConstants pc{};
+    pc.dim = {blocks_x, blocks_y, blocks_z};
+    pc.bytes_per_block_log2 = params.bytes_per_block_log2;
+    pc.origin = params.origin;
+    pc.slice_size = params.slice_size;
+    pc.block_size = params.block_size;
+    pc.x_shift = params.x_shift;
+    pc.block_height = params.block_height;
+    pc.block_height_mask = params.block_height_mask;
+    pc.block_depth = params.block_depth;
+    pc.block_depth_mask = params.block_depth_mask;
+
     scheduler.RequestOutsideRenderPassOperationContext();
 
+    compute_pass_descriptor_queue.Acquire(scheduler, 2);
+    compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, sw.buffer_offset + swizzled.offset,
+                                            image.guest_size_bytes - sw.buffer_offset);
+    compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size);
+
+    const void* descriptor_data = compute_pass_descriptor_queue.UpdateData();
+    const VkDescriptorSet set = descriptor_allocator.Commit();
+
+    const u32 gx = Common::DivCeil(blocks_x, 8u);
+    const u32 gy = Common::DivCeil(blocks_y, 8u);
+    const u32 gz = Common::DivCeil(blocks_z, 4u);
     const bool is_initialized = image.ExchangeInitialization();
+
+    const VkBuffer out_buffer = output.buffer;
+    const VkDeviceSize out_offset = output.offset;
     const VkImage dst_image = image.Handle();
     const VkImageAspectFlags aspect = image.AspectMask();
-
+    const VkExtent3D extent{
-    struct LevelCopy {
+        .width = image.info.size.width,
-        VkBuffer buffer;
+        .height = image.info.size.height,
-        VkBufferImageCopy copy;
+        .depth = image.info.size.depth,
     };
-    std::vector<LevelCopy> level_copies;
-    level_copies.reserve(swizzles.size());
 
-    for (const VideoCommon::SwizzleParameters& sw : swizzles) {
+    scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz, output_size, out_buffer,
-        const auto params =
+                      out_offset, dst_image, aspect, extent,
-            VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info);
-
-        const u32 blocks_x = sw.num_tiles.width;
-        const u32 blocks_y = sw.num_tiles.height;
-        const u32 blocks_z = sw.num_tiles.depth;
-        const u32 bytes_per_block = 1u << params.bytes_per_block_log2;
-        const VkDeviceSize output_size =
-            static_cast<VkDeviceSize>(blocks_x) * blocks_y * blocks_z * bytes_per_block;
-        if (output_size == 0) {
-            continue;
-        }
-
-        const StagingBufferRef output = staging_buffer_pool.Request(
-            static_cast<size_t>(output_size), MemoryUsage::DeviceLocal);
-
-        BlockLinearUnswizzle3DBufferPushConstants pc{};
-        pc.dim = {blocks_x, blocks_y, blocks_z};
-        pc.bytes_per_block_log2 = params.bytes_per_block_log2;
-        pc.origin = params.origin;
-        pc.slice_size = params.slice_size;
-        pc.block_size = params.block_size;
-        pc.x_shift = params.x_shift;
-        pc.block_height = params.block_height;
-        pc.block_height_mask = params.block_height_mask;
-        pc.block_depth = params.block_depth;
-        pc.block_depth_mask = params.block_depth_mask;
-
-        compute_pass_descriptor_queue.Acquire(scheduler, 2);
-        compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, sw.buffer_offset + swizzled.offset,
-                                                image.guest_size_bytes - sw.buffer_offset);
-        compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size);
-
-        const void* descriptor_data = compute_pass_descriptor_queue.UpdateData();
-        const VkDescriptorSet set = descriptor_allocator.Commit();
-
-        const u32 gx = Common::DivCeil(blocks_x, 8u);
-        const u32 gy = Common::DivCeil(blocks_y, 8u);
-        const u32 gz = Common::DivCeil(blocks_z, 4u);
-
-        scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz](vk::CommandBuffer cmdbuf) {
-            device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data);
-            cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline);
-            cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {});
-            cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
-            cmdbuf.Dispatch(gx, gy, gz);
-        });
-
-        const u32 level = static_cast<u32>(sw.level);
-        level_copies.push_back(LevelCopy{
-            .buffer = output.buffer,
-            .copy{
-                .bufferOffset = output.offset,
-                .bufferRowLength = 0,
-                .bufferImageHeight = 0,
-                .imageSubresource{
-                    .aspectMask = aspect,
-                    .mipLevel = level,
-                    .baseArrayLayer = 0,
-                    .layerCount = 1,
-                },
-                .imageOffset = {0, 0, 0},
-                .imageExtent{
-                    .width = MipSize(image.info.size.width, level),
-                    .height = MipSize(image.info.size.height, level),
-                    .depth = MipSize(image.info.size.depth, level),
-                },
-            },
-        });
-
-        if constexpr (BL3DB_VERIFY_AGAINST_CPU) {
-            VerifyAgainstCpu(swizzled, sw, image.info, output, output_size, blocks_x, blocks_y,
-                             blocks_z, bytes_per_block);
-        }
-    }
-
-    if (level_copies.empty() || dst_image == VK_NULL_HANDLE) {
-        return;
-    }
-
-    scheduler.Record([copies = std::move(level_copies), dst_image, aspect,
                       is_initialized](vk::CommandBuffer cmdbuf) {
-        const VkMemoryBarrier memory_barrier{
+        if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) {
-            .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+            return;
+        }
+        device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data);
+        cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline);
+        cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {});
+        cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
+        cmdbuf.Dispatch(gx, gy, gz);
+
+        const VkBufferMemoryBarrier buffer_barrier{
+            .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
             .pNext = nullptr,
             .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
             .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
+            .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+            .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+            .buffer = out_buffer,
+            .offset = out_offset,
+            .size = output_size,
         };
         const VkImageMemoryBarrier pre_copy{
             .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
@@ -1423,12 +1389,22 @@ void BlockLinearUnswizzle3DBufferPass::Unswizzle(
         cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |
                                    (is_initialized ? VK_PIPELINE_STAGE_ALL_COMMANDS_BIT
                                                    : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT),
-                               VK_PIPELINE_STAGE_TRANSFER_BIT, 0, memory_barrier, {}, pre_copy);
+                               VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, buffer_barrier, pre_copy);
 
-        for (const LevelCopy& level_copy : copies) {
+        const VkBufferImageCopy copy{
-            cmdbuf.CopyBufferToImage(level_copy.buffer, dst_image,
+            .bufferOffset = out_offset,
-                                     VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, level_copy.copy);
+            .bufferRowLength = 0,
-        }
+            .bufferImageHeight = 0,
+            .imageSubresource{
+                .aspectMask = aspect,
+                .mipLevel = 0,
+                .baseArrayLayer = 0,
+                .layerCount = 1,
+            },
+            .imageOffset = {0, 0, 0},
+            .imageExtent = extent,
+        };
+        cmdbuf.CopyBufferToImage(out_buffer, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy);
 
         const VkImageMemoryBarrier post_copy{
             .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
@@ -1451,6 +1427,11 @@ void BlockLinearUnswizzle3DBufferPass::Unswizzle(
         cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
                                0, {}, {}, post_copy);
     });
+
+    if constexpr (BL3DB_VERIFY_AGAINST_CPU) {
+        VerifyAgainstCpu(swizzled, sw, image.info, output, output_size, blocks_x, blocks_y,
+                         blocks_z, bytes_per_block);
+    }
 }
 
 void BlockLinearUnswizzle3DBufferPass::VerifyAgainstCpu(