Added vertex, uniform and indexes on the upload path

src/video_core/buffer_cache/buffer_cache.h

@@ -821,6 +821,20 @@ void BufferCache<P>::BindHostIndexBuffer() {
     const u32 size = channel_state->index_buffer.size;
     const auto& draw_state = maxwell3d->draw_manager.draw_state;
     if (draw_state.inline_index_draw_indexes.empty()) {
+        if constexpr (USE_UNIFIED_MEMORY && !HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT) {
+            const auto window =
+                TryResolveUnifiedRange(channel_state->index_buffer.device_addr, size);
+            if (window && runtime.IsUnifiedIndexRange(draw_state.topology,
+                                                     draw_state.index_buffer.format,
+                                                     window->offset)) {
+                runtime.BindIndexBuffer(draw_state.topology, draw_state.index_buffer.format,
+                                        draw_state.index_buffer.first,
+                                        draw_state.index_buffer.count,
+                                        runtime.UnifiedWindowBuffer(window->window),
+                                        static_cast<u32>(window->offset), size);
+                return;
+            }
+        }
         SynchronizeBuffer(buffer, channel_state->index_buffer.device_addr, size);
     } else {
         if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) {
@@ -874,6 +888,7 @@ void BufferCache<P>::UpdateVertexBufferSlot(u32 index, const Binding& binding) {
         enabled_vertex_buffers_mask |= (1u << index);
     } else {
         enabled_vertex_buffers_mask &= ~(1u << index);
+        unified_vertex_buffers_mask &= ~(1u << index);
     }
 }
 
@@ -906,15 +921,39 @@ void BufferCache<P>::BindHostVertexBuffers() {
             const Binding& binding = VertexBufferSlot(index);
             Buffer& buffer = slot_buffers[binding.buffer_id];
             TouchBuffer(buffer, binding.buffer_id);
-            SynchronizeBuffer(buffer, binding.device_addr, binding.size);
+            bool needs_bind = flags[Dirty::VertexBuffer0 + index];
-            if (!flags[Dirty::VertexBuffer0 + index]) {
+            u32 unified_window = NO_UNIFIED_WINDOW;
+            u64 unified_offset = 0;
+            if constexpr (USE_UNIFIED_MEMORY) {
+                const bool was_unified = ((unified_vertex_buffers_mask >> index) & 1) != 0;
+                if (needs_bind || was_unified) {
+                    const auto window = TryResolveUnifiedRange(binding.device_addr, binding.size);
+                    if (window) {
+                        unified_window = static_cast<u32>(window->window);
+                        unified_offset = window->offset;
+                    }
+                }
+                if (was_unified && unified_window == NO_UNIFIED_WINDOW) {
+                    needs_bind = true;
+                }
+            }
+            if (unified_window == NO_UNIFIED_WINDOW) {
+                SynchronizeBuffer(buffer, binding.device_addr, binding.size);
+            }
+            if (!needs_bind) {
                 flush_bindings();
                 continue;
             }
             flags[Dirty::VertexBuffer0 + index] = false;
             const u32 stride = maxwell3d->regs.vertex_streams[index].stride;
-            const u32 offset = buffer.Offset(binding.device_addr);
+            u32 offset = static_cast<u32>(unified_offset);
-            buffer.MarkUsage(offset, binding.size);
+            if (unified_window == NO_UNIFIED_WINDOW) {
+                offset = buffer.Offset(binding.device_addr);
+                buffer.MarkUsage(offset, binding.size);
+                unified_vertex_buffers_mask &= ~(1u << index);
+            } else {
+                unified_vertex_buffers_mask |= 1u << index;
+            }
             if (!bindings.buffers.empty() && index != last_index + 1) {
                 flush_bindings();
             }
@@ -925,6 +964,7 @@ void BufferCache<P>::BindHostVertexBuffers() {
             bindings.offsets.push_back(offset);
             bindings.sizes.push_back(binding.size);
             bindings.strides.push_back(stride);
+            bindings.unified_windows.push_back(unified_window);
             last_index = index;
         }
         flush_bindings();
@@ -932,12 +972,31 @@ void BufferCache<P>::BindHostVertexBuffers() {
         HostBindings<typename P::Buffer> host_bindings;
         bool any_valid{false};
         auto& flags = maxwell3d->dirty.flags;
+        std::array<u32, NUM_VERTEX_BUFFERS> unified_windows;
+        std::array<u64, NUM_VERTEX_BUFFERS> unified_offsets{};
+        unified_windows.fill(NO_UNIFIED_WINDOW);
         for (u32 index = 0; index < NUM_VERTEX_BUFFERS; ++index) {
             const Binding& binding = channel_state->vertex_buffers[index];
             Buffer& buffer = slot_buffers[binding.buffer_id];
             TouchBuffer(buffer, binding.buffer_id);
-            SynchronizeBuffer(buffer, binding.device_addr, binding.size);
+            bool needs_bind = flags[Dirty::VertexBuffer0 + index];
-            if (!flags[Dirty::VertexBuffer0 + index]) {
+            if constexpr (USE_UNIFIED_MEMORY) {
+                const bool was_unified = ((unified_vertex_buffers_mask >> index) & 1) != 0;
+                if (needs_bind || was_unified) {
+                    const auto window = TryResolveUnifiedRange(binding.device_addr, binding.size);
+                    if (window) {
+                        unified_windows[index] = static_cast<u32>(window->window);
+                        unified_offsets[index] = window->offset;
+                    }
+                }
+                if (was_unified && unified_windows[index] == NO_UNIFIED_WINDOW) {
+                    needs_bind = true;
+                }
+            }
+            if (unified_windows[index] == NO_UNIFIED_WINDOW) {
+                SynchronizeBuffer(buffer, binding.device_addr, binding.size);
+            }
+            if (!needs_bind) {
                 continue;
             }
             flags[Dirty::VertexBuffer0 + index] = false;
@@ -956,13 +1015,20 @@ void BufferCache<P>::BindHostVertexBuffers() {
                 Buffer& buffer = slot_buffers[binding.buffer_id];
 
                 const u32 stride = maxwell3d->regs.vertex_streams[index].stride;
-                const u32 offset = buffer.Offset(binding.device_addr);
+                u32 offset = static_cast<u32>(unified_offsets[index]);
-                buffer.MarkUsage(offset, binding.size);
+                if (unified_windows[index] == NO_UNIFIED_WINDOW) {
+                    offset = buffer.Offset(binding.device_addr);
+                    buffer.MarkUsage(offset, binding.size);
+                    unified_vertex_buffers_mask &= ~(1u << index);
+                } else {
+                    unified_vertex_buffers_mask |= 1u << index;
+                }
 
                 host_bindings.buffers.push_back(&buffer);
                 host_bindings.offsets.push_back(offset);
                 host_bindings.sizes.push_back(binding.size);
                 host_bindings.strides.push_back(stride);
+                host_bindings.unified_windows.push_back(unified_windows[index]);
             }
             runtime.BindVertexBuffers(host_bindings);
         }
@@ -1006,6 +1072,17 @@ void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32
     const u32 size = (std::min)(binding.size, (*channel_state->uniform_buffer_sizes)[stage][index]);
     Buffer& buffer = slot_buffers[binding.buffer_id];
     TouchBuffer(buffer, binding.buffer_id);
+    if constexpr (USE_UNIFIED_MEMORY) {
+        const auto window = TryResolveUnifiedRange(device_addr, size);
+        if (window && runtime.IsUnifiedUniformRange(size, window->offset)) {
+            channel_state->fast_bound_uniform_buffers[stage] &= ~(1u << binding_index);
+            channel_state->uniform_buffer_binding_sizes[stage][binding_index] = size;
+            runtime.BindUniformBuffer(runtime.UnifiedWindowBuffer(window->window),
+                                      runtime.UnifiedWindowAddress(window->window),
+                                      static_cast<u32>(window->offset), size);
+            return;
+        }
+    }
     const bool has_host_buffer = binding.buffer_id != NULL_BUFFER_ID;
     const u32 offset = has_host_buffer ? buffer.Offset(device_addr) : 0;
     const bool needs_alignment_stream = [&]() {
@@ -1868,7 +1945,7 @@ BufferCache<P>::TryResolveUnifiedRange([[maybe_unused]] DAddr device_addr,
         if (window_size == 0) {
             return std::nullopt;
         }
-        const u8* const first = device_memory.GetPointer<u8>(device_addr);
+        const u8* const first = device_memory.GetSpan(device_addr, size);
         if (first == nullptr) {
             return std::nullopt;
         }
@@ -1890,14 +1967,6 @@ BufferCache<P>::TryResolveUnifiedRange([[maybe_unused]] DAddr device_addr,
             IsRegionGpuModified(device_addr, size)) {
             return std::nullopt;
         }
-        u64 walked = Core::DEVICE_PAGESIZE - (device_addr & Core::DEVICE_PAGEMASK);
-        while (walked < size) {
-            const u8* const next = device_memory.GetPointer<u8>(device_addr + walked);
-            if (next != first + walked) {
-                return std::nullopt;
-            }
-            walked += Core::DEVICE_PAGESIZE;
-        }
         return UnifiedWindowRange{
             .window = static_cast<size_t>(relative / window_size),
             .offset = local_offset,

src/video_core/buffer_cache/buffer_cache_base.h

@@ -10,6 +10,7 @@
 #include <array>
 #include <bit>
 #include <functional>
+#include <limits>
 #include <memory>
 #include <mutex>
 #include <optional>
@@ -96,12 +97,15 @@ static constexpr Binding NULL_BINDING{
     .buffer_id = NULL_BUFFER_ID,
 };
 
+static constexpr u32 NO_UNIFIED_WINDOW = (std::numeric_limits<u32>::max)();
+
 template <typename Buffer>
 struct HostBindings {
     boost::container::static_vector<Buffer*, NUM_VERTEX_BUFFERS> buffers;
     boost::container::static_vector<u64, NUM_VERTEX_BUFFERS> offsets;
     boost::container::static_vector<u64, NUM_VERTEX_BUFFERS> sizes;
     boost::container::static_vector<u64, NUM_VERTEX_BUFFERS> strides;
+    boost::container::static_vector<u32, NUM_VERTEX_BUFFERS> unified_windows;
     u32 min_index{NUM_VERTEX_BUFFERS};
     u32 max_index{0};
 };
@@ -505,6 +509,7 @@ private:
     u32 last_index_count = 0;
 
     u32 enabled_vertex_buffers_mask = 0;
+    u32 unified_vertex_buffers_mask = 0;
     u64 vertex_buffers_serial = 0;
     std::array<Binding, 32> v_buffer{};
 

src/video_core/renderer_vulkan/vk_buffer_cache.cpp

@@ -7,6 +7,7 @@
 #include <algorithm>
 #include <array>
 #include <cstring>
+#include <limits>
 #include <span>
 #include <utility>
 #include <vector>
@@ -69,6 +70,16 @@ VkIndexType IndexTypeFromNumElements(const Device& device, u32 num_elements) {
     return VK_INDEX_TYPE_UINT32;
 }
 
+u32 GrowIndexCount(u32 current, u32 requested) {
+    constexpr u32 MinimumIndices = 4096;
+    constexpr u32 GrowthLimit = (std::numeric_limits<u32>::max)() / 2;
+    u32 grown = (std::max)(requested, MinimumIndices);
+    if (current <= GrowthLimit) {
+        grown = (std::max)(grown, current * 2);
+    }
+    return grown;
+}
+
 size_t BytesPerIndex(VkIndexType index_type) {
     switch (index_type) {
     case VK_INDEX_TYPE_UINT8_EXT:
@@ -185,13 +196,12 @@ public:
     virtual ~QuadIndexBuffer() = default;
 
     void UpdateBuffer(u32 num_indices_) {
+        ReleaseRetiredBuffers();
         if (num_indices_ <= num_indices) {
             return;
         }
 
-        scheduler.Finish();
+        num_indices = GrowIndexCount(num_indices, num_indices_);
-
-        num_indices = num_indices_;
         index_type = IndexTypeFromNumElements(device, num_indices);
 
         const u32 num_quads = GetQuadsNum(num_indices);
@@ -209,6 +219,12 @@ public:
             .queueFamilyIndexCount = 0,
             .pQueueFamilyIndices = nullptr,
         };
+        if (buffer) {
+            retired_buffers.push_back(RetiredBuffer{
+                .buffer = std::move(buffer),
+                .tick = scheduler.CurrentTick(),
+            });
+        }
         buffer = memory_allocator.CreateBuffer(buffer_ci, MemoryUsage::DeviceLocal);
         if (device.HasDebuggingToolAttached()) {
             buffer.SetObjectNameEXT("Quad LUT");
@@ -276,6 +292,17 @@ protected:
 
     virtual void MakeAndUpdateIndices(u8* staging_data, size_t quad_size, u32 quad, u32 first) = 0;
 
+    struct RetiredBuffer {
+        vk::Buffer buffer;
+        u64 tick;
+    };
+
+    void ReleaseRetiredBuffers() {
+        std::erase_if(retired_buffers, [this](const RetiredBuffer& entry) {
+            return scheduler.IsFree(entry.tick);
+        });
+    }
+
     const Device& device;
     MemoryAllocator& memory_allocator;
     Scheduler& scheduler;
@@ -283,6 +310,7 @@ protected:
 
     vk::Buffer buffer{};
     MemoryCommit memory_commit{};
+    std::vector<RetiredBuffer> retired_buffers;
     VkIndexType index_type{};
     u32 num_indices = 0;
 };
@@ -699,6 +727,25 @@ void BufferCacheRuntime::ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t si
     });
 }
 
+bool BufferCacheRuntime::IsUnifiedIndexRange(PrimitiveTopology topology, IndexFormat index_format,
+                                             u64 offset) const {
+    const VkIndexType vk_index_type = MaxwellToVK::IndexFormat(index_format);
+    const bool needs_uint8_pass =
+        vk_index_type == VK_INDEX_TYPE_UINT8_EXT && !device.IsExtIndexTypeUint8Supported();
+    if (topology == PrimitiveTopology::Quads || topology == PrimitiveTopology::QuadStrip ||
+        needs_uint8_pass) {
+        return (offset % device.GetStorageBufferAlignment()) == 0;
+    }
+    switch (vk_index_type) {
+    case VK_INDEX_TYPE_UINT32:
+        return (offset % 4) == 0;
+    case VK_INDEX_TYPE_UINT16:
+        return (offset % 2) == 0;
+    default:
+        return true;
+    }
+}
+
 void BufferCacheRuntime::BindIndexBuffer(PrimitiveTopology topology, IndexFormat index_format,
                                          u32 base_vertex, u32 num_indices, VkBuffer buffer,
                                          u32 offset, [[maybe_unused]] u32 size) {
@@ -774,6 +821,11 @@ void BufferCacheRuntime::BindVertexBuffer(u32 index, VkBuffer buffer, u32 offset
 void BufferCacheRuntime::BindVertexBuffers(VideoCommon::HostBindings<Buffer>& bindings) {
     boost::container::static_vector<VkBuffer, VideoCommon::NUM_VERTEX_BUFFERS> buffer_handles(bindings.buffers.size());
     for (u32 i = 0; i < bindings.buffers.size(); ++i) {
+        if (i < bindings.unified_windows.size() &&
+            bindings.unified_windows[i] != VideoCommon::NO_UNIFIED_WINDOW) {
+            buffer_handles[i] = unified_memory->GetWindowBuffer(bindings.unified_windows[i]);
+            continue;
+        }
         auto handle = bindings.buffers[i]->Handle();
         if (handle == VK_NULL_HANDLE) {
             bindings.offsets[i] = 0;

src/video_core/renderer_vulkan/vk_buffer_cache.h

@@ -190,6 +190,10 @@ public:
         BindBuffer(buffer, offset, size);
     }
 
+    void BindUniformBuffer(VkBuffer buffer, VkDeviceAddress address, u32 offset, u32 size) {
+        guest_descriptor_queue.AddBuffer(buffer, address, offset, size);
+    }
+
     void BindStorageBuffer(const Buffer& buffer, u32 offset, u32 size,
                            [[maybe_unused]] bool is_written) {
         BindBuffer(buffer, offset, size);
@@ -218,6 +222,14 @@ public:
                (offset % device.GetStorageBufferAlignment()) == 0;
     }
 
+    [[nodiscard]] bool IsUnifiedUniformRange(u32 size, u64 offset) const {
+        return size <= device.GetMaxUniformBufferRange() &&
+               (offset % device.GetUniformBufferAlignment()) == 0;
+    }
+
+    [[nodiscard]] bool IsUnifiedIndexRange(PrimitiveTopology topology, IndexFormat index_format,
+                                           u64 offset) const;
+
     void BindTextureBuffer(Buffer& buffer, u32 offset, u32 size,
                            VideoCore::Surface::PixelFormat format) {
         guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format),

src/video_core/vulkan_common/vulkan_device.h

@@ -351,6 +351,10 @@ public:
         return properties.properties.limits.maxStorageBufferRange;
     }
 
+    VkDeviceSize GetMaxUniformBufferRange() const {
+        return properties.properties.limits.maxUniformBufferRange;
+    }
+
     std::array<u32, 3> GetMaxComputeWorkGroupCount() const {
         const auto& count = properties.properties.limits.maxComputeWorkGroupCount;
         return {count[0], count[1], count[2]};