diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index be85cd63de..b09f26f4f1 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -838,6 +838,9 @@ void BufferCache

::BindHostIndexBuffer() { const u32 size = channel_state->index_buffer.size; const auto& draw_state = maxwell3d->draw_manager.draw_state; if (draw_state.inline_index_draw_indexes.empty()) { + if (BindVirtualIndexBuffer()) { + return; + } SynchronizeBuffer(buffer, channel_state->index_buffer.device_addr, size); } else { if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) { @@ -891,6 +894,7 @@ void BufferCache

::UpdateVertexBufferSlot(u32 index, const Binding& binding) { enabled_vertex_buffers_mask |= (1u << index); } else { enabled_vertex_buffers_mask &= ~(1u << index); + virtual_vertex_buffers_mask &= ~(1u << index); } } @@ -923,8 +927,16 @@ void BufferCache

::BindHostVertexBuffers() { const Binding& binding = VertexBufferSlot(index); Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); + if (StageVirtualVertexBuffer(index, binding, false)) { + continue; + } + bool needs_bind = flags[Dirty::VertexBuffer0 + index]; + if (((virtual_vertex_buffers_mask >> index) & 1) != 0) { + virtual_vertex_buffers_mask &= ~(1u << index); + needs_bind = true; + } SynchronizeBuffer(buffer, binding.device_addr, binding.size); - if (!flags[Dirty::VertexBuffer0 + index]) { + if (!needs_bind) { flush_bindings(); continue; } @@ -945,6 +957,7 @@ void BufferCache

::BindHostVertexBuffers() { last_index = index; } flush_bindings(); + BindStagedVertexBuffers(); } else { HostBindings host_bindings; bool any_valid{false}; @@ -953,8 +966,17 @@ void BufferCache

::BindHostVertexBuffers() { const Binding& binding = channel_state->vertex_buffers[index]; Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); + if (((enabled_vertex_buffers_mask >> index) & 1) != 0 && + StageVirtualVertexBuffer(index, binding, true)) { + continue; + } + bool needs_bind = flags[Dirty::VertexBuffer0 + index]; + if (((virtual_vertex_buffers_mask >> index) & 1) != 0) { + virtual_vertex_buffers_mask &= ~(1u << index); + needs_bind = true; + } SynchronizeBuffer(buffer, binding.device_addr, binding.size); - if (!flags[Dirty::VertexBuffer0 + index]) { + if (!needs_bind) { continue; } flags[Dirty::VertexBuffer0 + index] = false; @@ -983,6 +1005,7 @@ void BufferCache

::BindHostVertexBuffers() { } runtime.BindVertexBuffers(host_bindings); } + BindStagedVertexBuffers(); } } @@ -1038,9 +1061,12 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 }(); const bool use_fast_buffer = needs_alignment_stream || (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size - && !memory_tracker.IsRegionGpuModified(device_addr, size)); + && !memory_tracker.IsRegionGpuModified(device_addr, size) + && !HasPendingUnifiedWrites(device_addr, size)); if (use_fast_buffer) { - WaitForUnifiedWrites(device_addr, size); + if (needs_alignment_stream) { + WaitForUnifiedWrites(device_addr, size); + } if constexpr (IS_OPENGL) { if (runtime.HasFastBufferSubData()) { // Fast path for Nvidia @@ -1136,6 +1162,34 @@ void BufferCache

::ResolveMultiRangeStorage(Binding& binding, bool is_written, template bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_written, std::span pool) { + if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) { + const u64 key = GeometryKey(binding.gpu_addr, 0); + if (!PushMultiRangeSources(binding, is_written, pool, key)) { + return false; + } + return runtime.BindMultiRangeStorageBuffer(key, is_written); + } else { + return false; + } +} + +template +u64 BufferCache

::GeometryKey(GPUVAddr gpu_addr, u64 salt) const { + return ((static_cast(gpu_memory->GetID()) << 48) ^ gpu_addr) ^ salt; +} + +template +u32 BufferCache

::ClampToMappedRange(GPUVAddr gpu_addr, u32 size) const { + const size_t mapped = gpu_memory->MaxMappedRange(gpu_addr, size); + if (mapped == 0 || mapped >= size) { + return size; + } + return static_cast(mapped); +} + +template +bool BufferCache

::PushMultiRangeSources(const Binding& binding, bool is_written, + std::span pool, u64 key) { if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) { if (binding.segment_count < 2) { return false; @@ -1143,23 +1197,188 @@ bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_writt if (binding.segment_first + binding.segment_count > pool.size()) { return false; } - const u64 key = (static_cast(gpu_memory->GetID()) << 48) ^ binding.gpu_addr; + const std::span segments = + pool.subspan(binding.segment_first, binding.segment_count); + boost::container::small_vector buffer_ids; + for (const MultiRangeSegment& segment : segments) { + BufferId buffer_id = segment.buffer_id; + if (!buffer_id) { + buffer_id = page_table[segment.device_addr >> CACHING_PAGEBITS]; + } + if (!buffer_id || + !slot_buffers[buffer_id].IsInBounds(segment.device_addr, segment.size)) { + return false; + } + buffer_ids.push_back(buffer_id); + } runtime.ResetMultiRange(); for (u32 index = 0; index < binding.segment_count; ++index) { - const MultiRangeSegment& segment = pool[binding.segment_first + index]; - Buffer& buffer = slot_buffers[segment.buffer_id]; - TouchBuffer(buffer, segment.buffer_id); - if (SynchronizeBuffer(buffer, segment.device_addr, segment.size)) { + const MultiRangeSegment& segment = segments[index]; + const BufferId buffer_id = buffer_ids[index]; + Buffer& buffer = slot_buffers[buffer_id]; + TouchBuffer(buffer, buffer_id); + if (!SynchronizeBuffer(buffer, segment.device_addr, segment.size)) { runtime.InvalidateMultiRange(key); } const u32 offset = buffer.Offset(segment.device_addr); buffer.MarkUsage(offset, segment.size); if (is_written) { - MarkWrittenBuffer(segment.buffer_id, segment.device_addr, segment.size); + MarkWrittenBuffer(buffer_id, segment.device_addr, segment.size); } runtime.PushMultiRangeSource(buffer, offset, segment.size); } - return runtime.BindMultiRangeStorageBuffer(key, is_written); + return true; + } else { + return false; + } +} + +template +bool BufferCache

::TryResolveUnifiedSegments( + [[maybe_unused]] const Binding& binding, + [[maybe_unused]] std::span pool, + [[maybe_unused]] UnifiedExtents& extents) { + if constexpr (USE_UNIFIED_MEMORY) { + extents.clear(); + const auto push = [&](DAddr device_addr, u64 size) { + const std::optional relative = TryResolveUnifiedRange(device_addr, size); + if (!relative) { + return false; + } + if (!extents.empty() && extents.back().relative + extents.back().size == *relative) { + extents.back().size += size; + return true; + } + extents.push_back(UnifiedExtent{.relative = *relative, .size = size}); + return true; + }; + if (binding.segment_count < 2) { + return push(binding.device_addr, binding.size); + } + if (binding.segment_first + binding.segment_count > pool.size()) { + return false; + } + for (const MultiRangeSegment& segment : + pool.subspan(binding.segment_first, binding.segment_count)) { + if (!push(segment.device_addr, segment.size)) { + return false; + } + } + return true; + } else { + return false; + } +} + +template +void BufferCache

::ResolveGeometrySegments([[maybe_unused]] bool is_indexed) { + if constexpr (requires { runtime.BindStagedVertexBuffers(); }) { + const auto& draw_state = maxwell3d->draw_manager.draw_state; + if (is_indexed && draw_state.inline_index_draw_indexes.empty()) { + ResolveMultiRangeStorage(channel_state->index_buffer, false, graphics_segments); + } + u32 enabled_mask = enabled_vertex_buffers_mask; + while (enabled_mask != 0) { + const u32 index = std::countr_zero(enabled_mask); + enabled_mask &= enabled_mask - 1; + Binding& slot = VertexBufferSlot(index); + ResolveMultiRangeStorage(slot, false, graphics_segments); + Binding& channel_binding = channel_state->vertex_buffers[index]; + channel_binding.segment_first = slot.segment_first; + channel_binding.segment_count = slot.segment_count; + } + } +} + +template +bool BufferCache

::StageVirtualVertexBuffer([[maybe_unused]] u32 index, + [[maybe_unused]] const Binding& binding, + [[maybe_unused]] bool force) { + if constexpr (requires { runtime.BindStagedVertexBuffers(); }) { + auto& flags = maxwell3d->dirty.flags; + const bool rebind = force || flags[Dirty::VertexBuffer0 + index] || + ((virtual_vertex_buffers_mask >> index) & 1) == 0; + const u32 stride = maxwell3d->regs.vertex_streams[index].stride; + const u64 salt = VERTEX_GEOMETRY_SALT + (u64{index} << 40); + bool staged = false; + if constexpr (USE_UNIFIED_MEMORY) { + UnifiedExtents extents; + if (TryResolveUnifiedSegments(binding, graphics_segments, extents)) { + const u64 view_key = GeometryKey(binding.gpu_addr, salt + VIEW_GEOMETRY_SALT); + staged = runtime.StageUnifiedVertexBuffer(index, view_key, extents, binding.size, + stride, rebind); + } + } + if (!staged) { + const u64 key = GeometryKey(binding.gpu_addr, salt); + if (PushMultiRangeSources(binding, false, graphics_segments, key)) { + staged = runtime.StageMultiRangeVertexBuffer(index, key, binding.size, stride, + rebind); + } + } + if (!staged) { + return false; + } + flags[Dirty::VertexBuffer0 + index] = false; + virtual_vertex_buffers_mask |= 1u << index; + return true; + } else { + return false; + } +} + +template +bool BufferCache

::BindVirtualIndexBuffer() { + if constexpr (!HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT && + requires { runtime.BindStagedVertexBuffers(); }) { + const Binding& binding = channel_state->index_buffer; + const auto& draw_state = maxwell3d->draw_manager.draw_state; + const auto& index_ref = draw_state.index_buffer; + if constexpr (USE_UNIFIED_MEMORY) { + UnifiedExtents extents; + const u64 view_key = + GeometryKey(binding.gpu_addr, INDEX_GEOMETRY_SALT + VIEW_GEOMETRY_SALT); + if (TryResolveUnifiedSegments(binding, graphics_segments, extents) && + runtime.BindUnifiedIndexBuffer(draw_state.topology, index_ref.format, + index_ref.first, index_ref.count, view_key, + extents, binding.size)) { + return true; + } + } + const u64 key = GeometryKey(binding.gpu_addr, INDEX_GEOMETRY_SALT); + if (!PushMultiRangeSources(binding, false, graphics_segments, key)) { + return false; + } + return runtime.BindMultiRangeIndexBuffer(draw_state.topology, index_ref.format, + index_ref.first, index_ref.count, key, + binding.size); + } else { + return false; + } +} + +template +void BufferCache

::BindStagedVertexBuffers() { + if constexpr (requires { runtime.BindStagedVertexBuffers(); }) { + runtime.BindStagedVertexBuffers(); + } +} + +template +bool BufferCache

::HasPendingUnifiedWrites([[maybe_unused]] DAddr device_addr, + [[maybe_unused]] u64 size) { + if constexpr (USE_UNIFIED_MEMORY) { + if (unified_written_ranges.Empty()) { + return false; + } + if (runtime.KnownGpuTick() >= unified_write_tick) { + unified_written_ranges.Clear(); + return false; + } + bool overlaps = false; + unified_written_ranges.ForEachInRange(device_addr, size, + [&overlaps](DAddr, DAddr) { overlaps = true; }); + return overlaps; } else { return false; } @@ -1169,13 +1388,7 @@ template void BufferCache

::WaitForUnifiedWrites([[maybe_unused]] DAddr device_addr, [[maybe_unused]] u64 size) { if constexpr (USE_UNIFIED_MEMORY) { - if (unified_written_ranges.Empty()) { - return; - } - bool overlaps = false; - unified_written_ranges.ForEachInRange(device_addr, size, - [&overlaps](DAddr, DAddr) { overlaps = true; }); - if (!overlaps) { + if (!HasPendingUnifiedWrites(device_addr, size)) { return; } runtime.Wait(unified_write_tick); @@ -1183,12 +1396,63 @@ void BufferCache

::WaitForUnifiedWrites([[maybe_unused]] DAddr device_addr, } } +template +template +bool BufferCache

::CopyUnifiedWrites([[maybe_unused]] Buffer& buffer, + [[maybe_unused]] DAddr device_addr, + [[maybe_unused]] u64 size, + [[maybe_unused]] Func&& add_upload) { + if constexpr (USE_UNIFIED_MEMORY) { + if (!HasPendingUnifiedWrites(device_addr, size)) { + return false; + } + boost::container::small_vector, 4> overlaps; + unified_written_ranges.ForEachInRange(device_addr, size, + [&overlaps](DAddr start, DAddr end) { + overlaps.emplace_back(start, end); + }); + const DAddr buffer_start = buffer.CpuAddr(); + boost::container::small_vector window_ids; + UnifiedWindowGroups groups; + for (const auto& [start, end] : overlaps) { + if (!ResolveUnifiedWindows(start, start - buffer_start, end - start, window_ids, + groups)) { + runtime.Wait(unified_write_tick); + unified_written_ranges.Clear(); + return false; + } + } + for (size_t i = 0; i < window_ids.size(); ++i) { + const std::span group_span(groups[i].data(), groups[i].size()); + runtime.CopyFromUnifiedMemory(window_ids[i], buffer, group_span); + } + DAddr cursor = device_addr; + for (const auto& [start, end] : overlaps) { + buffer.MarkUsage(start - buffer_start, end - start); + if (start > cursor) { + add_upload(cursor, start - cursor); + } + cursor = end; + } + if (cursor < device_addr + size) { + add_upload(cursor, device_addr + size - cursor); + } + return true; + } else { + return false; + } +} + template bool BufferCache

::BindUnifiedStorage([[maybe_unused]] const Binding& binding, [[maybe_unused]] bool is_written) { if constexpr (USE_UNIFIED_MEMORY) { - const auto window = TryResolveUnifiedRange(binding.device_addr, binding.size); - if (!window || !runtime.IsUnifiedStorageRange(binding.size, window->offset)) { + const auto relative = TryResolveUnifiedRange(binding.device_addr, binding.size); + if (!relative) { + return false; + } + const auto range = runtime.ResolveUnifiedStorage(*relative, binding.size); + if (!range) { return false; } if (is_written) { @@ -1197,9 +1461,8 @@ bool BufferCache

::BindUnifiedStorage([[maybe_unused]] const Binding& binding, unified_write_tick = runtime.CurrentTick(); uncommitted_unified_writes = true; } - runtime.BindStorageBuffer(runtime.UnifiedWindowBuffer(window->window), - runtime.UnifiedWindowAddress(window->window), - static_cast(window->offset), binding.size, is_written); + runtime.BindStorageBuffer(range->buffer, range->address, range->offset, + binding.size, is_written); return true; } else { return false; @@ -1421,6 +1684,7 @@ void BufferCache

::DoUpdateGraphicsBuffers(bool is_indexed) { UpdateIndexBuffer(); } UpdateVertexBuffers(); + ResolveGeometrySegments(is_indexed); UpdateTransformFeedbackBuffers(); for (size_t stage = 0; stage < NUM_STAGES; ++stage) { UpdateUniformBuffers(stage); @@ -1477,13 +1741,17 @@ void BufferCache

::UpdateIndexBuffer() { const std::optional device_addr = gpu_memory->GpuToCpuAddress(gpu_addr_begin); const u32 address_size = static_cast(gpu_addr_end - gpu_addr_begin); const u32 draw_size = (index_buffer_ref.count + index_buffer_ref.first) * u32(index_buffer_ref.FormatSizeInBytes()); - const u32 size = (std::min)(address_size, draw_size); + u32 size = (std::min)(address_size, draw_size); if (size == 0 || !device_addr) { channel_state->index_buffer = NULL_BINDING; return; } + if constexpr (requires { runtime.BindStagedVertexBuffers(); }) { + size = ClampToMappedRange(gpu_addr_begin, size); + } channel_state->index_buffer = Binding{ .device_addr = *device_addr, + .gpu_addr = gpu_addr_begin, .size = size, .buffer_id = FindBuffer(*device_addr, size, false), }; @@ -1521,10 +1789,13 @@ void BufferCache

::UpdateVertexBuffer(u32 index) { } if (!gpu_memory->IsWithinGPUAddressRange(gpu_addr_end) || size >= 64_MiB) { size = static_cast(gpu_memory->MaxContinuousRange(gpu_addr_begin, size)); + } else if constexpr (requires { runtime.BindStagedVertexBuffers(); }) { + size = ClampToMappedRange(gpu_addr_begin, size); } const BufferId buffer_id = FindBuffer(*device_addr, size, false); const Binding binding{ .device_addr = *device_addr, + .gpu_addr = gpu_addr_begin, .size = size, .buffer_id = buffer_id, }; @@ -1881,19 +2152,30 @@ bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 si u64 total_size_bytes = 0; u64 largest_copy = 0; const DAddr buffer_start = buffer.cpu_addr_cached; - memory_tracker.ForEachUploadRange(device_addr, size, [&](u64 device_addr_out, u64 range_size) { + const auto add_upload = [&](u64 upload_addr, u64 upload_size) { upload_copies.push_back(BufferCopy{ .src_offset = total_size_bytes, - .dst_offset = device_addr_out - buffer_start, - .size = range_size, + .dst_offset = upload_addr - buffer_start, + .size = upload_size, }); - total_size_bytes += range_size; - largest_copy = (std::max)(largest_copy, range_size); + total_size_bytes += upload_size; + largest_copy = (std::max)(largest_copy, upload_size); + }; + bool copied_from_windows = false; + memory_tracker.ForEachUploadRange(device_addr, size, [&](u64 device_addr_out, u64 range_size) { + if (CopyUnifiedWrites(buffer, device_addr_out, range_size, add_upload)) { + copied_from_windows = true; + return; + } + add_upload(device_addr_out, range_size); }); if (total_size_bytes == 0) { + if (copied_from_windows) { + any_buffer_uploaded = true; + return false; + } return true; } - WaitForUnifiedWrites(device_addr, size); const std::span copies_span(upload_copies.data(), upload_copies.size()); UploadMemory(buffer, total_size_bytes, largest_copy, copies_span); any_buffer_uploaded = true; @@ -2004,17 +2286,12 @@ bool BufferCache

::ResolveUnifiedWindows( } template -std::optional::UnifiedWindowRange> -BufferCache

::TryResolveUnifiedRange([[maybe_unused]] DAddr device_addr, - [[maybe_unused]] u64 size) { +std::optional BufferCache

::TryResolveUnifiedRange([[maybe_unused]] DAddr device_addr, + [[maybe_unused]] u64 size) { if constexpr (USE_UNIFIED_MEMORY) { if (size == 0 || !runtime.IsUnifiedMemoryBindable()) { return std::nullopt; } - const u64 window_size = runtime.UnifiedMemoryWindowSize(); - if (window_size == 0) { - return std::nullopt; - } const u8* const first = device_memory.GetSpan(device_addr, size); if (first == nullptr) { return std::nullopt; @@ -2029,18 +2306,11 @@ BufferCache

::TryResolveUnifiedRange([[maybe_unused]] DAddr device_addr, if (relative >= unified_size || unified_size - relative < size) { return std::nullopt; } - const u64 local_offset = relative % window_size; - if (window_size - local_offset < size) { - return std::nullopt; - } if (memory_tracker.IsRegionGpuModified(device_addr, size) || IsRegionGpuModified(device_addr, size)) { return std::nullopt; } - return UnifiedWindowRange{ - .window = static_cast(relative / window_size), - .offset = local_offset, - }; + return relative; } else { return std::nullopt; } @@ -2245,6 +2515,15 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { replace(channel_state->transform_feedback_buffers); replace(channel_state->compute_uniform_buffers); replace(channel_state->compute_storage_buffers); + const auto drop_segments = [buffer_id](std::vector& pool) { + for (MultiRangeSegment& segment : pool) { + if (segment.buffer_id == buffer_id) { + segment.buffer_id = BufferId{}; + } + } + }; + drop_segments(graphics_segments); + drop_segments(compute_segments); // Mark the whole buffer as CPU written to stop tracking CPU writes if (!do_not_mark) { diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 0f2323b99f..211b67a883 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -96,6 +96,11 @@ struct MultiRangeSegment { u32 size{}; }; +struct UnifiedExtent { + u64 relative{}; + u64 size{}; +}; + struct TextureBufferBinding : Binding { PixelFormat format; }; @@ -192,6 +197,9 @@ class BufferCache : public VideoCommon::ChannelSetupCaches pool, u64 key); + void ResolveMultiRangeStorage(Binding& binding, bool is_written, std::vector& pool); @@ -468,15 +479,32 @@ private: bool TryUnifiedDownloadMemory(Buffer& buffer, std::span copies); - struct UnifiedWindowRange { - size_t window; - u64 offset; - }; + std::optional TryResolveUnifiedRange(DAddr device_addr, u64 size); - std::optional TryResolveUnifiedRange(DAddr device_addr, u64 size); + using UnifiedExtents = boost::container::small_vector; + + bool TryResolveUnifiedSegments(const Binding& binding, std::span pool, + UnifiedExtents& extents); + + void ResolveGeometrySegments(bool is_indexed); + + bool StageVirtualVertexBuffer(u32 index, const Binding& binding, bool force); + + bool BindVirtualIndexBuffer(); + + void BindStagedVertexBuffers(); + + [[nodiscard]] u64 GeometryKey(GPUVAddr gpu_addr, u64 salt) const; + + [[nodiscard]] u32 ClampToMappedRange(GPUVAddr gpu_addr, u32 size) const; + + bool HasPendingUnifiedWrites(DAddr device_addr, u64 size); void WaitForUnifiedWrites(DAddr device_addr, u64 size); + template + bool CopyUnifiedWrites(Buffer& buffer, DAddr device_addr, u64 size, Func&& add_upload); + using UnifiedWindowGroups = boost::container::small_vector, 4>; @@ -528,6 +556,7 @@ private: u32 last_index_count = 0; u32 enabled_vertex_buffers_mask = 0; + u32 virtual_vertex_buffers_mask = 0; u64 vertex_buffers_serial = 0; std::array v_buffer{}; diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index d5838a387d..9556115426 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -529,6 +529,29 @@ size_t MemoryManager::MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const { return range_so_far; } +size_t MemoryManager::MaxMappedRange(GPUVAddr gpu_addr, size_t size) const { + size_t range_so_far = 0; + bool stopped{false}; + auto stop = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, + [[maybe_unused]] std::size_t copy_amount) { + stopped = true; + return true; + }; + auto accumulate = [&]([[maybe_unused]] std::size_t page_index, + [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { + range_so_far += copy_amount; + return false; + }; + auto check_short_pages = [&](std::size_t page_index, std::size_t offset, + std::size_t copy_amount) { + GPUVAddr base = (page_index << big_page_bits) + offset; + MemoryOperation(base, copy_amount, false, accumulate, stop, stop); + return stopped; + }; + MemoryOperation(gpu_addr, size, true, accumulate, stop, check_short_pages); + return range_so_far; +} + size_t MemoryManager::GetMemoryLayoutSize(GPUVAddr gpu_addr, size_t max_size) const { std::unique_lock lock(guard); return kind_map.GetContinuousSizeFrom(gpu_addr); diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index 08978567d2..5eabcb2f8c 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -141,6 +141,8 @@ public: size_t MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const; + size_t MaxMappedRange(GPUVAddr gpu_addr, size_t size) const; + bool IsWithinGPUAddressRange(GPUVAddr gpu_addr) const { return gpu_addr < address_space_size; } diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 595acab8da..655db98450 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -12,6 +12,7 @@ #include #include +#include "common/alignment.h" #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/renderer_vulkan/vk_buffer_cache.h" @@ -435,6 +436,11 @@ void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size, size_t hardware_buffer_base) { unified_memory = memory_allocator.CreateHostMemoryImport( base, size, hardware_buffers, hardware_buffer_window, hardware_buffer_base); + if (unified_memory) { + unified_memory->CreateMirror(scheduler.submit_mutex, + device.GetSparseAddressSpaceSize() / 2); + multi_range_buffers.ReserveSparseAddressSpace(unified_memory->GetMirrorSize()); + } } void BufferCacheRuntime::CopyToUnifiedMemory( @@ -452,6 +458,74 @@ void BufferCacheRuntime::CopyToUnifiedMemory( std::ranges::transform(copies, pending.copies.begin(), MakeBufferCopy); } +void BufferCacheRuntime::CopyFromUnifiedMemory( + size_t window_index, VkBuffer dst_buffer, std::span copies) { + if (!unified_memory || dst_buffer == VK_NULL_HANDLE || copies.empty() || + window_index >= unified_memory->GetWindowCount()) { + return; + } + const VkBuffer window_buffer = unified_memory->GetWindowBuffer(window_index); + if (window_buffer == VK_NULL_HANDLE) { + return; + } + const bool foreign = unified_memory->NeedsForeignOwnershipTransfer(); + const u32 queue_family = device.GetGraphicsFamily(); + boost::container::small_vector vk_copies; + WindowRanges ranges; + for (const VideoCommon::BufferCopy& copy : copies) { + vk_copies.push_back(VkBufferCopy{ + .srcOffset = static_cast(copy.dst_offset), + .dstOffset = static_cast(copy.src_offset), + .size = static_cast(copy.size), + }); + ranges.emplace_back(copy.dst_offset, copy.dst_offset + copy.size); + } + CoalesceWindowRanges(ranges); + boost::container::small_vector acquire; + boost::container::small_vector release; + if (foreign) { + for (const WindowRange& range : ranges) { + acquire.push_back(VkBufferMemoryBarrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = 0, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .dstQueueFamilyIndex = queue_family, + .buffer = window_buffer, + .offset = range.first, + .size = range.second - range.first, + }); + release.push_back(VkBufferMemoryBarrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .dstAccessMask = 0, + .srcQueueFamilyIndex = queue_family, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .buffer = window_buffer, + .offset = range.first, + .size = range.second - range.first, + }); + } + } + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([window_buffer, dst_buffer, vk_copies, acquire = std::move(acquire), + release = std::move(release)](vk::CommandBuffer cmdbuf) { + if (!acquire.empty()) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, + VideoCommon::FixSmallVectorADL(acquire), {}); + } + cmdbuf.CopyBuffer(window_buffer, dst_buffer, VideoCommon::FixSmallVectorADL(vk_copies)); + if (!release.empty()) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, {}, + VideoCommon::FixSmallVectorADL(release), {}); + } + }); +} + void BufferCacheRuntime::FlushUnifiedMemoryCopies() { if (pending_unified_copies.empty()) { return; @@ -733,17 +807,17 @@ void BufferCacheRuntime::ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t si }); } -bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key, bool is_written) { +MultiRangeRef BufferCacheRuntime::AcquireMultiRange(u64 key, bool require_sparse) { if (multi_range_sources.empty() || multi_range_total == 0) { - return false; + return MultiRangeRef{}; } const MultiRangeRef ref = multi_range_buffers.Get(device, scheduler, memory_allocator, key, multi_range_sources, multi_range_total); if (ref.handle == VK_NULL_HANDLE) { - return false; + return MultiRangeRef{}; } - if (is_written && !ref.sparse) { - return false; + if (require_sparse && !ref.sparse) { + return MultiRangeRef{}; } if (ref.needs_gather) { PreCopyBarrier(); @@ -760,10 +834,220 @@ bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key, bool is_written) { PostCopyBarrier(); multi_range_buffers.MarkGathered(key); } + return ref; +} + +bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key, bool is_written) { + const MultiRangeRef ref = AcquireMultiRange(key, is_written); + if (ref.handle == VK_NULL_HANDLE) { + return false; + } guest_descriptor_queue.AddBuffer(ref.handle, ref.address, 0, ref.size); return true; } +std::optional BufferCacheRuntime::ResolveUnifiedExtents( + u64 key, std::span extents, u32 size) { + if (extents.size() == 1) { + return unified_memory->ResolveRange(extents.front().relative, size); + } + return AcquireUnifiedView(key, extents); +} + +std::optional BufferCacheRuntime::AcquireUnifiedView( + u64 key, std::span extents) { + const VkBufferUsageFlags usage = unified_memory->GetViewUsage(); + if (usage == 0 || !multi_range_buffers.use_sparse || extents.size() < 2) { + return std::nullopt; + } + const VkDeviceSize block = multi_range_buffers.block_size; + boost::container::small_vector sources; + VkDeviceSize total = 0; + VkDeviceSize padding = 0; + for (size_t index = 0; index < extents.size(); ++index) { + VkDeviceSize begin = extents[index].relative; + VkDeviceSize end = begin + extents[index].size; + if (index == 0) { + padding = begin % block; + begin -= padding; + } else if ((begin % block) != 0) { + return std::nullopt; + } + if (index + 1 == extents.size()) { + end = Common::AlignUp(end, block); + } else if ((end % block) != 0) { + return std::nullopt; + } + while (begin < end) { + const auto memory = unified_memory->ResolveViewMemory(begin); + if (!memory || (memory->offset % block) != 0) { + return std::nullopt; + } + const VkDeviceSize length = (std::min)(end - begin, memory->available); + if ((length % block) != 0) { + return std::nullopt; + } + sources.push_back(MultiRangeSource{ + .handle = memory->buffer, + .memory = memory->memory, + .memory_offset = 0, + .offset = memory->offset, + .size = length, + .write_tick = 0, + .memory_type = memory->memory_type, + }); + total += length; + begin += length; + } + } + const MultiRangeRef ref = multi_range_buffers.GetView( + device, scheduler, key, sources, total, usage, + VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID); + if (ref.handle == VK_NULL_HANDLE) { + return std::nullopt; + } + return HostMemoryImport::Range{ + .buffer = ref.handle, + .address = ref.address, + .offset = padding, + }; +} + +bool BufferCacheRuntime::StageUnifiedVertexBuffer( + u32 index, u64 key, std::span extents, u32 size, u32 stride, + bool force) { + if (!unified_memory || + (unified_memory->GetUsage() & VK_BUFFER_USAGE_VERTEX_BUFFER_BIT) == 0) { + return false; + } + const auto range = ResolveUnifiedExtents(key, extents, size); + if (!range) { + return false; + } + StageVertexBuffer( + StagedVertexBuffer{ + .index = index, + .buffer = range->buffer, + .offset = range->offset, + .size = size, + .stride = stride, + }, + force); + return true; +} + +bool BufferCacheRuntime::StageMultiRangeVertexBuffer(u32 index, u64 key, u32 size, u32 stride, + bool force) { + const MultiRangeRef ref = AcquireMultiRange(key, false); + if (ref.handle == VK_NULL_HANDLE) { + return false; + } + StageVertexBuffer( + StagedVertexBuffer{ + .index = index, + .buffer = ref.handle, + .offset = 0, + .size = size, + .stride = stride, + }, + force); + return true; +} + +void BufferCacheRuntime::StageVertexBuffer(const StagedVertexBuffer& target, bool force) { + StagedVertexBuffer& bound = bound_vertex_buffers[target.index]; + if (!force && bound == target) { + return; + } + bound = target; + if (target.index < device.GetMaxVertexInputBindings()) { + staged_vertex_buffers.push_back(target); + } +} + +void BufferCacheRuntime::BindStagedVertexBuffers() { + if (staged_vertex_buffers.empty()) { + return; + } + scheduler.Record([staged = staged_vertex_buffers, + extended = device.IsExtExtendedDynamicStateSupported()]( + vk::CommandBuffer cmdbuf) { + std::array buffers{}; + std::array offsets{}; + std::array sizes{}; + std::array strides{}; + size_t begin = 0; + while (begin < staged.size()) { + const u32 first = staged[begin].index; + u32 count = 0; + while (begin + count < staged.size() && staged[begin + count].index == first + count) { + const StagedVertexBuffer& entry = staged[begin + count]; + buffers[count] = entry.buffer; + offsets[count] = entry.offset; + sizes[count] = entry.size; + strides[count] = entry.stride; + ++count; + } + if (extended) { + cmdbuf.BindVertexBuffers2EXT(first, count, buffers.data(), offsets.data(), + sizes.data(), strides.data()); + } else { + cmdbuf.BindVertexBuffers(first, count, buffers.data(), offsets.data()); + } + begin += count; + } + }); + staged_vertex_buffers.clear(); +} + +bool BufferCacheRuntime::IsIndexRangeUsable(PrimitiveTopology topology, IndexFormat index_format, + VkDeviceSize offset, u32 size) const { + const VkIndexType vk_index_type = MaxwellToVK::IndexFormat(index_format); + const bool is_quad = + topology == PrimitiveTopology::Quads || topology == PrimitiveTopology::QuadStrip; + const bool is_emulated_uint8 = + vk_index_type == VK_INDEX_TYPE_UINT8_EXT && !device.IsExtIndexTypeUint8Supported(); + if (is_quad || is_emulated_uint8) { + return size <= device.GetMaxStorageBufferRange() && + (offset % device.GetStorageBufferAlignment()) == 0; + } + return (offset % BytesPerIndex(vk_index_type)) == 0; +} + +bool BufferCacheRuntime::BindUnifiedIndexBuffer(PrimitiveTopology topology, + IndexFormat index_format, u32 base_vertex, + u32 num_indices, u64 key, + std::span extents, + u32 size) { + constexpr VkBufferUsageFlags GeometryUsage = + VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + if (!unified_memory || (unified_memory->GetUsage() & GeometryUsage) != GeometryUsage) { + return false; + } + const auto range = ResolveUnifiedExtents(key, extents, size); + if (!range || range->offset > (std::numeric_limits::max)() - size || + !IsIndexRangeUsable(topology, index_format, range->offset, size)) { + return false; + } + BindIndexBuffer(topology, index_format, base_vertex, num_indices, range->buffer, + static_cast(range->offset), size); + return true; +} + +bool BufferCacheRuntime::BindMultiRangeIndexBuffer(PrimitiveTopology topology, + IndexFormat index_format, u32 base_vertex, + u32 num_indices, u64 key, u32 size) { + if (!IsIndexRangeUsable(topology, index_format, 0, size)) { + return false; + } + const MultiRangeRef ref = AcquireMultiRange(key, false); + if (ref.handle == VK_NULL_HANDLE) { + return false; + } + BindIndexBuffer(topology, index_format, base_vertex, num_indices, ref.handle, 0, size); + return true; +} + void BufferCacheRuntime::BindIndexBuffer(PrimitiveTopology topology, IndexFormat index_format, u32 base_vertex, u32 num_indices, VkBuffer buffer, u32 offset, [[maybe_unused]] u32 size) { diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 2cbbcca140..828bc0c339 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -135,6 +135,9 @@ public: void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer, std::span copies); + void CopyFromUnifiedMemory(size_t window_index, VkBuffer dst_buffer, + std::span copies); + void FlushUnifiedMemoryCopies(); void UnifiedMemoryHostBarrier(); @@ -229,6 +232,21 @@ public: bool BindMultiRangeStorageBuffer(u64 key, bool is_written); + bool StageUnifiedVertexBuffer(u32 index, u64 key, + std::span extents, u32 size, + u32 stride, bool force); + + bool StageMultiRangeVertexBuffer(u32 index, u64 key, u32 size, u32 stride, bool force); + + void BindStagedVertexBuffers(); + + bool BindUnifiedIndexBuffer(PrimitiveTopology topology, IndexFormat index_format, + u32 base_vertex, u32 num_indices, u64 key, + std::span extents, u32 size); + + bool BindMultiRangeIndexBuffer(PrimitiveTopology topology, IndexFormat index_format, + u32 base_vertex, u32 num_indices, u64 key, u32 size); + void InvalidateMultiRange(u64 key) { multi_range_buffers.Invalidate(key); } @@ -246,7 +264,7 @@ public: BindBuffer(buffer, offset, size); } - void BindStorageBuffer(VkBuffer buffer, VkDeviceAddress address, u32 offset, u32 size, + void BindStorageBuffer(VkBuffer buffer, VkDeviceAddress address, VkDeviceSize offset, u32 size, [[maybe_unused]] bool is_written) { guest_descriptor_queue.AddBuffer(buffer, address, offset, size); } @@ -256,17 +274,16 @@ public: unified_memory->IsBindable(); } - [[nodiscard]] VkBuffer UnifiedWindowBuffer(size_t index) const noexcept { - return unified_memory->GetWindowBuffer(index); - } - - [[nodiscard]] VkDeviceAddress UnifiedWindowAddress(size_t index) const noexcept { - return unified_memory->GetWindowAddress(index); - } - - [[nodiscard]] bool IsUnifiedStorageRange(u32 size, u64 offset) const { - return size <= device.GetMaxStorageBufferRange() && - (offset % device.GetStorageBufferAlignment()) == 0; + [[nodiscard]] std::optional ResolveUnifiedStorage(u64 relative, + u32 size) const { + if (size > device.GetMaxStorageBufferRange()) { + return std::nullopt; + } + const auto range = unified_memory->ResolveRange(relative, size); + if (!range || (range->offset % device.GetStorageBufferAlignment()) != 0) { + return std::nullopt; + } + return range; } void BindTextureBuffer(Buffer& buffer, u32 offset, u32 size, @@ -291,6 +308,29 @@ private: boost::container::small_vector copies; }; + struct StagedVertexBuffer { + u32 index; + VkBuffer buffer; + VkDeviceSize offset; + u32 size; + u32 stride; + + bool operator==(const StagedVertexBuffer&) const = default; + }; + + [[nodiscard]] MultiRangeRef AcquireMultiRange(u64 key, bool require_sparse); + + [[nodiscard]] std::optional ResolveUnifiedExtents( + u64 key, std::span extents, u32 size); + + [[nodiscard]] std::optional AcquireUnifiedView( + u64 key, std::span extents); + + void StageVertexBuffer(const StagedVertexBuffer& target, bool force); + + [[nodiscard]] bool IsIndexRangeUsable(PrimitiveTopology topology, IndexFormat index_format, + VkDeviceSize offset, u32 size) const; + void BindBuffer(const Buffer& buffer, u32 offset, u32 size) { const VkBuffer handle = buffer.Handle(); if (handle == VK_NULL_HANDLE) { @@ -324,6 +364,9 @@ private: MultiRangeBufferCache multi_range_buffers; boost::container::small_vector multi_range_sources; VkDeviceSize multi_range_total{}; + boost::container::static_vector + staged_vertex_buffers; + std::array bound_vertex_buffers{}; bool limit_dynamic_storage_buffers = false; u32 max_dynamic_storage_buffers = (std::numeric_limits::max)(); diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp index 4d84390002..84f25a4655 100644 --- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp +++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp @@ -13,7 +13,8 @@ namespace Vulkan { MultiRangeBufferCache::MultiRangeBufferCache(const Device& device) { sparse_usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | - VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | + VK_BUFFER_USAGE_INDEX_BUFFER_BIT; if (device.IsBufferDeviceAddressSupported()) { sparse_usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; } @@ -27,9 +28,14 @@ MultiRangeBufferCache::MultiRangeBufferCache(const Device& device) { } block_size = queried; sparse_memory_type_bits = memory_type_bits; + sparse_budget = device.GetSparseAddressSpaceSize(); use_sparse = true; } +void MultiRangeBufferCache::ReserveSparseAddressSpace(VkDeviceSize size) noexcept { + sparse_budget -= (std::min)(sparse_budget, size); +} + VkDeviceSize MultiRangeBufferCache::QueryBlockSize(const Device& device, u32& memory_type_bits) const { const VkDevice logical = *device.GetLogical(); @@ -100,15 +106,26 @@ bool MultiRangeBufferCache::CanBindSparse(std::span sour SparseBuffer MultiRangeBufferCache::CreateSparse(const Device& device, Scheduler& scheduler, std::span sources, - VkDeviceSize total) { + VkDeviceSize total, VkBufferCreateFlags flags, + VkBufferUsageFlags usage, + VkExternalMemoryHandleTypeFlags handle_types) { const VkDevice logical = *device.GetLogical(); const auto& dld = device.GetDispatchLoader(); + const VkExternalMemoryBufferCreateInfo external_info{ + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, + .pNext = nullptr, + .handleTypes = handle_types, + }; + const void* next = nullptr; + if (handle_types != 0) { + next = &external_info; + } const VkBufferCreateInfo buffer_ci{ .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, - .pNext = nullptr, - .flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT | VK_BUFFER_CREATE_SPARSE_ALIASED_BIT, + .pNext = next, + .flags = flags, .size = total, - .usage = sparse_usage, + .usage = usage, .sharingMode = VK_SHARING_MODE_EXCLUSIVE, .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, @@ -118,6 +135,27 @@ SparseBuffer MultiRangeBufferCache::CreateSparse(const Device& device, Scheduler return SparseBuffer{}; } SparseBuffer handle{raw, logical, dld}; + const VkBufferMemoryRequirementsInfo2 reqs_info{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2, + .pNext = nullptr, + .buffer = raw, + }; + VkMemoryRequirements2 reqs2{ + .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, + .pNext = nullptr, + .memoryRequirements = {}, + }; + dld.vkGetBufferMemoryRequirements2(logical, &reqs_info, &reqs2); + const VkMemoryRequirements& requirements = reqs2.memoryRequirements; + if (requirements.alignment == 0 || (block_size % requirements.alignment) != 0) { + return SparseBuffer{}; + } + for (const MultiRangeSource& source : sources) { + if (source.memory_type >= 32 || + ((requirements.memoryTypeBits >> source.memory_type) & 1) == 0) { + return SparseBuffer{}; + } + } std::vector binds; binds.reserve(sources.size()); VkDeviceSize resource_offset = 0; @@ -185,10 +223,16 @@ void MultiRangeBufferCache::RetireEntry(Scheduler& scheduler, Entry& entry) { scheduler.Wait(oldest); DrainRetired(scheduler); } + VkDeviceSize sparse_size = 0; + if (entry.sparse_handle) { + sparse_size = entry.size; + } + sparse_retiring += sparse_size; retired.push_back(Retired{ .handle = std::move(entry.sparse_handle), .gathered = std::move(entry.gathered), .tick = scheduler.CurrentTick(), + .sparse_size = sparse_size, }); } @@ -196,6 +240,8 @@ void MultiRangeBufferCache::DrainRetired(Scheduler& scheduler) { size_t index = 0; while (index < retired.size()) { if (scheduler.IsFree(retired[index].tick)) { + sparse_in_use -= (std::min)(sparse_in_use, retired[index].sparse_size); + sparse_retiring -= (std::min)(sparse_retiring, retired[index].sparse_size); if (index + 1 != retired.size()) { retired[index] = std::move(retired.back()); } @@ -221,6 +267,7 @@ MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& schedu const auto it = entries.find(key); if (it != entries.end() && it->second.geometry == geometry && it->second.size == total) { Entry& entry = it->second; + entry.last_use = scheduler.CurrentTick(); if (entry.content != content) { entry.content = content; entry.dirty = true; @@ -248,9 +295,14 @@ MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& schedu entry.geometry = geometry; entry.content = content; entry.size = total; - if (CanBindSparse(sources)) { - entry.sparse_handle = CreateSparse(device, scheduler, sources, total); + entry.last_use = scheduler.CurrentTick(); + if (CanBindSparse(sources) && FitsSparse(scheduler, total)) { + entry.sparse_handle = CreateSparse( + device, scheduler, sources, total, + VK_BUFFER_CREATE_SPARSE_BINDING_BIT | VK_BUFFER_CREATE_SPARSE_ALIASED_BIT, + sparse_usage, 0); if (entry.sparse_handle) { + sparse_in_use += total; entry.owners.reserve(sources.size()); for (const MultiRangeSource& source : sources) { entry.owners.push_back(source.handle); @@ -260,7 +312,9 @@ MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& schedu if (!entry.sparse_handle) { VkBufferUsageFlags flags = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | - VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | + VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | + VK_BUFFER_USAGE_INDEX_BUFFER_BIT; if (device.IsBufferDeviceAddressSupported()) { flags |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; } @@ -301,6 +355,86 @@ MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& schedu return ref; } +MultiRangeRef MultiRangeBufferCache::GetView(const Device& device, Scheduler& scheduler, u64 key, + std::span sources, + VkDeviceSize total, VkBufferUsageFlags usage, + VkExternalMemoryHandleTypeFlags handle_types) { + if (!use_sparse || !views_supported || sources.empty() || total == 0) { + return MultiRangeRef{}; + } + if (!retired.empty()) { + DrainRetired(scheduler); + } + const u64 geometry = HashSources(sources); + const auto it = entries.find(key); + if (it != entries.end() && it->second.sparse_handle && it->second.geometry == geometry && + it->second.size == total) { + it->second.last_use = scheduler.CurrentTick(); + return MultiRangeRef{ + .handle = *it->second.sparse_handle, + .address = it->second.address, + .size = total, + .sparse = true, + .needs_gather = false, + }; + } + if (it != entries.end()) { + RetireEntry(scheduler, it->second); + entries.erase(it); + } + if (!FitsSparse(scheduler, total)) { + return MultiRangeRef{}; + } + Entry entry{}; + entry.sparse_handle = CreateSparse(device, scheduler, sources, total, + VK_BUFFER_CREATE_SPARSE_BINDING_BIT, usage, handle_types); + if (!entry.sparse_handle) { + views_supported = false; + return MultiRangeRef{}; + } + sparse_in_use += total; + entry.geometry = geometry; + entry.size = total; + entry.last_use = scheduler.CurrentTick(); + entry.dirty = false; + if ((usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0) { + entry.address = device.GetLogical().GetBufferDeviceAddress(*entry.sparse_handle); + } + const MultiRangeRef ref{ + .handle = *entry.sparse_handle, + .address = entry.address, + .size = total, + .sparse = true, + .needs_gather = false, + }; + entries.emplace(key, std::move(entry)); + return ref; +} + +bool MultiRangeBufferCache::FitsSparse(Scheduler& scheduler, VkDeviceSize total) { + if (total > sparse_budget) { + return false; + } + const u64 current_tick = scheduler.CurrentTick(); + while (sparse_in_use - sparse_retiring > sparse_budget - total) { + auto victim = entries.end(); + for (auto it = entries.begin(); it != entries.end(); ++it) { + if (!it->second.sparse_handle || it->second.last_use >= current_tick) { + continue; + } + if (victim == entries.end() || it->second.last_use < victim->second.last_use) { + victim = it; + } + } + if (victim == entries.end()) { + break; + } + RetireEntry(scheduler, victim->second); + entries.erase(victim); + } + return sparse_in_use <= sparse_budget - total; +} + void MultiRangeBufferCache::MarkGathered(u64 key) { if (auto const it = entries.find(key); it != entries.end()) { it->second.dirty = false; diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h index f40d3aa1f8..a6734878cb 100644 --- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h +++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h @@ -53,12 +53,19 @@ public: std::span sources, VkDeviceSize total); + [[nodiscard]] MultiRangeRef GetView(const Device& device, Scheduler& scheduler, u64 key, + std::span sources, + VkDeviceSize total, VkBufferUsageFlags usage, + VkExternalMemoryHandleTypeFlags handle_types); + void MarkGathered(u64 key); void Invalidate(u64 key); void DropOwner(Scheduler& scheduler, VkBuffer owner); + void ReserveSparseAddressSpace(VkDeviceSize size) noexcept; + VkDeviceSize block_size{DEFAULT_BLOCK_SIZE}; bool use_sparse{}; @@ -67,6 +74,7 @@ private: SparseBuffer handle; vk::Buffer gathered; u64 tick{}; + VkDeviceSize sparse_size{}; }; struct Entry { @@ -77,6 +85,7 @@ private: VkDeviceSize size{}; u64 geometry{}; u64 content{}; + u64 last_use{}; bool dirty{true}; }; @@ -88,7 +97,11 @@ private: [[nodiscard]] SparseBuffer CreateSparse(const Device& device, Scheduler& scheduler, std::span sources, - VkDeviceSize total); + VkDeviceSize total, VkBufferCreateFlags flags, + VkBufferUsageFlags usage, + VkExternalMemoryHandleTypeFlags handle_types); + + [[nodiscard]] bool FitsSparse(Scheduler& scheduler, VkDeviceSize total); [[nodiscard]] VkDeviceSize QueryBlockSize(const Device& device, u32& memory_type_bits) const; @@ -100,6 +113,10 @@ private: boost::container::static_vector retired; u32 sparse_memory_type_bits{}; VkBufferUsageFlags sparse_usage{}; + VkDeviceSize sparse_budget{}; + VkDeviceSize sparse_in_use{}; + VkDeviceSize sparse_retiring{}; + bool views_supported{true}; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index dd1bfeefcc..82f34157d5 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -3198,44 +3198,35 @@ u64 TextureCacheRuntime::UnifiedMemorySize() const noexcept { return import->GetSize(); } -u64 TextureCacheRuntime::UnifiedMemoryWindowSize() const noexcept { - const HostMemoryImport* const import = memory_allocator.GetHostMemoryImport(); - if (import == nullptr) { - return 0; - } - return import->GetWindowSize(); -} - bool TextureCacheRuntime::CanUploadImageDirectly(const VideoCommon::ImageInfo& info) const { return bl2d_unswizzle_pass.has_value() && BlockLinearUnswizzle2DPass::IsSupported(device, info); } -VkBuffer TextureCacheRuntime::ResolveDirectWindow(size_t window_index, u64 window_offset, - u64 size) const { - if ((window_offset % device.GetStorageBufferAlignment()) != 0) { - return VK_NULL_HANDLE; - } +std::optional TextureCacheRuntime::ResolveDirectRange(u64 relative, + u64 size) const { if (size > device.GetMaxStorageBufferRange()) { - return VK_NULL_HANDLE; + return std::nullopt; } const HostMemoryImport* const import = memory_allocator.GetHostMemoryImport(); - if (import == nullptr || window_index >= import->GetWindowCount()) { - return VK_NULL_HANDLE; + if (import == nullptr) { + return std::nullopt; } - return import->GetWindowBuffer(window_index); + const auto range = import->ResolveRange(relative, size); + if (!range || range->buffer == VK_NULL_HANDLE || + (range->offset % device.GetStorageBufferAlignment()) != 0) { + return std::nullopt; + } + return range; } bool TextureCacheRuntime::UploadImageDirectly( - Image& image, size_t window_index, u64 window_offset, - std::span swizzles) { - const VkBuffer window_buffer = - ResolveDirectWindow(window_index, window_offset, image.guest_size_bytes); - if (window_buffer == VK_NULL_HANDLE) { + Image& image, u64 relative, std::span swizzles) { + const auto range = ResolveDirectRange(relative, image.guest_size_bytes); + if (!range) { return false; } - bl2d_unswizzle_pass->UnswizzleFrom(image, window_buffer, - static_cast(window_offset), swizzles); + bl2d_unswizzle_pass->UnswizzleFrom(image, range->buffer, range->offset, swizzles); return true; } @@ -3243,15 +3234,16 @@ bool TextureCacheRuntime::CanDownloadImageDirectly(const VideoCommon::ImageInfo& return bl2d_swizzle_pass.has_value() && BlockLinearUnswizzle2DPass::IsSupported(device, info); } -bool TextureCacheRuntime::DownloadImageDirectly(Image& image, size_t window_index, - u64 window_offset) { - const u64 size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes); - const VkBuffer window_buffer = ResolveDirectWindow(window_index, window_offset, size); - if (window_buffer == VK_NULL_HANDLE) { +bool TextureCacheRuntime::DownloadImageDirectly(Image& image, u64 relative) { + if (image.unswizzled_size_bytes > device.GetMaxStorageBufferRange()) { + return false; + } + const auto range = ResolveDirectRange(relative, image.guest_size_bytes); + if (!range) { return false; } const HostMemoryImport* const import = memory_allocator.GetHostMemoryImport(); - bl2d_swizzle_pass->SwizzleInto(image, window_buffer, static_cast(window_offset), + bl2d_swizzle_pass->SwizzleInto(image, range->buffer, range->offset, import->NeedsForeignOwnershipTransfer()); return true; } @@ -3260,7 +3252,7 @@ u64 TextureCacheRuntime::CurrentTick() const noexcept { return scheduler.CurrentTick(); } -bool TextureCacheRuntime::IsDirectUploadRetired(u64 tick) { +bool TextureCacheRuntime::IsTickRetired(u64 tick) { if (scheduler.IsFree(tick)) { return true; } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 65b88f6b20..946cf25cd4 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -107,23 +107,21 @@ public: [[nodiscard]] u64 UnifiedMemorySize() const noexcept; - [[nodiscard]] u64 UnifiedMemoryWindowSize() const noexcept; - [[nodiscard]] bool CanUploadImageDirectly(const VideoCommon::ImageInfo& info) const; - bool UploadImageDirectly(Image& image, size_t window_index, u64 window_offset, + bool UploadImageDirectly(Image& image, u64 relative, std::span swizzles); [[nodiscard]] bool CanDownloadImageDirectly(const VideoCommon::ImageInfo& info) const; - bool DownloadImageDirectly(Image& image, size_t window_index, u64 window_offset); + bool DownloadImageDirectly(Image& image, u64 relative); - [[nodiscard]] VkBuffer ResolveDirectWindow(size_t window_index, u64 window_offset, - u64 size) const; + [[nodiscard]] std::optional ResolveDirectRange(u64 relative, + u64 size) const; [[nodiscard]] u64 CurrentTick() const noexcept; - [[nodiscard]] bool IsDirectUploadRetired(u64 tick); + [[nodiscard]] bool IsTickRetired(u64 tick); void InsertUploadMemoryBarrier() {} diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h index 6915a4d9af..f1eb6aa5b7 100644 --- a/src/video_core/texture_cache/image_base.h +++ b/src/video_core/texture_cache/image_base.h @@ -97,6 +97,9 @@ struct ImageBase { bool has_scaled = false; u64 direct_upload_tick = 0; bool direct_upload_blocked = false; + bool eviction_pending = false; + u64 eviction_tick = 0; + u64 eviction_modification_tick = 0; size_t channel = 0; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index e74a8baae0..a18314ae0a 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -137,6 +137,9 @@ void TextureCache

::RunGarbageCollector() { if (True(image.flags & ImageFlagBits::IsDecoding)) { return false; } + if (image.eviction_pending) { + return false; + } const bool must_download = IsDownloadable(image) && False(image.flags & ImageFlagBits::BadOverlap); if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) { return false; @@ -146,15 +149,14 @@ void TextureCache

::RunGarbageCollector() { return false; } --num_downloads; - if (TryDownloadToUnifiedMemory(image)) { - runtime.Finish(); - } else { - auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes); - const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info)); - image.DownloadMemory(map, copies); - runtime.Finish(); - SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span, swizzle_data_buffer); + if (StartEviction(image_id, image)) { + return false; } + auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes); + const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info)); + image.DownloadMemory(map, copies); + runtime.Finish(); + SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span, swizzle_data_buffer); } if (True(image.flags & ImageFlagBits::Tracked)) { UntrackImage(image, image_id); @@ -180,6 +182,7 @@ void TextureCache

::RunGarbageCollector() { template void TextureCache

::TickFrame() { + FinishEvictions(); // If we can obtain the memory info, use it instead of the estimate. if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); @@ -600,7 +603,7 @@ void TextureCache

::WriteMemory(DAddr cpu_addr, size_t size) { if (image.direct_upload_tick != 0) { const u64 upload_tick = image.direct_upload_tick; image.direct_upload_tick = 0; - if (!runtime.IsDirectUploadRetired(upload_tick)) { + if (!runtime.IsTickRetired(upload_tick)) { image.direct_upload_blocked = true; } } @@ -910,10 +913,6 @@ void TextureCache

::CommitAsyncFlushes() { bool any_none_dma = false; for (PendingDownload& download_info : download_ids) { if (download_info.is_swizzle) { - if (TryDownloadToUnifiedMemory(slot_images[download_info.object_id])) { - download_info.is_unified = true; - continue; - } total_size_bytes += Common::AlignUp(slot_images[download_info.object_id].unswizzled_size_bytes, 64); any_none_dma = true; @@ -924,7 +923,7 @@ void TextureCache

::CommitAsyncFlushes() { if (any_none_dma) { auto download_map = runtime.DownloadStagingBuffer(total_size_bytes, true); for (const PendingDownload& download_info : download_ids) { - if (download_info.is_swizzle && !download_info.is_unified) { + if (download_info.is_swizzle) { Image& image = slot_images[download_info.object_id]; const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info)); image.DownloadMemory(download_map, copies); @@ -956,9 +955,6 @@ void TextureCache

::PopAsyncFlushes() { auto download_map = std::move(async_buffers.front()); for (size_t i = download_ids.size(); i > 0; i--) { auto& download_info = download_ids[i - 1]; - if (download_info.is_unified) { - continue; - } auto& download_buffer = download_map[download_info.async_buffer_id]; if (download_info.is_swizzle) { const ImageBase& image = slot_images[download_info.object_id]; @@ -1192,16 +1188,12 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { } template -std::optional> TextureCache

::ResolveUnifiedImageWindow( +std::optional TextureCache

::ResolveUnifiedImageOffset( [[maybe_unused]] const ImageBase& image) { if constexpr (USE_UNIFIED_MEMORY) { if (image.guest_size_bytes == 0 || !runtime.IsUnifiedMemoryBindable()) { return std::nullopt; } - const u64 window_size = runtime.UnifiedMemoryWindowSize(); - if (window_size == 0) { - return std::nullopt; - } const u8* const first = gpu_memory->GetSpan(image.gpu_addr, image.guest_size_bytes); if (first == nullptr) { return std::nullopt; @@ -1216,11 +1208,7 @@ std::optional> TextureCache

::ResolveUnifiedImageWindow if (relative >= unified_size || unified_size - relative < image.guest_size_bytes) { return std::nullopt; } - const u64 local_offset = relative % window_size; - if (window_size - local_offset < image.guest_size_bytes) { - return std::nullopt; - } - return std::pair{static_cast(relative / window_size), local_offset}; + return relative; } else { return std::nullopt; } @@ -1232,13 +1220,12 @@ bool TextureCache

::TryUploadFromUnifiedMemory([[maybe_unused]] Image& image) if (image.direct_upload_blocked || !runtime.CanUploadImageDirectly(image.info)) { return false; } - const auto window = ResolveUnifiedImageWindow(image); - if (!window) { + const auto relative = ResolveUnifiedImageOffset(image); + if (!relative) { return false; } const auto swizzles = FullUploadSwizzles(image.info); - if (!runtime.UploadImageDirectly(image, window->first, window->second, - FixSmallVectorADL(swizzles))) { + if (!runtime.UploadImageDirectly(image, *relative, FixSmallVectorADL(swizzles))) { return false; } image.direct_upload_tick = runtime.CurrentTick(); @@ -1258,16 +1245,75 @@ bool TextureCache

::TryDownloadToUnifiedMemory([[maybe_unused]] Image& image) image.info.layer_stride != CalculateLayerStride(image.info)) { return false; } - const auto window = ResolveUnifiedImageWindow(image); - if (!window) { + const auto relative = ResolveUnifiedImageOffset(image); + if (!relative) { return false; } - return runtime.DownloadImageDirectly(image, window->first, window->second); + return runtime.DownloadImageDirectly(image, *relative); } else { return false; } } +template +bool TextureCache

::StartEviction([[maybe_unused]] ImageId image_id, + [[maybe_unused]] Image& image) { + if constexpr (requires { runtime.IsTickRetired(u64{}); }) { + auto staging = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes, true); + const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info)); + image.DownloadMemory(staging, copies); + eviction_staging.emplace_back(image_id, staging); + image.eviction_pending = true; + image.eviction_tick = runtime.CurrentTick(); + image.eviction_modification_tick = image.modification_tick; + return true; + } else { + return false; + } +} + +template +void TextureCache

::FinishEvictions() { + if constexpr (requires { runtime.IsTickRetired(u64{}); }) { + size_t index = 0; + while (index < eviction_staging.size()) { + const ImageId image_id = eviction_staging[index].first; + Image& image = slot_images[image_id]; + if (!runtime.IsTickRetired(image.eviction_tick)) { + ++index; + continue; + } + const bool unchanged = image.modification_tick == image.eviction_modification_tick; + if (unchanged && False(image.flags & ImageFlagBits::CpuModified)) { + const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info)); + SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, + eviction_staging[index].second.mapped_span, swizzle_data_buffer); + } + CancelEviction(image_id); + if (!unchanged) { + continue; + } + if (True(image.flags & ImageFlagBits::Tracked)) { + UntrackImage(image, image_id); + } + UnregisterImage(image_id); + DeleteImage(image_id, image.scale_tick > frame_tick + 5); + } + } +} + +template +void TextureCache

::CancelEviction(ImageId image_id) { + slot_images[image_id].eviction_pending = false; + const auto it = std::ranges::find_if( + eviction_staging, [image_id](const auto& entry) { return entry.first == image_id; }); + if (it == eviction_staging.end()) { + return; + } + async_buffers_death_ring.emplace_back(std::move(it->second)); + eviction_staging.erase(it); +} + template template void TextureCache

::UploadImageContents(Image& image, StagingBuffer& staging) { @@ -2434,6 +2480,9 @@ void TextureCache

::UntrackImage(ImageBase& image, ImageId image_id) { template void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { ImageBase& image = slot_images[image_id]; + if (image.eviction_pending) { + CancelEviction(image_id); + } if (image.HasScaled()) { total_used_memory -= GetScaledImageSizeBytes(image); } @@ -2621,6 +2670,9 @@ void TextureCache

::SynchronizeAliases(ImageId image_id) { template void TextureCache

::PrepareImage(ImageId image_id, bool is_modification, bool invalidate) { Image& image = slot_images[image_id]; + if (image.eviction_pending) { + CancelEviction(image_id); + } if (invalidate) { image.flags &= ~(ImageFlagBits::CpuModified | ImageFlagBits::GpuModified); if (False(image.flags & ImageFlagBits::Tracked)) { diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 28205cdbe6..9e80204377 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -310,13 +310,18 @@ private: void RefreshContents(Image& image, ImageId image_id); - [[nodiscard]] std::optional> ResolveUnifiedImageWindow( - const ImageBase& image); + [[nodiscard]] std::optional ResolveUnifiedImageOffset(const ImageBase& image); bool TryUploadFromUnifiedMemory(Image& image); bool TryDownloadToUnifiedMemory(Image& image); + bool StartEviction(ImageId image_id, Image& image); + + void FinishEvictions(); + + void CancelEviction(ImageId image_id); + /// Upload data from guest to an image template void UploadImageContents(Image& image, StagingBuffer& staging_buffer); @@ -476,7 +481,6 @@ private: bool is_swizzle; size_t async_buffer_id; Common::SlotId object_id; - bool is_unified = false; }; Common::SlotVector slot_images; @@ -494,6 +498,7 @@ private: std::vector uncommitted_async_buffers; std::deque> async_buffers; std::deque async_buffers_death_ring; + std::vector> eviction_staging; struct LRUItemParams { using ObjectType = ImageId; diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index cdd683f016..236f93aa27 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -920,6 +920,10 @@ FN_MAX_LIMIT_LIST return properties.maintenance4.maxBufferSize; } + u64 GetSparseAddressSpaceSize() const { + return properties.properties.limits.sparseAddressSpaceSize; + } + u64 GetMaxMemoryAllocationSize() const { return properties.maintenance3.maxMemoryAllocationSize; } diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 96690a19c6..38c679fb5a 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -251,6 +251,7 @@ bool HostMemoryImport::ImportHostPointer(void *base, size_t size) { return false; } window_size = candidate_window; + buffer_usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; const auto &logical = device.GetLogical(); const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties; @@ -331,6 +332,9 @@ bool HostMemoryImport::ImportHostPointer(void *base, size_t size) { windows.push_back(Window{ .memory = std::move(memory), .buffer = new_buffer, + .address = 0, + .size = window_len, + .memory_type = *type_index, }); imported_size += static_cast(window_len); } @@ -361,7 +365,7 @@ bool HostMemoryImport::ImportHardwareBuffers( window_size = hardware_buffer_window; base_offset = hardware_buffer_base; - const auto import_all = [&](VkBufferUsageFlags usage, bool want_address) { + const auto import_all = [&](VkBufferUsageFlags usage, bool want_address, bool dedicated) { for (size_t i = 0; i < hardware_buffers.size(); ++i) { const size_t offset = hardware_buffer_base + i * hardware_buffer_window; if (offset >= size) { @@ -424,13 +428,17 @@ bool HostMemoryImport::ImportHardwareBuffers( .image = VK_NULL_HANDLE, .buffer = new_buffer, }; + const void *memory_next = &import_info; + if (dedicated) { + memory_next = &dedicated_info; + } const VkMemoryAllocateFlagsInfo flags_info{ .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, - .pNext = &dedicated_info, + .pNext = memory_next, .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, .deviceMask = 0, }; - const void *alloc_next = &dedicated_info; + const void *alloc_next = memory_next; if (want_address) { alloc_next = &flags_info; } @@ -457,6 +465,8 @@ bool HostMemoryImport::ImportHardwareBuffers( .memory = std::move(memory), .buffer = new_buffer, .address = address, + .size = window_len, + .memory_type = *type_index, }); imported_size += static_cast(window_len); } @@ -487,14 +497,26 @@ bool HostMemoryImport::ImportHardwareBuffers( imported_size = 0; }; - bindable = import_all(shader_usage, want_address); - if (!bindable) { + mirror_capable = SupportsMirror(shader_usage); + buffer_usage = shader_usage; + bindable = import_all(shader_usage, want_address, !mirror_capable); + if (!bindable && mirror_capable) { + mirror_capable = false; reset_windows(); - bindable = import_all(minimal_usage, want_address); + bindable = import_all(shader_usage, want_address, true); + } + if (bindable) { + mirror_usage = shader_usage; } if (!bindable) { reset_windows(); - import_all(TransferUsage, false); + buffer_usage = minimal_usage; + bindable = import_all(minimal_usage, want_address, true); + } + if (!bindable) { + reset_windows(); + buffer_usage = TransferUsage; + import_all(TransferUsage, false, true); } if (windows.empty()) { window_size = 0; @@ -508,7 +530,181 @@ bool HostMemoryImport::ImportHardwareBuffers( #endif } +bool HostMemoryImport::SupportsMirror([[maybe_unused]] VkBufferUsageFlags usage) const { +#ifdef __ANDROID__ + if (!device.IsSparseBindingSupported()) { + return false; + } + const auto supports = [&](VkBufferCreateFlags flags) { + const VkExternalMemoryProperties properties = + device.GetPhysical().GetExternalBufferProperties( + flags, usage, + VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID); + const VkExternalMemoryFeatureFlags features = properties.externalMemoryFeatures; + return (features & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT) != 0 && + (features & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT) == 0; + }; + return supports(0) && supports(VK_BUFFER_CREATE_SPARSE_BINDING_BIT); +#else + return false; +#endif +} + +std::optional HostMemoryImport::ResolveRange( + VkDeviceSize relative, VkDeviceSize size) const noexcept { + if (window_size == 0 || size == 0 || relative >= imported_size || + imported_size - relative < size) { + return std::nullopt; + } + const size_t index = static_cast(relative / window_size); + const VkDeviceSize local_offset = relative % window_size; + if (index < windows.size() && windows[index].buffer != VK_NULL_HANDLE && + local_offset < windows[index].size && windows[index].size - local_offset >= size) { + return Range{ + .buffer = windows[index].buffer, + .address = windows[index].address, + .offset = local_offset, + }; + } + if (mirror_buffer != VK_NULL_HANDLE && relative < mirror_size && + mirror_size - relative >= size) { + return Range{ + .buffer = mirror_buffer, + .address = mirror_address, + .offset = relative, + }; + } + return std::nullopt; +} + +std::optional HostMemoryImport::ResolveViewMemory( + VkDeviceSize relative) const noexcept { + if (!mirror_capable || !bindable || window_size == 0) { + return std::nullopt; + } + const size_t index = static_cast(relative / window_size); + if (index >= windows.size()) { + return std::nullopt; + } + const Window &window = windows[index]; + const VkDeviceSize local_offset = relative % window_size; + if (window.buffer == VK_NULL_HANDLE || local_offset >= window.size) { + return std::nullopt; + } + return ViewMemory{ + .buffer = window.buffer, + .memory = *window.memory, + .offset = local_offset, + .available = window.size - local_offset, + .memory_type = window.memory_type, + }; +} + +void HostMemoryImport::CreateMirror(std::mutex &submit_mutex, VkDeviceSize max_size) { + if (!mirror_capable || mirror_usage == 0 || !bindable || window_size == 0) { + return; + } + size_t mirror_windows = 0; + while (mirror_windows < windows.size() && windows[mirror_windows].size == window_size) { + ++mirror_windows; + } + mirror_windows = (std::min)(mirror_windows, static_cast(max_size / window_size)); + const u64 max_buffer_size = device.GetMaxBufferSize(); + if (max_buffer_size != 0) { + const size_t max_windows = static_cast(max_buffer_size / window_size); + mirror_windows = (std::min)(mirror_windows, max_windows); + } + if (mirror_windows < 2) { + return; + } + const auto &logical = device.GetLogical(); + const VkDeviceSize total_size = static_cast(mirror_windows) * window_size; + const VkExternalMemoryBufferCreateInfo external_info{ + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, + .pNext = nullptr, + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID, + }; + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = &external_info, + .flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT, + .size = total_size, + .usage = mirror_usage, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + VkBuffer buffer{}; + if (logical.CreateBufferRaw(buffer_ci, &buffer) != VK_SUCCESS) { + return; + } + const VkMemoryRequirements requirements = logical.GetBufferMemoryRequirements(buffer); + bool compatible = requirements.alignment != 0 && + (window_size % requirements.alignment) == 0 && + requirements.size <= total_size; + for (size_t index = 0; index < mirror_windows && compatible; ++index) { + compatible = ((requirements.memoryTypeBits >> windows[index].memory_type) & 1u) != 0; + } + if (!compatible) { + logical.DestroyBufferRaw(buffer); + return; + } + std::vector binds; + binds.reserve(mirror_windows); + for (size_t index = 0; index < mirror_windows; ++index) { + binds.push_back(VkSparseMemoryBind{ + .resourceOffset = static_cast(index) * window_size, + .size = window_size, + .memory = *windows[index].memory, + .memoryOffset = 0, + .flags = 0, + }); + } + const VkSparseBufferMemoryBindInfo buffer_bind{ + .buffer = buffer, + .bindCount = static_cast(binds.size()), + .pBinds = binds.data(), + }; + const VkBindSparseInfo bind_info{ + .sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO, + .pNext = nullptr, + .waitSemaphoreCount = 0, + .pWaitSemaphores = nullptr, + .bufferBindCount = 1, + .pBufferBinds = &buffer_bind, + .imageOpaqueBindCount = 0, + .pImageOpaqueBinds = nullptr, + .imageBindCount = 0, + .pImageBinds = nullptr, + .signalSemaphoreCount = 0, + .pSignalSemaphores = nullptr, + }; + vk::Fence fence = logical.CreateFence(VkFenceCreateInfo{ + .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + }); + VkResult result = VK_ERROR_UNKNOWN; + { + std::scoped_lock lock{submit_mutex}; + result = device.GetGraphicsQueue().BindSparse(bind_info, *fence); + } + if (result != VK_SUCCESS) { + logical.DestroyBufferRaw(buffer); + return; + } + fence.Wait(); + mirror_buffer = buffer; + mirror_size = total_size; + if ((mirror_usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0) { + mirror_address = logical.GetBufferDeviceAddress(buffer); + } +} + HostMemoryImport::~HostMemoryImport() { + if (mirror_buffer != VK_NULL_HANDLE) { + device.GetLogical().DestroyBufferRaw(mirror_buffer); + } for (Window &window : windows) { if (window.buffer != VK_NULL_HANDLE) { device.GetLogical().DestroyBufferRaw(window.buffer); diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index 39bf26d40e..402b6ef35c 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -7,6 +7,8 @@ #pragma once #include +#include +#include #include #include @@ -134,13 +136,54 @@ namespace Vulkan { return bindable; } + [[nodiscard]] VkBufferUsageFlags GetUsage() const noexcept { + return buffer_usage; + } + + struct Range { + VkBuffer buffer{}; + VkDeviceAddress address{}; + VkDeviceSize offset{}; + }; + + [[nodiscard]] std::optional ResolveRange(VkDeviceSize relative, + VkDeviceSize size) const noexcept; + + struct ViewMemory { + VkBuffer buffer{}; + VkDeviceMemory memory{}; + VkDeviceSize offset{}; + VkDeviceSize available{}; + u32 memory_type{}; + }; + + [[nodiscard]] std::optional ResolveViewMemory( + VkDeviceSize relative) const noexcept; + + [[nodiscard]] VkBufferUsageFlags GetViewUsage() const noexcept { + if (!mirror_capable || !bindable) { + return 0; + } + return mirror_usage; + } + + void CreateMirror(std::mutex &submit_mutex, VkDeviceSize max_size); + + [[nodiscard]] VkDeviceSize GetMirrorSize() const noexcept { + return mirror_size; + } + private: struct Window { vk::DeviceMemory memory; VkBuffer buffer{}; VkDeviceAddress address{}; + VkDeviceSize size{}; + u32 memory_type{}; }; + [[nodiscard]] bool SupportsMirror(VkBufferUsageFlags usage) const; + bool ImportHostPointer(void *base, size_t size); bool ImportHardwareBuffers(std::span hardware_buffers, @@ -154,6 +197,12 @@ namespace Vulkan { size_t base_offset{}; bool foreign_ownership{}; bool bindable{}; + bool mirror_capable{}; + VkBufferUsageFlags buffer_usage{}; + VkBufferUsageFlags mirror_usage{}; + VkBuffer mirror_buffer{}; + VkDeviceAddress mirror_address{}; + VkDeviceSize mirror_size{}; }; /// Memory allocator container. diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index 9c86ee15ad..db1325a183 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -316,6 +316,7 @@ bool Load(VkInstance instance, InstanceDispatch& dld) noexcept { X(vkDestroyDebugUtilsMessengerEXT); X(vkDestroyDebugReportCallbackEXT); X(vkDestroySurfaceKHR); + X(vkGetPhysicalDeviceExternalBufferProperties); X(vkGetPhysicalDeviceFeatures2); X(vkGetPhysicalDeviceFormatProperties2); X(vkGetPhysicalDeviceProperties2); @@ -1038,6 +1039,28 @@ VkPhysicalDeviceMemoryProperties2 PhysicalDevice::GetMemoryProperties( return properties; } +VkExternalMemoryProperties PhysicalDevice::GetExternalBufferProperties( + VkBufferCreateFlags flags, VkBufferUsageFlags usage, + VkExternalMemoryHandleTypeFlagBits handle_type) const noexcept { + VkExternalBufferProperties properties{ + .sType = VK_STRUCTURE_TYPE_EXTERNAL_BUFFER_PROPERTIES, + .pNext = nullptr, + .externalMemoryProperties = {}, + }; + if (!dld->vkGetPhysicalDeviceExternalBufferProperties) { + return properties.externalMemoryProperties; + } + const VkPhysicalDeviceExternalBufferInfo info{ + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_BUFFER_INFO, + .pNext = nullptr, + .flags = flags, + .usage = usage, + .handleType = handle_type, + }; + dld->vkGetPhysicalDeviceExternalBufferProperties(physical_device, &info, &properties); + return properties.externalMemoryProperties; +} + u32 AvailableVersion(const InstanceDispatch& dld) noexcept { PFN_vkEnumerateInstanceVersion vkEnumerateInstanceVersion; if (!Proc(vkEnumerateInstanceVersion, dld, "vkEnumerateInstanceVersion")) { diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index d23f8fd8a2..e32181f5ff 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -178,6 +178,7 @@ struct InstanceDispatch { PFN_vkEnumerateDeviceExtensionProperties vkEnumerateDeviceExtensionProperties{}; PFN_vkEnumeratePhysicalDevices vkEnumeratePhysicalDevices{}; PFN_vkGetDeviceProcAddr vkGetDeviceProcAddr{}; + PFN_vkGetPhysicalDeviceExternalBufferProperties vkGetPhysicalDeviceExternalBufferProperties{}; PFN_vkGetPhysicalDeviceFeatures2 vkGetPhysicalDeviceFeatures2{}; PFN_vkGetPhysicalDeviceFormatProperties vkGetPhysicalDeviceFormatProperties{}; PFN_vkGetPhysicalDeviceFormatProperties2 vkGetPhysicalDeviceFormatProperties2{}; @@ -1243,6 +1244,10 @@ public: VkPhysicalDeviceMemoryProperties2 GetMemoryProperties( void* next_structures = nullptr) const noexcept; + VkExternalMemoryProperties GetExternalBufferProperties( + VkBufferCreateFlags flags, VkBufferUsageFlags usage, + VkExternalMemoryHandleTypeFlagBits handle_type) const noexcept; + private: VkPhysicalDevice physical_device = nullptr; const InstanceDispatch* dld = nullptr;