debuggerx/eden
1
0
Fork 0
mirror of https://git.eden-emu.dev/eden-emu/eden.git synced 2026-09-07 04:33:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

[vulkan] Changes to lru eviction and virtual buffer policy adjustments

Browse Source
This commit is contained in:
CamilleLaVey
2026-09-06 16:26:30 -04:00
parent 6634646824
commit 25c5056706
12 changed files with 234 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/video_core/buffer_cache/buffer_cache.h
+22 -22
View File
@@ -114,7 +114,7 @@ void BufferCache<P>::TickFrame() {
template <class P>
void BufferCache<P>::UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size) {
if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}); }) {
if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) {
virtual_ranges.Unmap(as_id, gpu_addr, size);
}
}
@@ -215,8 +215,8 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am
BufferId buffer_b;
do {
channel_state->has_deleted_buffers = false;
buffer_a = FindBuffer(*cpu_src_address, static_cast<u32>(amount));
buffer_b = FindBuffer(*cpu_dest_address, static_cast<u32>(amount));
buffer_a = FindBuffer(*cpu_src_address, static_cast<u32>(amount), false);
buffer_b = FindBuffer(*cpu_dest_address, static_cast<u32>(amount), false);
} while (channel_state->has_deleted_buffers);
auto& src_buffer = slot_buffers[buffer_a];
auto& dest_buffer = slot_buffers[buffer_b];
@@ -272,7 +272,7 @@ bool BufferCache<P>::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) {
ClearDownload(*cpu_dst_address, size);
gpu_modified_ranges.Subtract(*cpu_dst_address, size);
const BufferId buffer = FindBuffer(*cpu_dst_address, static_cast<u32>(size));
const BufferId buffer = FindBuffer(*cpu_dst_address, static_cast<u32>(size), false);
Buffer& dest_buffer = slot_buffers[buffer];
const u32 offset = dest_buffer.Offset(*cpu_dst_address);
runtime.ClearBuffer(dest_buffer, offset, size, value);
@@ -294,7 +294,7 @@ std::pair<typename P::Buffer*, u32> BufferCache<P>::ObtainBuffer(GPUVAddr gpu_ad
template <class P>
std::pair<typename P::Buffer*, u32> BufferCache<P>::ObtainCPUBuffer(
DAddr device_addr, u32 size, ObtainBufferSynchronize sync_info, ObtainBufferOperation post_op) {
const BufferId buffer_id = FindBuffer(device_addr, size);
const BufferId buffer_id = FindBuffer(device_addr, size, false);
Buffer& buffer = slot_buffers[buffer_id];
// synchronize op
@@ -1010,7 +1010,7 @@ void BufferCache<P>::ResolveMultiRangeStorage(Binding& binding, bool is_written,
std::vector<MultiRangeSegment>& pool) {
binding.segment_first = 0;
binding.segment_count = 0;
if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}); }) {
if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) {
if (binding.gpu_addr == 0 || binding.size == 0) {
return;
}
@@ -1046,7 +1046,7 @@ void BufferCache<P>::ResolveMultiRangeStorage(Binding& binding, bool is_written,
template <class P>
bool BufferCache<P>::BindMultiRangeStorage(const Binding& binding, bool is_written,
std::span<const MultiRangeSegment> pool) {
if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}); }) {
if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) {
if (binding.segment_count < 2) {
return false;
}
@@ -1069,7 +1069,7 @@ bool BufferCache<P>::BindMultiRangeStorage(const Binding& binding, bool is_writt
}
runtime.PushMultiRangeSource(buffer, offset, segment.size);
}
return runtime.BindMultiRangeStorageBuffer(key);
return runtime.BindMultiRangeStorageBuffer(key, is_written);
} else {
return false;
}
@@ -1319,11 +1319,11 @@ void BufferCache<P>::UpdateIndexBuffer() {
auto inline_index_size = static_cast<u32>(draw_state.inline_index_draw_indexes.size());
u32 buffer_size = Common::AlignUp(inline_index_size, CACHING_PAGESIZE);
if (inline_buffer_id == NULL_BUFFER_ID) [[unlikely]] {
inline_buffer_id = CreateBuffer(0, buffer_size);
inline_buffer_id = CreateBuffer(0, buffer_size, false);
}
if (slot_buffers[inline_buffer_id].SizeBytes() < buffer_size) [[unlikely]] {
slot_buffers.erase(inline_buffer_id);
inline_buffer_id = CreateBuffer(0, buffer_size);
inline_buffer_id = CreateBuffer(0, buffer_size, false);
}
channel_state->index_buffer = Binding{
.device_addr = 0,
@@ -1346,7 +1346,7 @@ void BufferCache<P>::UpdateIndexBuffer() {
channel_state->index_buffer = Binding{
.device_addr = *device_addr,
.size = size,
.buffer_id = FindBuffer(*device_addr, size),
.buffer_id = FindBuffer(*device_addr, size, false),
};
}
@@ -1383,7 +1383,7 @@ void BufferCache<P>::UpdateVertexBuffer(u32 index) {
if (!gpu_memory->IsWithinGPUAddressRange(gpu_addr_end) || size >= 64_MiB) {
size = static_cast<u32>(gpu_memory->MaxContinuousRange(gpu_addr_begin, size));
}
const BufferId buffer_id = FindBuffer(*device_addr, size);
const BufferId buffer_id = FindBuffer(*device_addr, size, false);
const Binding binding{
.device_addr = *device_addr,
.size = size,
@@ -1404,7 +1404,7 @@ void BufferCache<P>::UpdateDrawIndirect() {
binding = Binding{
.device_addr = *device_addr,
.size = static_cast<u32>(size),
.buffer_id = FindBuffer(*device_addr, static_cast<u32>(size)),
.buffer_id = FindBuffer(*device_addr, static_cast<u32>(size), false),
};
};
if (current_draw_indirect->include_count) {
@@ -1428,7 +1428,7 @@ void BufferCache<P>::UpdateUniformBuffers(size_t stage) {
channel_state->dirty_uniform_buffers[stage] |= 1U << index;
}
// Resolve buffer
binding.buffer_id = FindBuffer(binding.device_addr, binding.size);
binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false);
});
}
@@ -1437,7 +1437,7 @@ void BufferCache<P>::UpdateStorageBuffers(size_t stage) {
ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) {
// Resolve buffer
Binding& binding = channel_state->storage_buffers[stage][index];
const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size);
const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size, false);
binding.buffer_id = buffer_id;
const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0;
ResolveMultiRangeStorage(binding, is_written, graphics_segments);
@@ -1448,7 +1448,7 @@ template <class P>
void BufferCache<P>::UpdateTextureBuffers(size_t stage) {
ForEachEnabledBit(channel_state->enabled_texture_buffers[stage], [&](u32 index) {
Binding& binding = channel_state->texture_buffers[stage][index];
binding.buffer_id = FindBuffer(binding.device_addr, binding.size);
binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false);
});
}
@@ -1472,7 +1472,7 @@ void BufferCache<P>::UpdateTransformFeedbackBuffer(u32 index) {
channel_state->transform_feedback_buffers[index] = NULL_BINDING;
return;
}
const BufferId buffer_id = FindBuffer(*device_addr, size);
const BufferId buffer_id = FindBuffer(*device_addr, size, false);
channel_state->transform_feedback_buffers[index] = Binding{
.device_addr = *device_addr,
.size = size,
@@ -1494,7 +1494,7 @@ void BufferCache<P>::UpdateComputeUniformBuffers() {
binding.size = cbuf.size;
}
}
binding.buffer_id = FindBuffer(binding.device_addr, binding.size);
binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false);
});
}
@@ -1503,7 +1503,7 @@ void BufferCache<P>::UpdateComputeStorageBuffers() {
ForEachEnabledBit(channel_state->enabled_compute_storage_buffers, [&](u32 index) {
// Resolve buffer
Binding& binding = channel_state->compute_storage_buffers[index];
binding.buffer_id = FindBuffer(binding.device_addr, binding.size);
binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false);
const bool is_written =
((channel_state->written_compute_storage_buffers >> index) & 1) != 0;
ResolveMultiRangeStorage(binding, is_written, compute_segments);
@@ -1514,7 +1514,7 @@ template <class P>
void BufferCache<P>::UpdateComputeTextureBuffers() {
ForEachEnabledBit(channel_state->enabled_compute_texture_buffers, [&](u32 index) {
Binding& binding = channel_state->compute_texture_buffers[index];
binding.buffer_id = FindBuffer(binding.device_addr, binding.size);
binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false);
});
}
@@ -1845,7 +1845,7 @@ void BufferCache<P>::InlineMemoryImplementation(DAddr dest_address, size_t copy_
ClearDownload(dest_address, copy_size);
gpu_modified_ranges.Subtract(dest_address, copy_size);
BufferId buffer_id = FindBuffer(dest_address, static_cast<u32>(copy_size));
BufferId buffer_id = FindBuffer(dest_address, static_cast<u32>(copy_size), false);
auto& buffer = slot_buffers[buffer_id];
SynchronizeBuffer(buffer, dest_address, static_cast<u32>(copy_size));
@@ -2043,9 +2043,9 @@ Binding BufferCache<P>::StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index,
}
const Binding binding{
.device_addr = *aligned_device_addr,
.gpu_addr = aligned_gpu_addr,
.size = binding_size,
.buffer_id = BufferId{},
.gpu_addr = aligned_gpu_addr,
};
return binding;
}
src/video_core/buffer_cache/buffer_cache_base.h
+4 -5
View File
@@ -82,9 +82,9 @@ static constexpr u32 DEFAULT_SKIP_CACHE_SIZE = static_cast<u32>(4_KiB);
struct Binding {
DAddr device_addr{};
GPUVAddr gpu_addr{};
u32 size{};
BufferId buffer_id;
GPUVAddr gpu_addr{};
u32 segment_first{};
u32 segment_count{};
};
@@ -432,8 +432,7 @@ private:
void MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size);
[[nodiscard]] BufferId FindBuffer(DAddr device_addr, u32 size,
bool sparse_compatible = false);
[[nodiscard]] BufferId FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible);
void WaitForGpuFenceIfNeeded(Buffer& buffer);
@@ -442,7 +441,7 @@ private:
void JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, bool accumulate_stream_score);
[[nodiscard]] BufferId CreateBuffer(DAddr device_addr, u32 wanted_size,
bool sparse_compatible = false);
bool sparse_compatible);
void Register(BufferId buffer_id);
@@ -533,10 +532,10 @@ private:
using TickType = u64;
};
Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
u64 frame_tick = 0;
VirtualRangeCache virtual_ranges;
std::vector<MultiRangeSegment> graphics_segments;
std::vector<MultiRangeSegment> compute_segments;
u64 frame_tick = 0;
u64 total_used_memory = 0;
u64 minimum_memory = 0;
u64 critical_memory = 0;
src/video_core/buffer_cache/virtual_range_cache.h
+28 -18
View File
@@ -7,12 +7,12 @@
#include <limits>
#include <mutex>
#include <optional>
#include <unordered_map>
#include <vector>
#include <boost/container/small_vector.hpp>
#include "common/common_types.h"
#include "common/container/unordered_map.h"
#include "video_core/memory_manager.h"
namespace VideoCommon {
@@ -27,10 +27,16 @@ using VirtualSegments = boost::container::small_vector<VirtualSegment, 8>;
class VirtualRangeCache {
public:
static constexpr size_t MAX_ENTRIES = 8192;
static constexpr size_t MAX_DEFERRED = 4096;
const VirtualSegments* Query(Tegra::MemoryManager& memory, GPUVAddr gpu_addr, u32 size) {
if (has_deferred.load(std::memory_order_acquire)) {
ApplyDeferred();
}
if (entries.size() > MAX_ENTRIES) {
entries.clear();
}
const size_t as_id = memory.GetID();
const u64 key = MakeKey(as_id, gpu_addr);
const auto it = entries.find(key);
@@ -38,7 +44,7 @@ public:
it->second.gpu_addr == gpu_addr && it->second.size == size) {
return &it->second.segments;
}
Entry entry;
Entry entry{};
entry.as_id = as_id;
entry.gpu_addr = gpu_addr;
entry.size = size;
@@ -87,30 +93,26 @@ public:
return;
}
}
deferred.push_back(DeferredUnmap{
.as_id = as_id,
.gpu_addr = gpu_addr,
.size = size,
});
if (deferred.size() >= MAX_DEFERRED) {
deferred.clear();
deferred_overflow = true;
} else {
deferred.push_back(DeferredUnmap{
.as_id = as_id,
.gpu_addr = gpu_addr,
.size = size,
});
}
}
has_deferred.store(true, std::memory_order_release);
}
void Clear() {
{
std::scoped_lock lock{deferred_mutex};
deferred.clear();
}
has_deferred.store(false, std::memory_order_release);
entries.clear();
}
private:
struct Entry {
VirtualSegments segments;
size_t as_id{};
GPUVAddr gpu_addr{};
u32 size{};
VirtualSegments segments;
};
struct DeferredUnmap {
@@ -125,10 +127,17 @@ private:
void ApplyDeferred() {
std::vector<DeferredUnmap> pending;
bool overflow = false;
{
std::scoped_lock lock{deferred_mutex};
has_deferred.store(false, std::memory_order_release);
pending.swap(deferred);
overflow = deferred_overflow;
deferred_overflow = false;
}
if (overflow) {
entries.clear();
return;
}
if (pending.empty() || entries.empty()) {
return;
@@ -154,10 +163,11 @@ private:
}
}
std::unordered_map<u64, Entry> entries;
::Common::unordered_map<u64, Entry> entries;
std::vector<DeferredUnmap> deferred;
std::mutex deferred_mutex;
std::atomic<bool> has_deferred{false};
bool deferred_overflow{};
};
} // namespace VideoCommon
src/video_core/renderer_opengl/gl_buffer_cache.h
+1 -1
View File
@@ -24,7 +24,7 @@ class BufferCacheRuntime;
class Buffer : public VideoCommon::BufferBase {
public:
explicit Buffer(BufferCacheRuntime&, DAddr cpu_addr, u64 size_bytes,
bool sparse_compatible = false);
bool sparse_compatible);
explicit Buffer(BufferCacheRuntime&, VideoCommon::NullBufferParams);
void ImmediateUpload(size_t offset, std::span<const u8> data) noexcept;
src/video_core/renderer_vulkan/vk_buffer_cache.cpp
+11 -6
View File
@@ -357,7 +357,7 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m
staging_pool{staging_pool_}, guest_descriptor_queue{guest_descriptor_queue_},
quad_index_pass(device, scheduler, descriptor_pool, staging_pool,
compute_pass_descriptor_queue),
multi_range_buffers(device_, memory_allocator_, scheduler_) {
multi_range_buffers(device_) {
const VkDriverIdKHR driver_id = device.GetDriverID();
limit_dynamic_storage_buffers = driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY ||
driver_id == VK_DRIVER_ID_ARM_PROPRIETARY;
@@ -411,6 +411,7 @@ u32 BufferCacheRuntime::GetStorageBufferAlignment() const {
}
void BufferCacheRuntime::TickFrame(Common::SlotVector<Buffer>& slot_buffers) noexcept {
multi_range_buffers.TickFrame(scheduler);
for (auto it = slot_buffers.begin(); it != slot_buffers.end(); it++) {
if (scheduler.IsFree(it->LastUsageTick())) {
it->ResetUsageTracking();
@@ -545,22 +546,26 @@ void BufferCacheRuntime::ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t si
});
}
bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key) {
bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key, bool is_written) {
if (multi_range_sources.empty() || multi_range_total == 0) {
return false;
}
const MultiRangeRef ref = multi_range_buffers.Get(key, multi_range_sources, multi_range_total);
const MultiRangeRef ref = multi_range_buffers.Get(device, scheduler, memory_allocator, key,
multi_range_sources, multi_range_total);
if (ref.handle == VK_NULL_HANDLE) {
return false;
}
if (is_written && !ref.sparse) {
return false;
}
if (ref.needs_gather) {
PreCopyBarrier();
VkDeviceSize dst_offset = 0;
for (const MultiRangeSource& source : multi_range_sources) {
const std::array<VideoCommon::BufferCopy, 1> copy{VideoCommon::BufferCopy{
.src_offset = static_cast<u64>(source.offset),
.dst_offset = static_cast<u64>(dst_offset),
.size = static_cast<size_t>(source.size),
.src_offset = u64(source.offset),
.dst_offset = u64(dst_offset),
.size = size_t(source.size),
}};
CopyBuffer(ref.handle, source.handle, copy, false);
dst_offset += source.size;
src/video_core/renderer_vulkan/vk_buffer_cache.h
+7 -6
View File
@@ -35,7 +35,7 @@ class Buffer : public VideoCommon::BufferBase {
public:
explicit Buffer(BufferCacheRuntime&, VideoCommon::NullBufferParams null_params);
explicit Buffer(BufferCacheRuntime& runtime, VAddr cpu_addr_, u64 size_bytes_,
bool sparse_compatible_ = false);
bool sparse_compatible_);
[[nodiscard]] VkBufferView View(u32 offset, u32 size, VideoCore::Surface::PixelFormat format);
@@ -169,14 +169,14 @@ public:
}
[[nodiscard]] VkDeviceSize SparseAlignmentFor(bool sparse_compatible) const noexcept {
if (!sparse_compatible || !multi_range_buffers.UsesSparse()) {
if (!sparse_compatible || !multi_range_buffers.use_sparse) {
return 0;
}
return multi_range_buffers.BlockSize();
return multi_range_buffers.block_size;
}
[[nodiscard]] bool PrefersSparseSources() const noexcept {
return multi_range_buffers.UsesSparse();
return multi_range_buffers.use_sparse;
}
void ResetMultiRange() noexcept {
@@ -192,19 +192,20 @@ public:
.memory_offset = location.offset,
.offset = offset,
.size = size,
.write_tick = buffer.getWriteTick(),
.memory_type = location.memory_type,
});
multi_range_total += size;
}
bool BindMultiRangeStorageBuffer(u64 key);
bool BindMultiRangeStorageBuffer(u64 key, bool is_written);
void InvalidateMultiRange(u64 key) {
multi_range_buffers.Invalidate(key);
}
void OnBufferDeleted(const Buffer& buffer) {
multi_range_buffers.DropOwner(buffer.Handle());
multi_range_buffers.DropOwner(scheduler, buffer.Handle());
}
void BindUniformBuffer(const Buffer& buffer, u32 offset, u32 size) {
src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp
+101 -88
View File
@@ -1,7 +1,9 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#include <algorithm>
#include <mutex>
#include <utility>
#include "video_core/renderer_vulkan/vk_multi_range_buffer.h"
#include "video_core/renderer_vulkan/vk_scheduler.h"
@@ -9,10 +11,7 @@
namespace Vulkan {
MultiRangeBufferCache::MultiRangeBufferCache(const Device& device_,
MemoryAllocator& memory_allocator_,
Scheduler& scheduler_)
: device{device_}, memory_allocator{memory_allocator_}, scheduler{scheduler_} {
MultiRangeBufferCache::MultiRangeBufferCache(const Device& device) {
sparse_usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT |
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
if (device.IsBufferDeviceAddressSupported()) {
@@ -22,7 +21,7 @@ MultiRangeBufferCache::MultiRangeBufferCache(const Device& device_,
return;
}
u32 memory_type_bits = 0;
const VkDeviceSize queried = QueryBlockSize(memory_type_bits);
const VkDeviceSize queried = QueryBlockSize(device, memory_type_bits);
if (queried == 0 || memory_type_bits == 0) {
return;
}
@@ -31,22 +30,8 @@ MultiRangeBufferCache::MultiRangeBufferCache(const Device& device_,
use_sparse = true;
}
MultiRangeBufferCache::~MultiRangeBufferCache() {
const VkDevice logical = *device.GetLogical();
const auto& dld = device.GetDispatchLoader();
for (auto& [key, entry] : entries) {
if (entry.sparse_handle != VK_NULL_HANDLE) {
dld.vkDestroyBuffer(logical, entry.sparse_handle, nullptr);
}
}
entries.clear();
for (const Retired& item : retired) {
dld.vkDestroyBuffer(logical, item.handle, nullptr);
}
retired.clear();
}
VkDeviceSize MultiRangeBufferCache::QueryBlockSize(u32& memory_type_bits) const {
VkDeviceSize MultiRangeBufferCache::QueryBlockSize(const Device& device,
u32& memory_type_bits) const {
const VkDevice logical = *device.GetLogical();
const auto& dld = device.GetDispatchLoader();
const VkBufferCreateInfo probe_ci{
@@ -63,6 +48,7 @@ VkDeviceSize MultiRangeBufferCache::QueryBlockSize(u32& memory_type_bits) const
if (dld.vkCreateBuffer(logical, &probe_ci, nullptr, &probe) != VK_SUCCESS) {
return 0;
}
const SparseBuffer owned{probe, logical, dld};
const VkBufferMemoryRequirementsInfo2 reqs_info{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2,
.pNext = nullptr,
@@ -74,7 +60,6 @@ VkDeviceSize MultiRangeBufferCache::QueryBlockSize(u32& memory_type_bits) const
.memoryRequirements = {},
};
dld.vkGetBufferMemoryRequirements2(logical, &reqs_info, &reqs2);
dld.vkDestroyBuffer(logical, probe, nullptr);
memory_type_bits = reqs2.memoryRequirements.memoryTypeBits;
return reqs2.memoryRequirements.alignment;
}
@@ -86,40 +71,36 @@ u64 MultiRangeBufferCache::HashSources(std::span<const MultiRangeSource> sources
hash *= 0x100000001b3ULL;
};
for (const MultiRangeSource& source : sources) {
mix(reinterpret_cast<u64>(source.handle));
mix(static_cast<u64>(source.offset));
mix(static_cast<u64>(source.size));
mix(u64(source.handle));
mix(u64(source.offset));
mix(u64(source.size));
}
return hash;
}
u64 MultiRangeBufferCache::HashContent(std::span<const MultiRangeSource> sources) const {
u64 hash = 0xcbf29ce484222325ULL;
for (const MultiRangeSource& source : sources) {
hash ^= source.write_tick;
hash *= 0x100000001b3ULL;
}
return hash;
}
bool MultiRangeBufferCache::CanBindSparse(std::span<const MultiRangeSource> sources) const {
if (!UsesSparse()) {
return false;
}
for (const MultiRangeSource& source : sources) {
if (source.memory == VK_NULL_HANDLE) {
return false;
}
if (source.memory_type >= 32) {
return false;
}
if (((sparse_memory_type_bits >> source.memory_type) & 1) == 0) {
return false;
}
const VkDeviceSize memory_offset = source.memory_offset + source.offset;
if ((memory_offset % block_size) != 0) {
return false;
}
if ((source.size % block_size) != 0) {
return false;
}
}
return true;
return use_sparse &&
std::none_of(sources.begin(), sources.end(),
[block = block_size, bits = sparse_memory_type_bits](auto const& e) {
const VkDeviceSize memory_offset = e.memory_offset + e.offset;
return e.memory == VK_NULL_HANDLE || e.memory_type >= 32 ||
((bits >> e.memory_type) & 1) == 0 ||
(memory_offset % block) != 0 || (e.size % block) != 0;
});
}
VkBuffer MultiRangeBufferCache::CreateSparse(std::span<const MultiRangeSource> sources,
VkDeviceSize total) {
SparseBuffer MultiRangeBufferCache::CreateSparse(const Device& device, Scheduler& scheduler,
std::span<const MultiRangeSource> sources,
VkDeviceSize total) {
const VkDevice logical = *device.GetLogical();
const auto& dld = device.GetDispatchLoader();
const VkBufferCreateInfo buffer_ci{
@@ -132,10 +113,11 @@ VkBuffer MultiRangeBufferCache::CreateSparse(std::span<const MultiRangeSource> s
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = nullptr,
};
VkBuffer handle{};
if (dld.vkCreateBuffer(logical, &buffer_ci, nullptr, &handle) != VK_SUCCESS) {
return VK_NULL_HANDLE;
VkBuffer raw{};
if (dld.vkCreateBuffer(logical, &buffer_ci, nullptr, &raw) != VK_SUCCESS) {
return SparseBuffer{};
}
SparseBuffer handle{raw, logical, dld};
std::vector<VkSparseMemoryBind> binds;
binds.reserve(sources.size());
VkDeviceSize resource_offset = 0;
@@ -150,7 +132,7 @@ VkBuffer MultiRangeBufferCache::CreateSparse(std::span<const MultiRangeSource> s
resource_offset += source.size;
}
const VkSparseBufferMemoryBindInfo buffer_bind{
.buffer = handle,
.buffer = raw,
.bindCount = static_cast<u32>(binds.size()),
.pBinds = binds.data(),
};
@@ -180,31 +162,34 @@ VkBuffer MultiRangeBufferCache::CreateSparse(std::span<const MultiRangeSource> s
bind_result = device.GetGraphicsQueue().BindSparse(bind_info, *fence);
}
if (bind_result != VK_SUCCESS) {
dld.vkDestroyBuffer(logical, handle, nullptr);
return VK_NULL_HANDLE;
return SparseBuffer{};
}
fence.Wait();
return handle;
}
void MultiRangeBufferCache::DestroySparse(VkBuffer handle) {
if (handle == VK_NULL_HANDLE) {
return;
bool MultiRangeBufferCache::DestroySparse(Scheduler& scheduler, SparseBuffer&& handle) {
if (!handle) {
return true;
}
if (retired.size() == retired.capacity()) {
DrainRetired(scheduler);
}
if (retired.size() == retired.capacity()) {
return false;
}
retired.push_back(Retired{
.handle = handle,
.handle = std::move(handle),
.tick = scheduler.CurrentTick(),
});
return true;
}
void MultiRangeBufferCache::DrainRetired() {
const VkDevice logical = *device.GetLogical();
const auto& dld = device.GetDispatchLoader();
void MultiRangeBufferCache::DrainRetired(Scheduler& scheduler) {
size_t index = 0;
while (index < retired.size()) {
if (scheduler.IsFree(retired[index].tick)) {
dld.vkDestroyBuffer(logical, retired[index].handle, nullptr);
retired[index] = retired.back();
retired[index] = std::move(retired.back());
retired.pop_back();
} else {
++index;
@@ -212,48 +197,64 @@ void MultiRangeBufferCache::DrainRetired() {
}
}
MultiRangeRef MultiRangeBufferCache::Get(u64 key, std::span<const MultiRangeSource> sources,
MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& scheduler,
MemoryAllocator& memory_allocator, u64 key,
std::span<const MultiRangeSource> sources,
VkDeviceSize total) {
if (sources.empty() || total == 0) {
return MultiRangeRef{};
}
if (!retired.empty()) {
DrainRetired();
DrainRetired(scheduler);
}
const u64 geometry = HashSources(sources);
const u64 content = HashContent(sources);
const auto it = entries.find(key);
if (it != entries.end() && it->second.geometry == geometry && it->second.size == total) {
Entry& entry = it->second;
entry.frame = frame_tick;
entry.gpu_tick = scheduler.CurrentTick();
if (entry.content != content) {
entry.content = content;
entry.dirty = true;
}
MultiRangeRef ref{
.handle = entry.sparse_handle,
.handle = *entry.sparse_handle,
.address = entry.address,
.size = entry.size,
.sparse = true,
.needs_gather = false,
};
if (entry.sparse_handle == VK_NULL_HANDLE) {
if (!entry.sparse_handle) {
ref.handle = *entry.gathered;
ref.sparse = false;
ref.needs_gather = entry.dirty;
}
return ref;
}
if (it != entries.end()) {
DestroySparse(it->second.sparse_handle);
if (!DestroySparse(scheduler, std::move(it->second.sparse_handle))) {
return MultiRangeRef{};
}
entries.erase(it);
}
Entry entry;
Entry entry{};
entry.geometry = geometry;
entry.content = content;
entry.size = total;
entry.frame = frame_tick;
entry.gpu_tick = scheduler.CurrentTick();
if (CanBindSparse(sources)) {
entry.sparse_handle = CreateSparse(sources, total);
if (entry.sparse_handle != VK_NULL_HANDLE) {
entry.sparse_handle = CreateSparse(device, scheduler, sources, total);
if (entry.sparse_handle) {
entry.owners.reserve(sources.size());
for (const MultiRangeSource& source : sources) {
entry.owners.push_back(source.handle);
}
}
}
if (entry.sparse_handle == VK_NULL_HANDLE) {
if (!entry.sparse_handle) {
VkBufferUsageFlags flags = VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
VK_BUFFER_USAGE_TRANSFER_DST_BIT |
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
@@ -274,21 +275,23 @@ MultiRangeRef MultiRangeBufferCache::Get(u64 key, std::span<const MultiRangeSour
entry.dirty = true;
}
if (device.IsBufferDeviceAddressSupported()) {
VkBuffer address_handle = entry.sparse_handle;
if (address_handle == VK_NULL_HANDLE) {
VkBuffer address_handle = *entry.sparse_handle;
if (!entry.sparse_handle) {
address_handle = *entry.gathered;
}
entry.address = device.GetLogical().GetBufferDeviceAddress(address_handle);
}
MultiRangeRef ref{
.handle = entry.sparse_handle,
.handle = *entry.sparse_handle,
.address = entry.address,
.size = entry.size,
.sparse = true,
.needs_gather = false,
};
if (entry.sparse_handle == VK_NULL_HANDLE) {
if (!entry.sparse_handle) {
ref.handle = *entry.gathered;
ref.sparse = false;
ref.needs_gather = true;
}
entries.emplace(key, std::move(entry));
@@ -296,13 +299,12 @@ MultiRangeRef MultiRangeBufferCache::Get(u64 key, std::span<const MultiRangeSour
}
void MultiRangeBufferCache::MarkGathered(u64 key) {
const auto it = entries.find(key);
if (it != entries.end()) {
if (auto const it = entries.find(key); it != entries.end()) {
it->second.dirty = false;
}
}
void MultiRangeBufferCache::DropOwner(VkBuffer owner) {
void MultiRangeBufferCache::DropOwner(Scheduler& scheduler, VkBuffer owner) {
if (owner == VK_NULL_HANDLE) {
return;
}
@@ -315,8 +317,7 @@ void MultiRangeBufferCache::DropOwner(VkBuffer owner) {
break;
}
}
if (owned) {
DestroySparse(entry.sparse_handle);
if (owned && DestroySparse(scheduler, std::move(entry.sparse_handle))) {
it = entries.erase(it);
} else {
++it;
@@ -325,17 +326,29 @@ void MultiRangeBufferCache::DropOwner(VkBuffer owner) {
}
void MultiRangeBufferCache::Invalidate(u64 key) {
const auto it = entries.find(key);
if (it != entries.end()) {
if (auto const it = entries.find(key); it != entries.end()) {
it->second.dirty = true;
}
}
void MultiRangeBufferCache::Clear() {
for (auto& [key, entry] : entries) {
DestroySparse(entry.sparse_handle);
void MultiRangeBufferCache::TickFrame(Scheduler& scheduler) {
++frame_tick;
if (!retired.empty()) {
DrainRetired(scheduler);
}
if (entries.empty()) {
return;
}
for (auto it = entries.begin(); it != entries.end();) {
Entry& entry = it->second;
const bool expired =
frame_tick - entry.frame > FRAMES_TO_LIVE && scheduler.IsFree(entry.gpu_tick);
if (expired && DestroySparse(scheduler, std::move(entry.sparse_handle))) {
it = entries.erase(it);
} else {
++it;
}
}
entries.clear();
}
} // namespace Vulkan
src/video_core/renderer_vulkan/vk_multi_range_buffer.h
+37 -32
View File
@@ -4,15 +4,20 @@
#pragma once
#include <span>
#include <unordered_map>
#include <vector>
#include <boost/container/static_vector.hpp>
#include "common/common_funcs.h"
#include "common/common_types.h"
#include "common/container/unordered_map.h"
#include "video_core/vulkan_common/vulkan_memory_allocator.h"
#include "video_core/vulkan_common/vulkan_wrapper.h"
namespace Vulkan {
using SparseBuffer = vk::Handle<VkBuffer, VkDevice, vk::DeviceDispatch>;
class Device;
class Scheduler;
@@ -22,6 +27,7 @@ struct MultiRangeSource {
VkDeviceSize memory_offset{};
VkDeviceSize offset{};
VkDeviceSize size{};
u64 write_tick{};
u32 memory_type{};
};
@@ -29,77 +35,76 @@ struct MultiRangeRef {
VkBuffer handle{};
VkDeviceAddress address{};
VkDeviceSize size{};
bool sparse{};
bool needs_gather{};
};
class MultiRangeBufferCache final {
public:
static constexpr VkDeviceSize DEFAULT_BLOCK_SIZE = 64 * 1024;
static constexpr u64 FRAMES_TO_LIVE = 120;
static constexpr size_t MAX_RETIRED = 256;
explicit MultiRangeBufferCache(const Device& device_, MemoryAllocator& memory_allocator_,
Scheduler& scheduler_);
~MultiRangeBufferCache();
explicit MultiRangeBufferCache(const Device& device);
MultiRangeBufferCache(const MultiRangeBufferCache&) = delete;
MultiRangeBufferCache& operator=(const MultiRangeBufferCache&) = delete;
YUZU_NON_COPYABLE(MultiRangeBufferCache);
[[nodiscard]] bool UsesSparse() const noexcept {
return use_sparse;
}
[[nodiscard]] VkDeviceSize BlockSize() const noexcept {
return block_size;
}
[[nodiscard]] MultiRangeRef Get(u64 key, std::span<const MultiRangeSource> sources,
[[nodiscard]] MultiRangeRef Get(const Device& device, Scheduler& scheduler,
MemoryAllocator& memory_allocator, u64 key,
std::span<const MultiRangeSource> sources,
VkDeviceSize total);
void MarkGathered(u64 key);
void Invalidate(u64 key);
void DropOwner(VkBuffer owner);
void DropOwner(Scheduler& scheduler, VkBuffer owner);
void Clear();
void TickFrame(Scheduler& scheduler);
VkDeviceSize block_size{DEFAULT_BLOCK_SIZE};
bool use_sparse{};
private:
struct Retired {
VkBuffer handle{};
SparseBuffer handle;
u64 tick{};
};
struct Entry {
vk::Buffer gathered;
VkBuffer sparse_handle{};
SparseBuffer sparse_handle;
std::vector<VkBuffer> owners;
VkDeviceAddress address{};
VkDeviceSize size{};
u64 geometry{};
u64 content{};
u64 frame{};
u64 gpu_tick{};
bool dirty{true};
std::vector<VkBuffer> owners;
};
[[nodiscard]] u64 HashSources(std::span<const MultiRangeSource> sources) const;
[[nodiscard]] u64 HashContent(std::span<const MultiRangeSource> sources) const;
[[nodiscard]] bool CanBindSparse(std::span<const MultiRangeSource> sources) const;
[[nodiscard]] VkBuffer CreateSparse(std::span<const MultiRangeSource> sources,
VkDeviceSize total);
[[nodiscard]] SparseBuffer CreateSparse(const Device& device, Scheduler& scheduler,
std::span<const MultiRangeSource> sources,
VkDeviceSize total);
[[nodiscard]] VkDeviceSize QueryBlockSize(u32& memory_type_bits) const;
[[nodiscard]] VkDeviceSize QueryBlockSize(const Device& device, u32& memory_type_bits) const;
void DestroySparse(VkBuffer handle);
bool DestroySparse(Scheduler& scheduler, SparseBuffer&& handle);
void DrainRetired();
void DrainRetired(Scheduler& scheduler);
const Device& device;
MemoryAllocator& memory_allocator;
Scheduler& scheduler;
bool use_sparse{};
VkDeviceSize block_size{DEFAULT_BLOCK_SIZE};
::Common::unordered_map<u64, Entry> entries;
boost::container::static_vector<Retired, MAX_RETIRED> retired;
u64 frame_tick{};
u32 sparse_memory_type_bits{};
VkBufferUsageFlags sparse_usage{};
std::unordered_map<u64, Entry> entries;
std::vector<Retired> retired;
};
} // namespace Vulkan
src/video_core/vulkan_common/vulkan_device.h
+1 -1
View File
@@ -1150,8 +1150,8 @@ private:
bool owns_static_pipeline_cache{};
u32 instance_version{}; ///< Vulkan instance version.
u32 graphics_family{}; ///< Main graphics queue family index.
bool graphics_family_sparse_binding{};
u32 present_family{}; ///< Main present queue family index.
bool graphics_family_sparse_binding{};
struct Extensions {
#define EXTENSION(prefix, macro_name, var_name) bool var_name{};
src/video_core/vulkan_common/vulkan_memory_allocator.cpp
+13 -4
View File
@@ -275,9 +275,13 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa
const std::span<u8> mapped_data = data ? std::span<u8>{data, ci.size} : std::span<u8>{};
const bool is_coherent = (property_flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
return vk::Buffer(handle, *device.GetLogical(), allocator, allocation, mapped_data,
is_coherent,
device.GetDispatchLoader());
const vk::MemoryLocation location{
.memory = alloc_info.deviceMemory,
.offset = alloc_info.offset,
.memory_type = alloc_info.memoryType,
};
return vk::Buffer(handle, *device.GetLogical(), allocator, allocation, mapped_data, is_coherent,
location, device.GetDispatchLoader());
}
vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage,
@@ -321,8 +325,13 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa
}
const bool is_coherent = (property_flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
const vk::MemoryLocation location{
.memory = alloc_info.deviceMemory,
.offset = alloc_info.offset,
.memory_type = alloc_info.memoryType,
};
return vk::Buffer(handle, *device.GetLogical(), allocator, allocation, mapped_data, is_coherent,
device.GetDispatchLoader());
location, device.GetDispatchLoader());
}
MemoryCommit MemoryAllocator::Commit(const VkMemoryRequirements &reqs, MemoryUsage usage)
src/video_core/vulkan_common/vulkan_wrapper.cpp
-13
View File
@@ -540,19 +540,6 @@ void Buffer::SetObjectNameEXT(const char* name) const {
SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_BUFFER, name);
}
MemoryLocation Buffer::Location() const noexcept {
if (!allocation) {
return MemoryLocation{};
}
VmaAllocationInfo info{};
vmaGetAllocationInfo(allocator, allocation, &info);
return MemoryLocation{
.memory = info.deviceMemory,
.offset = info.offset,
.memory_type = info.memoryType,
};
}
void Buffer::Release() const noexcept {
if (handle) {
vmaDestroyBuffer(allocator, handle, allocation);
src/video_core/vulkan_common/vulkan_wrapper.h
+9 -4
View File
@@ -751,9 +751,10 @@ class Buffer {
public:
explicit Buffer(VkBuffer handle_, VkDevice owner_, VmaAllocator allocator_,
VmaAllocation allocation_, std::span<u8> mapped_, bool is_coherent_,
const DeviceDispatch& dld_) noexcept
MemoryLocation location_, const DeviceDispatch& dld_) noexcept
: handle{handle_}, owner{owner_}, allocator{allocator_},
allocation{allocation_}, mapped{mapped_}, is_coherent{is_coherent_}, dld{&dld_} {}
allocation{allocation_}, mapped{mapped_}, is_coherent{is_coherent_},
location{location_}, dld{&dld_} {}
Buffer() = default;
Buffer(const Buffer&) = delete;
@@ -762,7 +763,7 @@ public:
Buffer(Buffer&& rhs) noexcept
: handle{std::exchange(rhs.handle, VkBuffer{})}, owner{rhs.owner}, allocator{rhs.allocator},
allocation{rhs.allocation}, mapped{rhs.mapped},
is_coherent{rhs.is_coherent}, dld{rhs.dld} {}
is_coherent{rhs.is_coherent}, location{rhs.location}, dld{rhs.dld} {}
Buffer& operator=(Buffer&& rhs) noexcept {
Release();
@@ -772,6 +773,7 @@ public:
allocation = rhs.allocation;
mapped = rhs.mapped;
is_coherent = rhs.is_coherent;
location = rhs.location;
dld = rhs.dld;
return *this;
}
@@ -818,7 +820,9 @@ public:
void SetObjectNameEXT(const char* name) const;
MemoryLocation Location() const noexcept;
MemoryLocation Location() const noexcept {
return location;
}
private:
void Release() const noexcept;
@@ -828,6 +832,7 @@ private:
VmaAllocator allocator = nullptr;
VmaAllocation allocation = nullptr;
std::span<u8> mapped = {};
MemoryLocation location{};
bool is_coherent = false;
const DeviceDispatch* dld = nullptr;
};