diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index 4f979a162a..e1689390c0 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -1004,37 +1004,66 @@ void BufferCache
::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32
}
template
-bool BufferCache::BindMultiRangeStorage(const Binding& binding, bool is_written) {
+void BufferCache
::ResolveMultiRangeStorage(Binding& binding, bool is_written,
+ std::vector& pool) {
+ binding.segment_first = 0;
+ binding.segment_count = 0;
if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}); }) {
if (binding.gpu_addr == 0 || binding.size == 0) {
- return false;
+ return;
}
if (is_written && !runtime.PrefersSparseSources()) {
- return false;
+ return;
}
- const VirtualSegments* segments =
+ const VirtualSegments* found =
virtual_ranges.Query(*gpu_memory, binding.gpu_addr, binding.size);
- if (!segments || segments->size() < 2) {
- return false;
+ if (!found || found->size() < 2) {
+ return;
}
- const u64 key = (static_cast(gpu_memory->GetID()) << 48) ^ binding.gpu_addr;
+ const VirtualSegments segments = *found;
+ const u32 first = static_cast(pool.size());
const bool prefer_sparse = runtime.PrefersSparseSources();
- runtime.ResetMultiRange();
- for (const VirtualSegment& segment : *segments) {
+ for (const VirtualSegment& segment : segments) {
const BufferId buffer_id =
FindBuffer(segment.device_addr, segment.size, prefer_sparse);
if (!buffer_id) {
- return false;
+ pool.resize(first);
+ return;
}
- Buffer& buffer = slot_buffers[buffer_id];
- TouchBuffer(buffer, buffer_id);
+ pool.push_back(MultiRangeSegment{
+ .buffer_id = buffer_id,
+ .device_addr = segment.device_addr,
+ .size = segment.size,
+ });
+ }
+ binding.segment_first = first;
+ binding.segment_count = static_cast(segments.size());
+ }
+}
+
+template
+bool BufferCache::BindMultiRangeStorage(const Binding& binding, bool is_written,
+ std::span pool) {
+ if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}); }) {
+ if (binding.segment_count < 2) {
+ return false;
+ }
+ if (binding.segment_first + binding.segment_count > pool.size()) {
+ return false;
+ }
+ const u64 key = (static_cast(gpu_memory->GetID()) << 48) ^ binding.gpu_addr;
+ runtime.ResetMultiRange();
+ for (u32 index = 0; index < binding.segment_count; ++index) {
+ const MultiRangeSegment& segment = pool[binding.segment_first + index];
+ Buffer& buffer = slot_buffers[segment.buffer_id];
+ TouchBuffer(buffer, segment.buffer_id);
if (SynchronizeBuffer(buffer, segment.device_addr, segment.size)) {
runtime.InvalidateMultiRange(key);
}
const u32 offset = buffer.Offset(segment.device_addr);
buffer.MarkUsage(offset, segment.size);
if (is_written) {
- MarkWrittenBuffer(buffer_id, segment.device_addr, segment.size);
+ MarkWrittenBuffer(segment.buffer_id, segment.device_addr, segment.size);
}
runtime.PushMultiRangeSource(buffer, offset, segment.size);
}
@@ -1050,7 +1079,7 @@ void BufferCache::BindHostGraphicsStorageBuffers(size_t stage) {
ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) {
const Binding& binding = channel_state->storage_buffers[stage][index];
const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0;
- if (BindMultiRangeStorage(binding, is_written)) {
+ if (BindMultiRangeStorage(binding, is_written, graphics_segments)) {
return;
}
Buffer& buffer = slot_buffers[binding.buffer_id];
@@ -1190,7 +1219,7 @@ void BufferCache
::BindHostComputeStorageBuffers() {
const Binding& binding = channel_state->compute_storage_buffers[index];
const bool is_written =
((channel_state->written_compute_storage_buffers >> index) & 1) != 0;
- if (BindMultiRangeStorage(binding, is_written)) {
+ if (BindMultiRangeStorage(binding, is_written, compute_segments)) {
return;
}
Buffer& buffer = slot_buffers[binding.buffer_id];
@@ -1245,6 +1274,7 @@ void BufferCache
::BindHostComputeTextureBuffers() {
template
void BufferCache::DoUpdateGraphicsBuffers(bool is_indexed) {
+ graphics_segments.clear();
BufferOperations([&]() {
if (is_indexed) {
UpdateIndexBuffer();
@@ -1264,6 +1294,7 @@ void BufferCache
::DoUpdateGraphicsBuffers(bool is_indexed) {
template
void BufferCache::DoUpdateComputeBuffers() {
+ compute_segments.clear();
BufferOperations([&]() {
UpdateComputeUniformBuffers();
UpdateComputeStorageBuffers();
@@ -1406,6 +1437,8 @@ void BufferCache
::UpdateStorageBuffers(size_t stage) {
Binding& binding = channel_state->storage_buffers[stage][index];
const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size);
binding.buffer_id = buffer_id;
+ const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0;
+ ResolveMultiRangeStorage(binding, is_written, graphics_segments);
});
}
@@ -1469,6 +1502,9 @@ void BufferCache
::UpdateComputeStorageBuffers() {
// Resolve buffer
Binding& binding = channel_state->compute_storage_buffers[index];
binding.buffer_id = FindBuffer(binding.device_addr, binding.size);
+ const bool is_written =
+ ((channel_state->written_compute_storage_buffers >> index) & 1) != 0;
+ ResolveMultiRangeStorage(binding, is_written, compute_segments);
});
}
diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h
index c90ccb1cc7..18a8d19fe9 100644
--- a/src/video_core/buffer_cache/buffer_cache_base.h
+++ b/src/video_core/buffer_cache/buffer_cache_base.h
@@ -85,6 +85,14 @@ struct Binding {
u32 size{};
BufferId buffer_id;
GPUVAddr gpu_addr{};
+ u32 segment_first{};
+ u32 segment_count{};
+};
+
+struct MultiRangeSegment {
+ BufferId buffer_id;
+ DAddr device_addr{};
+ u32 size{};
};
struct TextureBufferBinding : Binding {
@@ -217,7 +225,11 @@ public:
void TickFrame();
- bool BindMultiRangeStorage(const Binding& binding, bool is_written);
+ bool BindMultiRangeStorage(const Binding& binding, bool is_written,
+ std::span pool);
+
+ void ResolveMultiRangeStorage(Binding& binding, bool is_written,
+ std::vector& pool);
void UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size);
@@ -523,6 +535,8 @@ private:
Common::LeastRecentlyUsedCache lru_cache;
u64 frame_tick = 0;
VirtualRangeCache virtual_ranges;
+ std::vector graphics_segments;
+ std::vector compute_segments;
u64 total_used_memory = 0;
u64 minimum_memory = 0;
u64 critical_memory = 0;
diff --git a/src/video_core/buffer_cache/virtual_range_cache.h b/src/video_core/buffer_cache/virtual_range_cache.h
index 7c279ed0da..3682acf5bc 100644
--- a/src/video_core/buffer_cache/virtual_range_cache.h
+++ b/src/video_core/buffer_cache/virtual_range_cache.h
@@ -43,20 +43,31 @@ public:
entry.gpu_addr = gpu_addr;
entry.size = size;
const auto ranges = memory.GetSubmappedRange(gpu_addr, size);
+ GPUVAddr expected = gpu_addr;
+ bool contiguous = true;
for (const auto& [range_addr, range_size] : ranges) {
- const std::optional device_addr = memory.GpuToCpuAddress(range_addr);
- if (!device_addr) {
+ if (range_addr != expected || range_size == 0) {
+ contiguous = false;
break;
}
- u32 segment_size = (std::numeric_limits::max)();
- if (range_size < static_cast(segment_size)) {
- segment_size = static_cast(range_size);
+ const std::optional device_addr = memory.GpuToCpuAddress(range_addr);
+ if (!device_addr || *device_addr == 0) {
+ contiguous = false;
+ break;
+ }
+ if (range_size > static_cast((std::numeric_limits::max)())) {
+ contiguous = false;
+ break;
}
entry.segments.push_back(VirtualSegment{
.gpu_addr = range_addr,
.device_addr = *device_addr,
- .size = segment_size,
+ .size = static_cast(range_size),
});
+ expected += range_size;
+ }
+ if (!contiguous || expected != gpu_addr + size) {
+ entry.segments.clear();
}
const auto result = entries.insert_or_assign(key, std::move(entry));
return &result.first->second.segments;
diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h
index 4e0b45cf0a..e9b162fb0f 100644
--- a/src/video_core/renderer_vulkan/vk_buffer_cache.h
+++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h
@@ -192,6 +192,7 @@ public:
.memory_offset = location.offset,
.offset = offset,
.size = size,
+ .memory_type = location.memory_type,
});
multi_range_total += size;
}
diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp
index 8b0158924d..79086c4244 100644
--- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp
+++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp
@@ -21,11 +21,13 @@ MultiRangeBufferCache::MultiRangeBufferCache(const Device& device_,
if (!device.IsSparseBindingSupported()) {
return;
}
- const VkDeviceSize queried = QueryBlockSize();
- if (queried == 0) {
+ u32 memory_type_bits = 0;
+ const VkDeviceSize queried = QueryBlockSize(memory_type_bits);
+ if (queried == 0 || memory_type_bits == 0) {
return;
}
block_size = queried;
+ sparse_memory_type_bits = memory_type_bits;
use_sparse = true;
}
@@ -44,7 +46,7 @@ MultiRangeBufferCache::~MultiRangeBufferCache() {
retired.clear();
}
-VkDeviceSize MultiRangeBufferCache::QueryBlockSize() const {
+VkDeviceSize MultiRangeBufferCache::QueryBlockSize(u32& memory_type_bits) const {
const VkDevice logical = *device.GetLogical();
const auto& dld = device.GetDispatchLoader();
const VkBufferCreateInfo probe_ci{
@@ -73,6 +75,7 @@ VkDeviceSize MultiRangeBufferCache::QueryBlockSize() const {
};
dld.vkGetBufferMemoryRequirements2(logical, &reqs_info, &reqs2);
dld.vkDestroyBuffer(logical, probe, nullptr);
+ memory_type_bits = reqs2.memoryRequirements.memoryTypeBits;
return reqs2.memoryRequirements.alignment;
}
@@ -98,6 +101,12 @@ bool MultiRangeBufferCache::CanBindSparse(std::span sour
if (source.memory == VK_NULL_HANDLE) {
return false;
}
+ if (source.memory_type >= 32) {
+ return false;
+ }
+ if (((sparse_memory_type_bits >> source.memory_type) & 1) == 0) {
+ return false;
+ }
const VkDeviceSize memory_offset = source.memory_offset + source.offset;
if ((memory_offset % block_size) != 0) {
return false;
diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h
index e809897bc1..6c9b738876 100644
--- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h
+++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h
@@ -22,6 +22,7 @@ struct MultiRangeSource {
VkDeviceSize memory_offset{};
VkDeviceSize offset{};
VkDeviceSize size{};
+ u32 memory_type{};
};
struct MultiRangeRef {
@@ -81,7 +82,7 @@ private:
[[nodiscard]] VkBuffer CreateSparse(std::span sources,
VkDeviceSize total);
- [[nodiscard]] VkDeviceSize QueryBlockSize() const;
+ [[nodiscard]] VkDeviceSize QueryBlockSize(u32& memory_type_bits) const;
void DestroySparse(VkBuffer handle);
@@ -92,6 +93,7 @@ private:
Scheduler& scheduler;
bool use_sparse{};
VkDeviceSize block_size{DEFAULT_BLOCK_SIZE};
+ u32 sparse_memory_type_bits{};
VkBufferUsageFlags sparse_usage{};
std::unordered_map entries;
std::vector retired;
diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp
index 4f070c6f88..ab8c8b238d 100644
--- a/src/video_core/vulkan_common/vulkan_wrapper.cpp
+++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp
@@ -549,6 +549,7 @@ MemoryLocation Buffer::Location() const noexcept {
return MemoryLocation{
.memory = info.deviceMemory,
.offset = info.offset,
+ .memory_type = info.memoryType,
};
}
diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h
index 3d7c3b9fe8..1535b91688 100644
--- a/src/video_core/vulkan_common/vulkan_wrapper.h
+++ b/src/video_core/vulkan_common/vulkan_wrapper.h
@@ -744,6 +744,7 @@ private:
struct MemoryLocation {
VkDeviceMemory memory{};
VkDeviceSize offset{};
+ u32 memory_type{};
};
class Buffer {