debuggerx/eden
1
0
Fork 0
mirror of https://git.eden-emu.dev/eden-emu/eden.git synced 2026-09-15 07:47:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Experiments on window mirroring + add vertex/index to the virtual page

Browse Source
This commit is contained in:
CamilleLaVey
2026-09-15 03:02:55 -04:00
parent 08ba001529
commit 36b1b23d40
18 changed files with 1296 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/video_core/buffer_cache/buffer_cache.h
+324 -45
View File
@@ -838,6 +838,9 @@ void BufferCache<P>::BindHostIndexBuffer() {
const u32 size = channel_state->index_buffer.size;
const auto& draw_state = maxwell3d->draw_manager.draw_state;
if (draw_state.inline_index_draw_indexes.empty()) {
if (BindVirtualIndexBuffer()) {
return;
}
SynchronizeBuffer(buffer, channel_state->index_buffer.device_addr, size);
} else {
if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) {
@@ -891,6 +894,7 @@ void BufferCache<P>::UpdateVertexBufferSlot(u32 index, const Binding& binding) {
enabled_vertex_buffers_mask |= (1u << index);
} else {
enabled_vertex_buffers_mask &= ~(1u << index);
virtual_vertex_buffers_mask &= ~(1u << index);
}
}
@@ -923,8 +927,16 @@ void BufferCache<P>::BindHostVertexBuffers() {
const Binding& binding = VertexBufferSlot(index);
Buffer& buffer = slot_buffers[binding.buffer_id];
TouchBuffer(buffer, binding.buffer_id);
if (StageVirtualVertexBuffer(index, binding, false)) {
continue;
}
bool needs_bind = flags[Dirty::VertexBuffer0 + index];
if (((virtual_vertex_buffers_mask >> index) & 1) != 0) {
virtual_vertex_buffers_mask &= ~(1u << index);
needs_bind = true;
}
SynchronizeBuffer(buffer, binding.device_addr, binding.size);
if (!flags[Dirty::VertexBuffer0 + index]) {
if (!needs_bind) {
flush_bindings();
continue;
}
@@ -945,6 +957,7 @@ void BufferCache<P>::BindHostVertexBuffers() {
last_index = index;
}
flush_bindings();
BindStagedVertexBuffers();
} else {
HostBindings<typename P::Buffer> host_bindings;
bool any_valid{false};
@@ -953,8 +966,17 @@ void BufferCache<P>::BindHostVertexBuffers() {
const Binding& binding = channel_state->vertex_buffers[index];
Buffer& buffer = slot_buffers[binding.buffer_id];
TouchBuffer(buffer, binding.buffer_id);
if (((enabled_vertex_buffers_mask >> index) & 1) != 0 &&
StageVirtualVertexBuffer(index, binding, true)) {
continue;
}
bool needs_bind = flags[Dirty::VertexBuffer0 + index];
if (((virtual_vertex_buffers_mask >> index) & 1) != 0) {
virtual_vertex_buffers_mask &= ~(1u << index);
needs_bind = true;
}
SynchronizeBuffer(buffer, binding.device_addr, binding.size);
if (!flags[Dirty::VertexBuffer0 + index]) {
if (!needs_bind) {
continue;
}
flags[Dirty::VertexBuffer0 + index] = false;
@@ -983,6 +1005,7 @@ void BufferCache<P>::BindHostVertexBuffers() {
}
runtime.BindVertexBuffers(host_bindings);
}
BindStagedVertexBuffers();
}
}
@@ -1038,9 +1061,12 @@ void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32
}();
const bool use_fast_buffer = needs_alignment_stream
|| (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size
&& !memory_tracker.IsRegionGpuModified(device_addr, size));
&& !memory_tracker.IsRegionGpuModified(device_addr, size)
&& !HasPendingUnifiedWrites(device_addr, size));
if (use_fast_buffer) {
WaitForUnifiedWrites(device_addr, size);
if (needs_alignment_stream) {
WaitForUnifiedWrites(device_addr, size);
}
if constexpr (IS_OPENGL) {
if (runtime.HasFastBufferSubData()) {
// Fast path for Nvidia
@@ -1136,6 +1162,34 @@ void BufferCache<P>::ResolveMultiRangeStorage(Binding& binding, bool is_written,
template <class P>
bool BufferCache<P>::BindMultiRangeStorage(const Binding& binding, bool is_written,
std::span<const MultiRangeSegment> pool) {
if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) {
const u64 key = GeometryKey(binding.gpu_addr, 0);
if (!PushMultiRangeSources(binding, is_written, pool, key)) {
return false;
}
return runtime.BindMultiRangeStorageBuffer(key, is_written);
} else {
return false;
}
}
template <class P>
u64 BufferCache<P>::GeometryKey(GPUVAddr gpu_addr, u64 salt) const {
return ((static_cast<u64>(gpu_memory->GetID()) << 48) ^ gpu_addr) ^ salt;
}
template <class P>
u32 BufferCache<P>::ClampToMappedRange(GPUVAddr gpu_addr, u32 size) const {
const size_t mapped = gpu_memory->MaxMappedRange(gpu_addr, size);
if (mapped == 0 || mapped >= size) {
return size;
}
return static_cast<u32>(mapped);
}
template <class P>
bool BufferCache<P>::PushMultiRangeSources(const Binding& binding, bool is_written,
std::span<const MultiRangeSegment> pool, u64 key) {
if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) {
if (binding.segment_count < 2) {
return false;
@@ -1143,23 +1197,188 @@ bool BufferCache<P>::BindMultiRangeStorage(const Binding& binding, bool is_writt
if (binding.segment_first + binding.segment_count > pool.size()) {
return false;
}
const u64 key = (static_cast<u64>(gpu_memory->GetID()) << 48) ^ binding.gpu_addr;
const std::span<const MultiRangeSegment> segments =
pool.subspan(binding.segment_first, binding.segment_count);
boost::container::small_vector<BufferId, 8> buffer_ids;
for (const MultiRangeSegment& segment : segments) {
BufferId buffer_id = segment.buffer_id;
if (!buffer_id) {
buffer_id = page_table[segment.device_addr >> CACHING_PAGEBITS];
}
if (!buffer_id ||
!slot_buffers[buffer_id].IsInBounds(segment.device_addr, segment.size)) {
return false;
}
buffer_ids.push_back(buffer_id);
}
runtime.ResetMultiRange();
for (u32 index = 0; index < binding.segment_count; ++index) {
const MultiRangeSegment& segment = pool[binding.segment_first + index];
Buffer& buffer = slot_buffers[segment.buffer_id];
TouchBuffer(buffer, segment.buffer_id);
if (SynchronizeBuffer(buffer, segment.device_addr, segment.size)) {
const MultiRangeSegment& segment = segments[index];
const BufferId buffer_id = buffer_ids[index];
Buffer& buffer = slot_buffers[buffer_id];
TouchBuffer(buffer, buffer_id);
if (!SynchronizeBuffer(buffer, segment.device_addr, segment.size)) {
runtime.InvalidateMultiRange(key);
}
const u32 offset = buffer.Offset(segment.device_addr);
buffer.MarkUsage(offset, segment.size);
if (is_written) {
MarkWrittenBuffer(segment.buffer_id, segment.device_addr, segment.size);
MarkWrittenBuffer(buffer_id, segment.device_addr, segment.size);
}
runtime.PushMultiRangeSource(buffer, offset, segment.size);
}
return runtime.BindMultiRangeStorageBuffer(key, is_written);
return true;
} else {
return false;
}
}
template <class P>
bool BufferCache<P>::TryResolveUnifiedSegments(
[[maybe_unused]] const Binding& binding,
[[maybe_unused]] std::span<const MultiRangeSegment> pool,
[[maybe_unused]] UnifiedExtents& extents) {
if constexpr (USE_UNIFIED_MEMORY) {
extents.clear();
const auto push = [&](DAddr device_addr, u64 size) {
const std::optional<u64> relative = TryResolveUnifiedRange(device_addr, size);
if (!relative) {
return false;
}
if (!extents.empty() && extents.back().relative + extents.back().size == *relative) {
extents.back().size += size;
return true;
}
extents.push_back(UnifiedExtent{.relative = *relative, .size = size});
return true;
};
if (binding.segment_count < 2) {
return push(binding.device_addr, binding.size);
}
if (binding.segment_first + binding.segment_count > pool.size()) {
return false;
}
for (const MultiRangeSegment& segment :
pool.subspan(binding.segment_first, binding.segment_count)) {
if (!push(segment.device_addr, segment.size)) {
return false;
}
}
return true;
} else {
return false;
}
}
template <class P>
void BufferCache<P>::ResolveGeometrySegments([[maybe_unused]] bool is_indexed) {
if constexpr (requires { runtime.BindStagedVertexBuffers(); }) {
const auto& draw_state = maxwell3d->draw_manager.draw_state;
if (is_indexed && draw_state.inline_index_draw_indexes.empty()) {
ResolveMultiRangeStorage(channel_state->index_buffer, false, graphics_segments);
}
u32 enabled_mask = enabled_vertex_buffers_mask;
while (enabled_mask != 0) {
const u32 index = std::countr_zero(enabled_mask);
enabled_mask &= enabled_mask - 1;
Binding& slot = VertexBufferSlot(index);
ResolveMultiRangeStorage(slot, false, graphics_segments);
Binding& channel_binding = channel_state->vertex_buffers[index];
channel_binding.segment_first = slot.segment_first;
channel_binding.segment_count = slot.segment_count;
}
}
}
template <class P>
bool BufferCache<P>::StageVirtualVertexBuffer([[maybe_unused]] u32 index,
[[maybe_unused]] const Binding& binding,
[[maybe_unused]] bool force) {
if constexpr (requires { runtime.BindStagedVertexBuffers(); }) {
auto& flags = maxwell3d->dirty.flags;
const bool rebind = force || flags[Dirty::VertexBuffer0 + index] ||
((virtual_vertex_buffers_mask >> index) & 1) == 0;
const u32 stride = maxwell3d->regs.vertex_streams[index].stride;
const u64 salt = VERTEX_GEOMETRY_SALT + (u64{index} << 40);
bool staged = false;
if constexpr (USE_UNIFIED_MEMORY) {
UnifiedExtents extents;
if (TryResolveUnifiedSegments(binding, graphics_segments, extents)) {
const u64 view_key = GeometryKey(binding.gpu_addr, salt + VIEW_GEOMETRY_SALT);
staged = runtime.StageUnifiedVertexBuffer(index, view_key, extents, binding.size,
stride, rebind);
}
}
if (!staged) {
const u64 key = GeometryKey(binding.gpu_addr, salt);
if (PushMultiRangeSources(binding, false, graphics_segments, key)) {
staged = runtime.StageMultiRangeVertexBuffer(index, key, binding.size, stride,
rebind);
}
}
if (!staged) {
return false;
}
flags[Dirty::VertexBuffer0 + index] = false;
virtual_vertex_buffers_mask |= 1u << index;
return true;
} else {
return false;
}
}
template <class P>
bool BufferCache<P>::BindVirtualIndexBuffer() {
if constexpr (!HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT &&
requires { runtime.BindStagedVertexBuffers(); }) {
const Binding& binding = channel_state->index_buffer;
const auto& draw_state = maxwell3d->draw_manager.draw_state;
const auto& index_ref = draw_state.index_buffer;
if constexpr (USE_UNIFIED_MEMORY) {
UnifiedExtents extents;
const u64 view_key =
GeometryKey(binding.gpu_addr, INDEX_GEOMETRY_SALT + VIEW_GEOMETRY_SALT);
if (TryResolveUnifiedSegments(binding, graphics_segments, extents) &&
runtime.BindUnifiedIndexBuffer(draw_state.topology, index_ref.format,
index_ref.first, index_ref.count, view_key,
extents, binding.size)) {
return true;
}
}
const u64 key = GeometryKey(binding.gpu_addr, INDEX_GEOMETRY_SALT);
if (!PushMultiRangeSources(binding, false, graphics_segments, key)) {
return false;
}
return runtime.BindMultiRangeIndexBuffer(draw_state.topology, index_ref.format,
index_ref.first, index_ref.count, key,
binding.size);
} else {
return false;
}
}
template <class P>
void BufferCache<P>::BindStagedVertexBuffers() {
if constexpr (requires { runtime.BindStagedVertexBuffers(); }) {
runtime.BindStagedVertexBuffers();
}
}
template <class P>
bool BufferCache<P>::HasPendingUnifiedWrites([[maybe_unused]] DAddr device_addr,
[[maybe_unused]] u64 size) {
if constexpr (USE_UNIFIED_MEMORY) {
if (unified_written_ranges.Empty()) {
return false;
}
if (runtime.KnownGpuTick() >= unified_write_tick) {
unified_written_ranges.Clear();
return false;
}
bool overlaps = false;
unified_written_ranges.ForEachInRange(device_addr, size,
[&overlaps](DAddr, DAddr) { overlaps = true; });
return overlaps;
} else {
return false;
}
@@ -1169,13 +1388,7 @@ template <class P>
void BufferCache<P>::WaitForUnifiedWrites([[maybe_unused]] DAddr device_addr,
[[maybe_unused]] u64 size) {
if constexpr (USE_UNIFIED_MEMORY) {
if (unified_written_ranges.Empty()) {
return;
}
bool overlaps = false;
unified_written_ranges.ForEachInRange(device_addr, size,
[&overlaps](DAddr, DAddr) { overlaps = true; });
if (!overlaps) {
if (!HasPendingUnifiedWrites(device_addr, size)) {
return;
}
runtime.Wait(unified_write_tick);
@@ -1183,12 +1396,63 @@ void BufferCache<P>::WaitForUnifiedWrites([[maybe_unused]] DAddr device_addr,
}
}
template <class P>
template <typename Func>
bool BufferCache<P>::CopyUnifiedWrites([[maybe_unused]] Buffer& buffer,
[[maybe_unused]] DAddr device_addr,
[[maybe_unused]] u64 size,
[[maybe_unused]] Func&& add_upload) {
if constexpr (USE_UNIFIED_MEMORY) {
if (!HasPendingUnifiedWrites(device_addr, size)) {
return false;
}
boost::container::small_vector<std::pair<DAddr, DAddr>, 4> overlaps;
unified_written_ranges.ForEachInRange(device_addr, size,
[&overlaps](DAddr start, DAddr end) {
overlaps.emplace_back(start, end);
});
const DAddr buffer_start = buffer.CpuAddr();
boost::container::small_vector<u64, 4> window_ids;
UnifiedWindowGroups groups;
for (const auto& [start, end] : overlaps) {
if (!ResolveUnifiedWindows(start, start - buffer_start, end - start, window_ids,
groups)) {
runtime.Wait(unified_write_tick);
unified_written_ranges.Clear();
return false;
}
}
for (size_t i = 0; i < window_ids.size(); ++i) {
const std::span<const BufferCopy> group_span(groups[i].data(), groups[i].size());
runtime.CopyFromUnifiedMemory(window_ids[i], buffer, group_span);
}
DAddr cursor = device_addr;
for (const auto& [start, end] : overlaps) {
buffer.MarkUsage(start - buffer_start, end - start);
if (start > cursor) {
add_upload(cursor, start - cursor);
}
cursor = end;
}
if (cursor < device_addr + size) {
add_upload(cursor, device_addr + size - cursor);
}
return true;
} else {
return false;
}
}
template <class P>
bool BufferCache<P>::BindUnifiedStorage([[maybe_unused]] const Binding& binding,
[[maybe_unused]] bool is_written) {
if constexpr (USE_UNIFIED_MEMORY) {
const auto window = TryResolveUnifiedRange(binding.device_addr, binding.size);
if (!window || !runtime.IsUnifiedStorageRange(binding.size, window->offset)) {
const auto relative = TryResolveUnifiedRange(binding.device_addr, binding.size);
if (!relative) {
return false;
}
const auto range = runtime.ResolveUnifiedStorage(*relative, binding.size);
if (!range) {
return false;
}
if (is_written) {
@@ -1197,9 +1461,8 @@ bool BufferCache<P>::BindUnifiedStorage([[maybe_unused]] const Binding& binding,
unified_write_tick = runtime.CurrentTick();
uncommitted_unified_writes = true;
}
runtime.BindStorageBuffer(runtime.UnifiedWindowBuffer(window->window),
runtime.UnifiedWindowAddress(window->window),
static_cast<u32>(window->offset), binding.size, is_written);
runtime.BindStorageBuffer(range->buffer, range->address, range->offset,
binding.size, is_written);
return true;
} else {
return false;
@@ -1421,6 +1684,7 @@ void BufferCache<P>::DoUpdateGraphicsBuffers(bool is_indexed) {
UpdateIndexBuffer();
}
UpdateVertexBuffers();
ResolveGeometrySegments(is_indexed);
UpdateTransformFeedbackBuffers();
for (size_t stage = 0; stage < NUM_STAGES; ++stage) {
UpdateUniformBuffers(stage);
@@ -1477,13 +1741,17 @@ void BufferCache<P>::UpdateIndexBuffer() {
const std::optional<DAddr> device_addr = gpu_memory->GpuToCpuAddress(gpu_addr_begin);
const u32 address_size = static_cast<u32>(gpu_addr_end - gpu_addr_begin);
const u32 draw_size = (index_buffer_ref.count + index_buffer_ref.first) * u32(index_buffer_ref.FormatSizeInBytes());
const u32 size = (std::min)(address_size, draw_size);
u32 size = (std::min)(address_size, draw_size);
if (size == 0 || !device_addr) {
channel_state->index_buffer = NULL_BINDING;
return;
}
if constexpr (requires { runtime.BindStagedVertexBuffers(); }) {
size = ClampToMappedRange(gpu_addr_begin, size);
}
channel_state->index_buffer = Binding{
.device_addr = *device_addr,
.gpu_addr = gpu_addr_begin,
.size = size,
.buffer_id = FindBuffer(*device_addr, size, false),
};
@@ -1521,10 +1789,13 @@ void BufferCache<P>::UpdateVertexBuffer(u32 index) {
}
if (!gpu_memory->IsWithinGPUAddressRange(gpu_addr_end) || size >= 64_MiB) {
size = static_cast<u32>(gpu_memory->MaxContinuousRange(gpu_addr_begin, size));
} else if constexpr (requires { runtime.BindStagedVertexBuffers(); }) {
size = ClampToMappedRange(gpu_addr_begin, size);
}
const BufferId buffer_id = FindBuffer(*device_addr, size, false);
const Binding binding{
.device_addr = *device_addr,
.gpu_addr = gpu_addr_begin,
.size = size,
.buffer_id = buffer_id,
};
@@ -1881,19 +2152,30 @@ bool BufferCache<P>::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 si
u64 total_size_bytes = 0;
u64 largest_copy = 0;
const DAddr buffer_start = buffer.cpu_addr_cached;
memory_tracker.ForEachUploadRange(device_addr, size, [&](u64 device_addr_out, u64 range_size) {
const auto add_upload = [&](u64 upload_addr, u64 upload_size) {
upload_copies.push_back(BufferCopy{
.src_offset = total_size_bytes,
.dst_offset = device_addr_out - buffer_start,
.size = range_size,
.dst_offset = upload_addr - buffer_start,
.size = upload_size,
});
total_size_bytes += range_size;
largest_copy = (std::max)(largest_copy, range_size);
total_size_bytes += upload_size;
largest_copy = (std::max)(largest_copy, upload_size);
};
bool copied_from_windows = false;
memory_tracker.ForEachUploadRange(device_addr, size, [&](u64 device_addr_out, u64 range_size) {
if (CopyUnifiedWrites(buffer, device_addr_out, range_size, add_upload)) {
copied_from_windows = true;
return;
}
add_upload(device_addr_out, range_size);
});
if (total_size_bytes == 0) {
if (copied_from_windows) {
any_buffer_uploaded = true;
return false;
}
return true;
}
WaitForUnifiedWrites(device_addr, size);
const std::span<BufferCopy> copies_span(upload_copies.data(), upload_copies.size());
UploadMemory(buffer, total_size_bytes, largest_copy, copies_span);
any_buffer_uploaded = true;
@@ -2004,17 +2286,12 @@ bool BufferCache<P>::ResolveUnifiedWindows(
}
template <class P>
std::optional<typename BufferCache<P>::UnifiedWindowRange>
BufferCache<P>::TryResolveUnifiedRange([[maybe_unused]] DAddr device_addr,
[[maybe_unused]] u64 size) {
std::optional<u64> BufferCache<P>::TryResolveUnifiedRange([[maybe_unused]] DAddr device_addr,
[[maybe_unused]] u64 size) {
if constexpr (USE_UNIFIED_MEMORY) {
if (size == 0 || !runtime.IsUnifiedMemoryBindable()) {
return std::nullopt;
}
const u64 window_size = runtime.UnifiedMemoryWindowSize();
if (window_size == 0) {
return std::nullopt;
}
const u8* const first = device_memory.GetSpan(device_addr, size);
if (first == nullptr) {
return std::nullopt;
@@ -2029,18 +2306,11 @@ BufferCache<P>::TryResolveUnifiedRange([[maybe_unused]] DAddr device_addr,
if (relative >= unified_size || unified_size - relative < size) {
return std::nullopt;
}
const u64 local_offset = relative % window_size;
if (window_size - local_offset < size) {
return std::nullopt;
}
if (memory_tracker.IsRegionGpuModified(device_addr, size) ||
IsRegionGpuModified(device_addr, size)) {
return std::nullopt;
}
return UnifiedWindowRange{
.window = static_cast<size_t>(relative / window_size),
.offset = local_offset,
};
return relative;
} else {
return std::nullopt;
}
@@ -2245,6 +2515,15 @@ void BufferCache<P>::DeleteBuffer(BufferId buffer_id, bool do_not_mark) {
replace(channel_state->transform_feedback_buffers);
replace(channel_state->compute_uniform_buffers);
replace(channel_state->compute_storage_buffers);
const auto drop_segments = [buffer_id](std::vector<MultiRangeSegment>& pool) {
for (MultiRangeSegment& segment : pool) {
if (segment.buffer_id == buffer_id) {
segment.buffer_id = BufferId{};
}
}
};
drop_segments(graphics_segments);
drop_segments(compute_segments);
// Mark the whole buffer as CPU written to stop tracking CPU writes
if (!do_not_mark) {
src/video_core/buffer_cache/buffer_cache_base.h
+34 -5
View File
@@ -96,6 +96,11 @@ struct MultiRangeSegment {
u32 size{};
};
struct UnifiedExtent {
u64 relative{};
u64 size{};
};
struct TextureBufferBinding : Binding {
PixelFormat format;
};
@@ -192,6 +197,9 @@ class BufferCache : public VideoCommon::ChannelSetupCaches<BufferCacheChannelInf
static constexpr bool SEPARATE_IMAGE_BUFFERS_BINDINGS = P::SEPARATE_IMAGE_BUFFER_BINDINGS;
static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = P::USE_MEMORY_MAPS_FOR_UPLOADS;
static constexpr bool USE_UNIFIED_MEMORY = P::USE_UNIFIED_MEMORY;
static constexpr u64 VERTEX_GEOMETRY_SALT = u64{1} << 47;
static constexpr u64 INDEX_GEOMETRY_SALT = u64{1} << 46;
static constexpr u64 VIEW_GEOMETRY_SALT = u64{1} << 45;
#ifdef YUZU_LEGACY
static constexpr s64 TARGET_THRESHOLD = 3_GiB;
@@ -232,6 +240,9 @@ public:
bool BindUnifiedStorage(const Binding& binding, bool is_written);
bool PushMultiRangeSources(const Binding& binding, bool is_written,
std::span<const MultiRangeSegment> pool, u64 key);
void ResolveMultiRangeStorage(Binding& binding, bool is_written,
std::vector<MultiRangeSegment>& pool);
@@ -468,15 +479,32 @@ private:
bool TryUnifiedDownloadMemory(Buffer& buffer, std::span<BufferCopy> copies);
struct UnifiedWindowRange {
size_t window;
u64 offset;
};
std::optional<u64> TryResolveUnifiedRange(DAddr device_addr, u64 size);
std::optional<UnifiedWindowRange> TryResolveUnifiedRange(DAddr device_addr, u64 size);
using UnifiedExtents = boost::container::small_vector<UnifiedExtent, 8>;
bool TryResolveUnifiedSegments(const Binding& binding, std::span<const MultiRangeSegment> pool,
UnifiedExtents& extents);
void ResolveGeometrySegments(bool is_indexed);
bool StageVirtualVertexBuffer(u32 index, const Binding& binding, bool force);
bool BindVirtualIndexBuffer();
void BindStagedVertexBuffers();
[[nodiscard]] u64 GeometryKey(GPUVAddr gpu_addr, u64 salt) const;
[[nodiscard]] u32 ClampToMappedRange(GPUVAddr gpu_addr, u32 size) const;
bool HasPendingUnifiedWrites(DAddr device_addr, u64 size);
void WaitForUnifiedWrites(DAddr device_addr, u64 size);
template <typename Func>
bool CopyUnifiedWrites(Buffer& buffer, DAddr device_addr, u64 size, Func&& add_upload);
using UnifiedWindowGroups =
boost::container::small_vector<boost::container::small_vector<BufferCopy, 16>, 4>;
@@ -528,6 +556,7 @@ private:
u32 last_index_count = 0;
u32 enabled_vertex_buffers_mask = 0;
u32 virtual_vertex_buffers_mask = 0;
u64 vertex_buffers_serial = 0;
std::array<Binding, 32> v_buffer{};
src/video_core/memory_manager.cpp
+23
View File
@@ -529,6 +529,29 @@ size_t MemoryManager::MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const {
return range_so_far;
}
size_t MemoryManager::MaxMappedRange(GPUVAddr gpu_addr, size_t size) const {
size_t range_so_far = 0;
bool stopped{false};
auto stop = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset,
[[maybe_unused]] std::size_t copy_amount) {
stopped = true;
return true;
};
auto accumulate = [&]([[maybe_unused]] std::size_t page_index,
[[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
range_so_far += copy_amount;
return false;
};
auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation(base, copy_amount, false, accumulate, stop, stop);
return stopped;
};
MemoryOperation(gpu_addr, size, true, accumulate, stop, check_short_pages);
return range_so_far;
}
size_t MemoryManager::GetMemoryLayoutSize(GPUVAddr gpu_addr, size_t max_size) const {
std::unique_lock<std::mutex> lock(guard);
return kind_map.GetContinuousSizeFrom(gpu_addr);
src/video_core/memory_manager.h
+2
View File
@@ -141,6 +141,8 @@ public:
size_t MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const;
size_t MaxMappedRange(GPUVAddr gpu_addr, size_t size) const;
bool IsWithinGPUAddressRange(GPUVAddr gpu_addr) const {
return gpu_addr < address_space_size;
}
src/video_core/renderer_vulkan/vk_buffer_cache.cpp
+289 -5
View File
@@ -12,6 +12,7 @@
#include <utility>
#include <vector>
#include "common/alignment.h"
#include "video_core/buffer_cache/buffer_cache_base.h"
#include "video_core/renderer_vulkan/vk_buffer_cache.h"
@@ -435,6 +436,11 @@ void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size,
size_t hardware_buffer_base) {
unified_memory = memory_allocator.CreateHostMemoryImport(
base, size, hardware_buffers, hardware_buffer_window, hardware_buffer_base);
if (unified_memory) {
unified_memory->CreateMirror(scheduler.submit_mutex,
device.GetSparseAddressSpaceSize() / 2);
multi_range_buffers.ReserveSparseAddressSpace(unified_memory->GetMirrorSize());
}
}
void BufferCacheRuntime::CopyToUnifiedMemory(
@@ -452,6 +458,74 @@ void BufferCacheRuntime::CopyToUnifiedMemory(
std::ranges::transform(copies, pending.copies.begin(), MakeBufferCopy);
}
void BufferCacheRuntime::CopyFromUnifiedMemory(
size_t window_index, VkBuffer dst_buffer, std::span<const VideoCommon::BufferCopy> copies) {
if (!unified_memory || dst_buffer == VK_NULL_HANDLE || copies.empty() ||
window_index >= unified_memory->GetWindowCount()) {
return;
}
const VkBuffer window_buffer = unified_memory->GetWindowBuffer(window_index);
if (window_buffer == VK_NULL_HANDLE) {
return;
}
const bool foreign = unified_memory->NeedsForeignOwnershipTransfer();
const u32 queue_family = device.GetGraphicsFamily();
boost::container::small_vector<VkBufferCopy, 8> vk_copies;
WindowRanges ranges;
for (const VideoCommon::BufferCopy& copy : copies) {
vk_copies.push_back(VkBufferCopy{
.srcOffset = static_cast<VkDeviceSize>(copy.dst_offset),
.dstOffset = static_cast<VkDeviceSize>(copy.src_offset),
.size = static_cast<VkDeviceSize>(copy.size),
});
ranges.emplace_back(copy.dst_offset, copy.dst_offset + copy.size);
}
CoalesceWindowRanges(ranges);
boost::container::small_vector<VkBufferMemoryBarrier, MAX_WINDOW_BARRIER_RANGES> acquire;
boost::container::small_vector<VkBufferMemoryBarrier, MAX_WINDOW_BARRIER_RANGES> release;
if (foreign) {
for (const WindowRange& range : ranges) {
acquire.push_back(VkBufferMemoryBarrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = 0,
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT,
.dstQueueFamilyIndex = queue_family,
.buffer = window_buffer,
.offset = range.first,
.size = range.second - range.first,
});
release.push_back(VkBufferMemoryBarrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
.dstAccessMask = 0,
.srcQueueFamilyIndex = queue_family,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT,
.buffer = window_buffer,
.offset = range.first,
.size = range.second - range.first,
});
}
}
scheduler.RequestOutsideRenderPassOperationContext();
scheduler.Record([window_buffer, dst_buffer, vk_copies, acquire = std::move(acquire),
release = std::move(release)](vk::CommandBuffer cmdbuf) {
if (!acquire.empty()) {
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {},
VideoCommon::FixSmallVectorADL(acquire), {});
}
cmdbuf.CopyBuffer(window_buffer, dst_buffer, VideoCommon::FixSmallVectorADL(vk_copies));
if (!release.empty()) {
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, {},
VideoCommon::FixSmallVectorADL(release), {});
}
});
}
void BufferCacheRuntime::FlushUnifiedMemoryCopies() {
if (pending_unified_copies.empty()) {
return;
@@ -733,17 +807,17 @@ void BufferCacheRuntime::ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t si
});
}
bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key, bool is_written) {
MultiRangeRef BufferCacheRuntime::AcquireMultiRange(u64 key, bool require_sparse) {
if (multi_range_sources.empty() || multi_range_total == 0) {
return false;
return MultiRangeRef{};
}
const MultiRangeRef ref = multi_range_buffers.Get(device, scheduler, memory_allocator, key,
multi_range_sources, multi_range_total);
if (ref.handle == VK_NULL_HANDLE) {
return false;
return MultiRangeRef{};
}
if (is_written && !ref.sparse) {
return false;
if (require_sparse && !ref.sparse) {
return MultiRangeRef{};
}
if (ref.needs_gather) {
PreCopyBarrier();
@@ -760,10 +834,220 @@ bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key, bool is_written) {
PostCopyBarrier();
multi_range_buffers.MarkGathered(key);
}
return ref;
}
bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key, bool is_written) {
const MultiRangeRef ref = AcquireMultiRange(key, is_written);
if (ref.handle == VK_NULL_HANDLE) {
return false;
}
guest_descriptor_queue.AddBuffer(ref.handle, ref.address, 0, ref.size);
return true;
}
std::optional<HostMemoryImport::Range> BufferCacheRuntime::ResolveUnifiedExtents(
u64 key, std::span<const VideoCommon::UnifiedExtent> extents, u32 size) {
if (extents.size() == 1) {
return unified_memory->ResolveRange(extents.front().relative, size);
}
return AcquireUnifiedView(key, extents);
}
std::optional<HostMemoryImport::Range> BufferCacheRuntime::AcquireUnifiedView(
u64 key, std::span<const VideoCommon::UnifiedExtent> extents) {
const VkBufferUsageFlags usage = unified_memory->GetViewUsage();
if (usage == 0 || !multi_range_buffers.use_sparse || extents.size() < 2) {
return std::nullopt;
}
const VkDeviceSize block = multi_range_buffers.block_size;
boost::container::small_vector<MultiRangeSource, 16> sources;
VkDeviceSize total = 0;
VkDeviceSize padding = 0;
for (size_t index = 0; index < extents.size(); ++index) {
VkDeviceSize begin = extents[index].relative;
VkDeviceSize end = begin + extents[index].size;
if (index == 0) {
padding = begin % block;
begin -= padding;
} else if ((begin % block) != 0) {
return std::nullopt;
}
if (index + 1 == extents.size()) {
end = Common::AlignUp(end, block);
} else if ((end % block) != 0) {
return std::nullopt;
}
while (begin < end) {
const auto memory = unified_memory->ResolveViewMemory(begin);
if (!memory || (memory->offset % block) != 0) {
return std::nullopt;
}
const VkDeviceSize length = (std::min)(end - begin, memory->available);
if ((length % block) != 0) {
return std::nullopt;
}
sources.push_back(MultiRangeSource{
.handle = memory->buffer,
.memory = memory->memory,
.memory_offset = 0,
.offset = memory->offset,
.size = length,
.write_tick = 0,
.memory_type = memory->memory_type,
});
total += length;
begin += length;
}
}
const MultiRangeRef ref = multi_range_buffers.GetView(
device, scheduler, key, sources, total, usage,
VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID);
if (ref.handle == VK_NULL_HANDLE) {
return std::nullopt;
}
return HostMemoryImport::Range{
.buffer = ref.handle,
.address = ref.address,
.offset = padding,
};
}
bool BufferCacheRuntime::StageUnifiedVertexBuffer(
u32 index, u64 key, std::span<const VideoCommon::UnifiedExtent> extents, u32 size, u32 stride,
bool force) {
if (!unified_memory ||
(unified_memory->GetUsage() & VK_BUFFER_USAGE_VERTEX_BUFFER_BIT) == 0) {
return false;
}
const auto range = ResolveUnifiedExtents(key, extents, size);
if (!range) {
return false;
}
StageVertexBuffer(
StagedVertexBuffer{
.index = index,
.buffer = range->buffer,
.offset = range->offset,
.size = size,
.stride = stride,
},
force);
return true;
}
bool BufferCacheRuntime::StageMultiRangeVertexBuffer(u32 index, u64 key, u32 size, u32 stride,
bool force) {
const MultiRangeRef ref = AcquireMultiRange(key, false);
if (ref.handle == VK_NULL_HANDLE) {
return false;
}
StageVertexBuffer(
StagedVertexBuffer{
.index = index,
.buffer = ref.handle,
.offset = 0,
.size = size,
.stride = stride,
},
force);
return true;
}
void BufferCacheRuntime::StageVertexBuffer(const StagedVertexBuffer& target, bool force) {
StagedVertexBuffer& bound = bound_vertex_buffers[target.index];
if (!force && bound == target) {
return;
}
bound = target;
if (target.index < device.GetMaxVertexInputBindings()) {
staged_vertex_buffers.push_back(target);
}
}
void BufferCacheRuntime::BindStagedVertexBuffers() {
if (staged_vertex_buffers.empty()) {
return;
}
scheduler.Record([staged = staged_vertex_buffers,
extended = device.IsExtExtendedDynamicStateSupported()](
vk::CommandBuffer cmdbuf) {
std::array<VkBuffer, VideoCommon::NUM_VERTEX_BUFFERS> buffers{};
std::array<VkDeviceSize, VideoCommon::NUM_VERTEX_BUFFERS> offsets{};
std::array<VkDeviceSize, VideoCommon::NUM_VERTEX_BUFFERS> sizes{};
std::array<VkDeviceSize, VideoCommon::NUM_VERTEX_BUFFERS> strides{};
size_t begin = 0;
while (begin < staged.size()) {
const u32 first = staged[begin].index;
u32 count = 0;
while (begin + count < staged.size() && staged[begin + count].index == first + count) {
const StagedVertexBuffer& entry = staged[begin + count];
buffers[count] = entry.buffer;
offsets[count] = entry.offset;
sizes[count] = entry.size;
strides[count] = entry.stride;
++count;
}
if (extended) {
cmdbuf.BindVertexBuffers2EXT(first, count, buffers.data(), offsets.data(),
sizes.data(), strides.data());
} else {
cmdbuf.BindVertexBuffers(first, count, buffers.data(), offsets.data());
}
begin += count;
}
});
staged_vertex_buffers.clear();
}
bool BufferCacheRuntime::IsIndexRangeUsable(PrimitiveTopology topology, IndexFormat index_format,
VkDeviceSize offset, u32 size) const {
const VkIndexType vk_index_type = MaxwellToVK::IndexFormat(index_format);
const bool is_quad =
topology == PrimitiveTopology::Quads || topology == PrimitiveTopology::QuadStrip;
const bool is_emulated_uint8 =
vk_index_type == VK_INDEX_TYPE_UINT8_EXT && !device.IsExtIndexTypeUint8Supported();
if (is_quad || is_emulated_uint8) {
return size <= device.GetMaxStorageBufferRange() &&
(offset % device.GetStorageBufferAlignment()) == 0;
}
return (offset % BytesPerIndex(vk_index_type)) == 0;
}
bool BufferCacheRuntime::BindUnifiedIndexBuffer(PrimitiveTopology topology,
IndexFormat index_format, u32 base_vertex,
u32 num_indices, u64 key,
std::span<const VideoCommon::UnifiedExtent> extents,
u32 size) {
constexpr VkBufferUsageFlags GeometryUsage =
VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
if (!unified_memory || (unified_memory->GetUsage() & GeometryUsage) != GeometryUsage) {
return false;
}
const auto range = ResolveUnifiedExtents(key, extents, size);
if (!range || range->offset > (std::numeric_limits<u32>::max)() - size ||
!IsIndexRangeUsable(topology, index_format, range->offset, size)) {
return false;
}
BindIndexBuffer(topology, index_format, base_vertex, num_indices, range->buffer,
static_cast<u32>(range->offset), size);
return true;
}
bool BufferCacheRuntime::BindMultiRangeIndexBuffer(PrimitiveTopology topology,
IndexFormat index_format, u32 base_vertex,
u32 num_indices, u64 key, u32 size) {
if (!IsIndexRangeUsable(topology, index_format, 0, size)) {
return false;
}
const MultiRangeRef ref = AcquireMultiRange(key, false);
if (ref.handle == VK_NULL_HANDLE) {
return false;
}
BindIndexBuffer(topology, index_format, base_vertex, num_indices, ref.handle, 0, size);
return true;
}
void BufferCacheRuntime::BindIndexBuffer(PrimitiveTopology topology, IndexFormat index_format,
u32 base_vertex, u32 num_indices, VkBuffer buffer,
u32 offset, [[maybe_unused]] u32 size) {
src/video_core/renderer_vulkan/vk_buffer_cache.h
+55 -12
View File
@@ -135,6 +135,9 @@ public:
void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer,
std::span<const VideoCommon::BufferCopy> copies);
void CopyFromUnifiedMemory(size_t window_index, VkBuffer dst_buffer,
std::span<const VideoCommon::BufferCopy> copies);
void FlushUnifiedMemoryCopies();
void UnifiedMemoryHostBarrier();
@@ -229,6 +232,21 @@ public:
bool BindMultiRangeStorageBuffer(u64 key, bool is_written);
bool StageUnifiedVertexBuffer(u32 index, u64 key,
std::span<const VideoCommon::UnifiedExtent> extents, u32 size,
u32 stride, bool force);
bool StageMultiRangeVertexBuffer(u32 index, u64 key, u32 size, u32 stride, bool force);
void BindStagedVertexBuffers();
bool BindUnifiedIndexBuffer(PrimitiveTopology topology, IndexFormat index_format,
u32 base_vertex, u32 num_indices, u64 key,
std::span<const VideoCommon::UnifiedExtent> extents, u32 size);
bool BindMultiRangeIndexBuffer(PrimitiveTopology topology, IndexFormat index_format,
u32 base_vertex, u32 num_indices, u64 key, u32 size);
void InvalidateMultiRange(u64 key) {
multi_range_buffers.Invalidate(key);
}
@@ -246,7 +264,7 @@ public:
BindBuffer(buffer, offset, size);
}
void BindStorageBuffer(VkBuffer buffer, VkDeviceAddress address, u32 offset, u32 size,
void BindStorageBuffer(VkBuffer buffer, VkDeviceAddress address, VkDeviceSize offset, u32 size,
[[maybe_unused]] bool is_written) {
guest_descriptor_queue.AddBuffer(buffer, address, offset, size);
}
@@ -256,17 +274,16 @@ public:
unified_memory->IsBindable();
}
[[nodiscard]] VkBuffer UnifiedWindowBuffer(size_t index) const noexcept {
return unified_memory->GetWindowBuffer(index);
}
[[nodiscard]] VkDeviceAddress UnifiedWindowAddress(size_t index) const noexcept {
return unified_memory->GetWindowAddress(index);
}
[[nodiscard]] bool IsUnifiedStorageRange(u32 size, u64 offset) const {
return size <= device.GetMaxStorageBufferRange() &&
(offset % device.GetStorageBufferAlignment()) == 0;
[[nodiscard]] std::optional<HostMemoryImport::Range> ResolveUnifiedStorage(u64 relative,
u32 size) const {
if (size > device.GetMaxStorageBufferRange()) {
return std::nullopt;
}
const auto range = unified_memory->ResolveRange(relative, size);
if (!range || (range->offset % device.GetStorageBufferAlignment()) != 0) {
return std::nullopt;
}
return range;
}
void BindTextureBuffer(Buffer& buffer, u32 offset, u32 size,
@@ -291,6 +308,29 @@ private:
boost::container::small_vector<VkBufferCopy, 8> copies;
};
struct StagedVertexBuffer {
u32 index;
VkBuffer buffer;
VkDeviceSize offset;
u32 size;
u32 stride;
bool operator==(const StagedVertexBuffer&) const = default;
};
[[nodiscard]] MultiRangeRef AcquireMultiRange(u64 key, bool require_sparse);
[[nodiscard]] std::optional<HostMemoryImport::Range> ResolveUnifiedExtents(
u64 key, std::span<const VideoCommon::UnifiedExtent> extents, u32 size);
[[nodiscard]] std::optional<HostMemoryImport::Range> AcquireUnifiedView(
u64 key, std::span<const VideoCommon::UnifiedExtent> extents);
void StageVertexBuffer(const StagedVertexBuffer& target, bool force);
[[nodiscard]] bool IsIndexRangeUsable(PrimitiveTopology topology, IndexFormat index_format,
VkDeviceSize offset, u32 size) const;
void BindBuffer(const Buffer& buffer, u32 offset, u32 size) {
const VkBuffer handle = buffer.Handle();
if (handle == VK_NULL_HANDLE) {
@@ -324,6 +364,9 @@ private:
MultiRangeBufferCache multi_range_buffers;
boost::container::small_vector<MultiRangeSource, 16> multi_range_sources;
VkDeviceSize multi_range_total{};
boost::container::static_vector<StagedVertexBuffer, VideoCommon::NUM_VERTEX_BUFFERS>
staged_vertex_buffers;
std::array<StagedVertexBuffer, VideoCommon::NUM_VERTEX_BUFFERS> bound_vertex_buffers{};
bool limit_dynamic_storage_buffers = false;
u32 max_dynamic_storage_buffers = (std::numeric_limits<u32>::max)();
src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp
+142 -8
View File
@@ -13,7 +13,8 @@ namespace Vulkan {
MultiRangeBufferCache::MultiRangeBufferCache(const Device& device) {
sparse_usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT |
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT |
VK_BUFFER_USAGE_INDEX_BUFFER_BIT;
if (device.IsBufferDeviceAddressSupported()) {
sparse_usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
}
@@ -27,9 +28,14 @@ MultiRangeBufferCache::MultiRangeBufferCache(const Device& device) {
}
block_size = queried;
sparse_memory_type_bits = memory_type_bits;
sparse_budget = device.GetSparseAddressSpaceSize();
use_sparse = true;
}
void MultiRangeBufferCache::ReserveSparseAddressSpace(VkDeviceSize size) noexcept {
sparse_budget -= (std::min)(sparse_budget, size);
}
VkDeviceSize MultiRangeBufferCache::QueryBlockSize(const Device& device,
u32& memory_type_bits) const {
const VkDevice logical = *device.GetLogical();
@@ -100,15 +106,26 @@ bool MultiRangeBufferCache::CanBindSparse(std::span<const MultiRangeSource> sour
SparseBuffer MultiRangeBufferCache::CreateSparse(const Device& device, Scheduler& scheduler,
std::span<const MultiRangeSource> sources,
VkDeviceSize total) {
VkDeviceSize total, VkBufferCreateFlags flags,
VkBufferUsageFlags usage,
VkExternalMemoryHandleTypeFlags handle_types) {
const VkDevice logical = *device.GetLogical();
const auto& dld = device.GetDispatchLoader();
const VkExternalMemoryBufferCreateInfo external_info{
.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
.pNext = nullptr,
.handleTypes = handle_types,
};
const void* next = nullptr;
if (handle_types != 0) {
next = &external_info;
}
const VkBufferCreateInfo buffer_ci{
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
.pNext = nullptr,
.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT | VK_BUFFER_CREATE_SPARSE_ALIASED_BIT,
.pNext = next,
.flags = flags,
.size = total,
.usage = sparse_usage,
.usage = usage,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = nullptr,
@@ -118,6 +135,27 @@ SparseBuffer MultiRangeBufferCache::CreateSparse(const Device& device, Scheduler
return SparseBuffer{};
}
SparseBuffer handle{raw, logical, dld};
const VkBufferMemoryRequirementsInfo2 reqs_info{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2,
.pNext = nullptr,
.buffer = raw,
};
VkMemoryRequirements2 reqs2{
.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
.pNext = nullptr,
.memoryRequirements = {},
};
dld.vkGetBufferMemoryRequirements2(logical, &reqs_info, &reqs2);
const VkMemoryRequirements& requirements = reqs2.memoryRequirements;
if (requirements.alignment == 0 || (block_size % requirements.alignment) != 0) {
return SparseBuffer{};
}
for (const MultiRangeSource& source : sources) {
if (source.memory_type >= 32 ||
((requirements.memoryTypeBits >> source.memory_type) & 1) == 0) {
return SparseBuffer{};
}
}
std::vector<VkSparseMemoryBind> binds;
binds.reserve(sources.size());
VkDeviceSize resource_offset = 0;
@@ -185,10 +223,16 @@ void MultiRangeBufferCache::RetireEntry(Scheduler& scheduler, Entry& entry) {
scheduler.Wait(oldest);
DrainRetired(scheduler);
}
VkDeviceSize sparse_size = 0;
if (entry.sparse_handle) {
sparse_size = entry.size;
}
sparse_retiring += sparse_size;
retired.push_back(Retired{
.handle = std::move(entry.sparse_handle),
.gathered = std::move(entry.gathered),
.tick = scheduler.CurrentTick(),
.sparse_size = sparse_size,
});
}
@@ -196,6 +240,8 @@ void MultiRangeBufferCache::DrainRetired(Scheduler& scheduler) {
size_t index = 0;
while (index < retired.size()) {
if (scheduler.IsFree(retired[index].tick)) {
sparse_in_use -= (std::min)(sparse_in_use, retired[index].sparse_size);
sparse_retiring -= (std::min)(sparse_retiring, retired[index].sparse_size);
if (index + 1 != retired.size()) {
retired[index] = std::move(retired.back());
}
@@ -221,6 +267,7 @@ MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& schedu
const auto it = entries.find(key);
if (it != entries.end() && it->second.geometry == geometry && it->second.size == total) {
Entry& entry = it->second;
entry.last_use = scheduler.CurrentTick();
if (entry.content != content) {
entry.content = content;
entry.dirty = true;
@@ -248,9 +295,14 @@ MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& schedu
entry.geometry = geometry;
entry.content = content;
entry.size = total;
if (CanBindSparse(sources)) {
entry.sparse_handle = CreateSparse(device, scheduler, sources, total);
entry.last_use = scheduler.CurrentTick();
if (CanBindSparse(sources) && FitsSparse(scheduler, total)) {
entry.sparse_handle = CreateSparse(
device, scheduler, sources, total,
VK_BUFFER_CREATE_SPARSE_BINDING_BIT | VK_BUFFER_CREATE_SPARSE_ALIASED_BIT,
sparse_usage, 0);
if (entry.sparse_handle) {
sparse_in_use += total;
entry.owners.reserve(sources.size());
for (const MultiRangeSource& source : sources) {
entry.owners.push_back(source.handle);
@@ -260,7 +312,9 @@ MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& schedu
if (!entry.sparse_handle) {
VkBufferUsageFlags flags = VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
VK_BUFFER_USAGE_TRANSFER_DST_BIT |
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
VK_BUFFER_USAGE_VERTEX_BUFFER_BIT |
VK_BUFFER_USAGE_INDEX_BUFFER_BIT;
if (device.IsBufferDeviceAddressSupported()) {
flags |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
}
@@ -301,6 +355,86 @@ MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& schedu
return ref;
}
MultiRangeRef MultiRangeBufferCache::GetView(const Device& device, Scheduler& scheduler, u64 key,
std::span<const MultiRangeSource> sources,
VkDeviceSize total, VkBufferUsageFlags usage,
VkExternalMemoryHandleTypeFlags handle_types) {
if (!use_sparse || !views_supported || sources.empty() || total == 0) {
return MultiRangeRef{};
}
if (!retired.empty()) {
DrainRetired(scheduler);
}
const u64 geometry = HashSources(sources);
const auto it = entries.find(key);
if (it != entries.end() && it->second.sparse_handle && it->second.geometry == geometry &&
it->second.size == total) {
it->second.last_use = scheduler.CurrentTick();
return MultiRangeRef{
.handle = *it->second.sparse_handle,
.address = it->second.address,
.size = total,
.sparse = true,
.needs_gather = false,
};
}
if (it != entries.end()) {
RetireEntry(scheduler, it->second);
entries.erase(it);
}
if (!FitsSparse(scheduler, total)) {
return MultiRangeRef{};
}
Entry entry{};
entry.sparse_handle = CreateSparse(device, scheduler, sources, total,
VK_BUFFER_CREATE_SPARSE_BINDING_BIT, usage, handle_types);
if (!entry.sparse_handle) {
views_supported = false;
return MultiRangeRef{};
}
sparse_in_use += total;
entry.geometry = geometry;
entry.size = total;
entry.last_use = scheduler.CurrentTick();
entry.dirty = false;
if ((usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0) {
entry.address = device.GetLogical().GetBufferDeviceAddress(*entry.sparse_handle);
}
const MultiRangeRef ref{
.handle = *entry.sparse_handle,
.address = entry.address,
.size = total,
.sparse = true,
.needs_gather = false,
};
entries.emplace(key, std::move(entry));
return ref;
}
bool MultiRangeBufferCache::FitsSparse(Scheduler& scheduler, VkDeviceSize total) {
if (total > sparse_budget) {
return false;
}
const u64 current_tick = scheduler.CurrentTick();
while (sparse_in_use - sparse_retiring > sparse_budget - total) {
auto victim = entries.end();
for (auto it = entries.begin(); it != entries.end(); ++it) {
if (!it->second.sparse_handle || it->second.last_use >= current_tick) {
continue;
}
if (victim == entries.end() || it->second.last_use < victim->second.last_use) {
victim = it;
}
}
if (victim == entries.end()) {
break;
}
RetireEntry(scheduler, victim->second);
entries.erase(victim);
}
return sparse_in_use <= sparse_budget - total;
}
void MultiRangeBufferCache::MarkGathered(u64 key) {
if (auto const it = entries.find(key); it != entries.end()) {
it->second.dirty = false;
src/video_core/renderer_vulkan/vk_multi_range_buffer.h
+18 -1
View File
@@ -53,12 +53,19 @@ public:
std::span<const MultiRangeSource> sources,
VkDeviceSize total);
[[nodiscard]] MultiRangeRef GetView(const Device& device, Scheduler& scheduler, u64 key,
std::span<const MultiRangeSource> sources,
VkDeviceSize total, VkBufferUsageFlags usage,
VkExternalMemoryHandleTypeFlags handle_types);
void MarkGathered(u64 key);
void Invalidate(u64 key);
void DropOwner(Scheduler& scheduler, VkBuffer owner);
void ReserveSparseAddressSpace(VkDeviceSize size) noexcept;
VkDeviceSize block_size{DEFAULT_BLOCK_SIZE};
bool use_sparse{};
@@ -67,6 +74,7 @@ private:
SparseBuffer handle;
vk::Buffer gathered;
u64 tick{};
VkDeviceSize sparse_size{};
};
struct Entry {
@@ -77,6 +85,7 @@ private:
VkDeviceSize size{};
u64 geometry{};
u64 content{};
u64 last_use{};
bool dirty{true};
};
@@ -88,7 +97,11 @@ private:
[[nodiscard]] SparseBuffer CreateSparse(const Device& device, Scheduler& scheduler,
std::span<const MultiRangeSource> sources,
VkDeviceSize total);
VkDeviceSize total, VkBufferCreateFlags flags,
VkBufferUsageFlags usage,
VkExternalMemoryHandleTypeFlags handle_types);
[[nodiscard]] bool FitsSparse(Scheduler& scheduler, VkDeviceSize total);
[[nodiscard]] VkDeviceSize QueryBlockSize(const Device& device, u32& memory_type_bits) const;
@@ -100,6 +113,10 @@ private:
boost::container::static_vector<Retired, MAX_RETIRED> retired;
u32 sparse_memory_type_bits{};
VkBufferUsageFlags sparse_usage{};
VkDeviceSize sparse_budget{};
VkDeviceSize sparse_in_use{};
VkDeviceSize sparse_retiring{};
bool views_supported{true};
};
} // namespace Vulkan
src/video_core/renderer_vulkan/vk_texture_cache.cpp
+23 -31
View File
@@ -3198,44 +3198,35 @@ u64 TextureCacheRuntime::UnifiedMemorySize() const noexcept {
return import->GetSize();
}
u64 TextureCacheRuntime::UnifiedMemoryWindowSize() const noexcept {
const HostMemoryImport* const import = memory_allocator.GetHostMemoryImport();
if (import == nullptr) {
return 0;
}
return import->GetWindowSize();
}
bool TextureCacheRuntime::CanUploadImageDirectly(const VideoCommon::ImageInfo& info) const {
return bl2d_unswizzle_pass.has_value() &&
BlockLinearUnswizzle2DPass::IsSupported(device, info);
}
VkBuffer TextureCacheRuntime::ResolveDirectWindow(size_t window_index, u64 window_offset,
u64 size) const {
if ((window_offset % device.GetStorageBufferAlignment()) != 0) {
return VK_NULL_HANDLE;
}
std::optional<HostMemoryImport::Range> TextureCacheRuntime::ResolveDirectRange(u64 relative,
u64 size) const {
if (size > device.GetMaxStorageBufferRange()) {
return VK_NULL_HANDLE;
return std::nullopt;
}
const HostMemoryImport* const import = memory_allocator.GetHostMemoryImport();
if (import == nullptr || window_index >= import->GetWindowCount()) {
return VK_NULL_HANDLE;
if (import == nullptr) {
return std::nullopt;
}
return import->GetWindowBuffer(window_index);
const auto range = import->ResolveRange(relative, size);
if (!range || range->buffer == VK_NULL_HANDLE ||
(range->offset % device.GetStorageBufferAlignment()) != 0) {
return std::nullopt;
}
return range;
}
bool TextureCacheRuntime::UploadImageDirectly(
Image& image, size_t window_index, u64 window_offset,
std::span<const VideoCommon::SwizzleParameters> swizzles) {
const VkBuffer window_buffer =
ResolveDirectWindow(window_index, window_offset, image.guest_size_bytes);
if (window_buffer == VK_NULL_HANDLE) {
Image& image, u64 relative, std::span<const VideoCommon::SwizzleParameters> swizzles) {
const auto range = ResolveDirectRange(relative, image.guest_size_bytes);
if (!range) {
return false;
}
bl2d_unswizzle_pass->UnswizzleFrom(image, window_buffer,
static_cast<VkDeviceSize>(window_offset), swizzles);
bl2d_unswizzle_pass->UnswizzleFrom(image, range->buffer, range->offset, swizzles);
return true;
}
@@ -3243,15 +3234,16 @@ bool TextureCacheRuntime::CanDownloadImageDirectly(const VideoCommon::ImageInfo&
return bl2d_swizzle_pass.has_value() && BlockLinearUnswizzle2DPass::IsSupported(device, info);
}
bool TextureCacheRuntime::DownloadImageDirectly(Image& image, size_t window_index,
u64 window_offset) {
const u64 size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes);
const VkBuffer window_buffer = ResolveDirectWindow(window_index, window_offset, size);
if (window_buffer == VK_NULL_HANDLE) {
bool TextureCacheRuntime::DownloadImageDirectly(Image& image, u64 relative) {
if (image.unswizzled_size_bytes > device.GetMaxStorageBufferRange()) {
return false;
}
const auto range = ResolveDirectRange(relative, image.guest_size_bytes);
if (!range) {
return false;
}
const HostMemoryImport* const import = memory_allocator.GetHostMemoryImport();
bl2d_swizzle_pass->SwizzleInto(image, window_buffer, static_cast<VkDeviceSize>(window_offset),
bl2d_swizzle_pass->SwizzleInto(image, range->buffer, range->offset,
import->NeedsForeignOwnershipTransfer());
return true;
}
@@ -3260,7 +3252,7 @@ u64 TextureCacheRuntime::CurrentTick() const noexcept {
return scheduler.CurrentTick();
}
bool TextureCacheRuntime::IsDirectUploadRetired(u64 tick) {
bool TextureCacheRuntime::IsTickRetired(u64 tick) {
if (scheduler.IsFree(tick)) {
return true;
}
src/video_core/renderer_vulkan/vk_texture_cache.h
+5 -7
View File
@@ -107,23 +107,21 @@ public:
[[nodiscard]] u64 UnifiedMemorySize() const noexcept;
[[nodiscard]] u64 UnifiedMemoryWindowSize() const noexcept;
[[nodiscard]] bool CanUploadImageDirectly(const VideoCommon::ImageInfo& info) const;
bool UploadImageDirectly(Image& image, size_t window_index, u64 window_offset,
bool UploadImageDirectly(Image& image, u64 relative,
std::span<const VideoCommon::SwizzleParameters> swizzles);
[[nodiscard]] bool CanDownloadImageDirectly(const VideoCommon::ImageInfo& info) const;
bool DownloadImageDirectly(Image& image, size_t window_index, u64 window_offset);
bool DownloadImageDirectly(Image& image, u64 relative);
[[nodiscard]] VkBuffer ResolveDirectWindow(size_t window_index, u64 window_offset,
u64 size) const;
[[nodiscard]] std::optional<HostMemoryImport::Range> ResolveDirectRange(u64 relative,
u64 size) const;
[[nodiscard]] u64 CurrentTick() const noexcept;
[[nodiscard]] bool IsDirectUploadRetired(u64 tick);
[[nodiscard]] bool IsTickRetired(u64 tick);
void InsertUploadMemoryBarrier() {}
src/video_core/texture_cache/image_base.h
+3
View File
@@ -97,6 +97,9 @@ struct ImageBase {
bool has_scaled = false;
u64 direct_upload_tick = 0;
bool direct_upload_blocked = false;
bool eviction_pending = false;
u64 eviction_tick = 0;
u64 eviction_modification_tick = 0;
size_t channel = 0;
src/video_core/texture_cache/texture_cache.h
+86 -34
View File
@@ -137,6 +137,9 @@ void TextureCache<P>::RunGarbageCollector() {
if (True(image.flags & ImageFlagBits::IsDecoding)) {
return false;
}
if (image.eviction_pending) {
return false;
}
const bool must_download = IsDownloadable(image) && False(image.flags & ImageFlagBits::BadOverlap);
if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) {
return false;
@@ -146,15 +149,14 @@ void TextureCache<P>::RunGarbageCollector() {
return false;
}
--num_downloads;
if (TryDownloadToUnifiedMemory(image)) {
runtime.Finish();
} else {
auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes);
const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info));
image.DownloadMemory(map, copies);
runtime.Finish();
SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span, swizzle_data_buffer);
if (StartEviction(image_id, image)) {
return false;
}
auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes);
const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info));
image.DownloadMemory(map, copies);
runtime.Finish();
SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span, swizzle_data_buffer);
}
if (True(image.flags & ImageFlagBits::Tracked)) {
UntrackImage(image, image_id);
@@ -180,6 +182,7 @@ void TextureCache<P>::RunGarbageCollector() {
template <class P>
void TextureCache<P>::TickFrame() {
FinishEvictions();
// If we can obtain the memory info, use it instead of the estimate.
if (runtime.CanReportMemoryUsage()) {
total_used_memory = runtime.GetDeviceMemoryUsage();
@@ -600,7 +603,7 @@ void TextureCache<P>::WriteMemory(DAddr cpu_addr, size_t size) {
if (image.direct_upload_tick != 0) {
const u64 upload_tick = image.direct_upload_tick;
image.direct_upload_tick = 0;
if (!runtime.IsDirectUploadRetired(upload_tick)) {
if (!runtime.IsTickRetired(upload_tick)) {
image.direct_upload_blocked = true;
}
}
@@ -910,10 +913,6 @@ void TextureCache<P>::CommitAsyncFlushes() {
bool any_none_dma = false;
for (PendingDownload& download_info : download_ids) {
if (download_info.is_swizzle) {
if (TryDownloadToUnifiedMemory(slot_images[download_info.object_id])) {
download_info.is_unified = true;
continue;
}
total_size_bytes +=
Common::AlignUp(slot_images[download_info.object_id].unswizzled_size_bytes, 64);
any_none_dma = true;
@@ -924,7 +923,7 @@ void TextureCache<P>::CommitAsyncFlushes() {
if (any_none_dma) {
auto download_map = runtime.DownloadStagingBuffer(total_size_bytes, true);
for (const PendingDownload& download_info : download_ids) {
if (download_info.is_swizzle && !download_info.is_unified) {
if (download_info.is_swizzle) {
Image& image = slot_images[download_info.object_id];
const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info));
image.DownloadMemory(download_map, copies);
@@ -956,9 +955,6 @@ void TextureCache<P>::PopAsyncFlushes() {
auto download_map = std::move(async_buffers.front());
for (size_t i = download_ids.size(); i > 0; i--) {
auto& download_info = download_ids[i - 1];
if (download_info.is_unified) {
continue;
}
auto& download_buffer = download_map[download_info.async_buffer_id];
if (download_info.is_swizzle) {
const ImageBase& image = slot_images[download_info.object_id];
@@ -1192,16 +1188,12 @@ void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) {
}
template <class P>
std::optional<std::pair<size_t, u64>> TextureCache<P>::ResolveUnifiedImageWindow(
std::optional<u64> TextureCache<P>::ResolveUnifiedImageOffset(
[[maybe_unused]] const ImageBase& image) {
if constexpr (USE_UNIFIED_MEMORY) {
if (image.guest_size_bytes == 0 || !runtime.IsUnifiedMemoryBindable()) {
return std::nullopt;
}
const u64 window_size = runtime.UnifiedMemoryWindowSize();
if (window_size == 0) {
return std::nullopt;
}
const u8* const first = gpu_memory->GetSpan(image.gpu_addr, image.guest_size_bytes);
if (first == nullptr) {
return std::nullopt;
@@ -1216,11 +1208,7 @@ std::optional<std::pair<size_t, u64>> TextureCache<P>::ResolveUnifiedImageWindow
if (relative >= unified_size || unified_size - relative < image.guest_size_bytes) {
return std::nullopt;
}
const u64 local_offset = relative % window_size;
if (window_size - local_offset < image.guest_size_bytes) {
return std::nullopt;
}
return std::pair{static_cast<size_t>(relative / window_size), local_offset};
return relative;
} else {
return std::nullopt;
}
@@ -1232,13 +1220,12 @@ bool TextureCache<P>::TryUploadFromUnifiedMemory([[maybe_unused]] Image& image)
if (image.direct_upload_blocked || !runtime.CanUploadImageDirectly(image.info)) {
return false;
}
const auto window = ResolveUnifiedImageWindow(image);
if (!window) {
const auto relative = ResolveUnifiedImageOffset(image);
if (!relative) {
return false;
}
const auto swizzles = FullUploadSwizzles(image.info);
if (!runtime.UploadImageDirectly(image, window->first, window->second,
FixSmallVectorADL(swizzles))) {
if (!runtime.UploadImageDirectly(image, *relative, FixSmallVectorADL(swizzles))) {
return false;
}
image.direct_upload_tick = runtime.CurrentTick();
@@ -1258,16 +1245,75 @@ bool TextureCache<P>::TryDownloadToUnifiedMemory([[maybe_unused]] Image& image)
image.info.layer_stride != CalculateLayerStride(image.info)) {
return false;
}
const auto window = ResolveUnifiedImageWindow(image);
if (!window) {
const auto relative = ResolveUnifiedImageOffset(image);
if (!relative) {
return false;
}
return runtime.DownloadImageDirectly(image, window->first, window->second);
return runtime.DownloadImageDirectly(image, *relative);
} else {
return false;
}
}
template <class P>
bool TextureCache<P>::StartEviction([[maybe_unused]] ImageId image_id,
[[maybe_unused]] Image& image) {
if constexpr (requires { runtime.IsTickRetired(u64{}); }) {
auto staging = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes, true);
const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info));
image.DownloadMemory(staging, copies);
eviction_staging.emplace_back(image_id, staging);
image.eviction_pending = true;
image.eviction_tick = runtime.CurrentTick();
image.eviction_modification_tick = image.modification_tick;
return true;
} else {
return false;
}
}
template <class P>
void TextureCache<P>::FinishEvictions() {
if constexpr (requires { runtime.IsTickRetired(u64{}); }) {
size_t index = 0;
while (index < eviction_staging.size()) {
const ImageId image_id = eviction_staging[index].first;
Image& image = slot_images[image_id];
if (!runtime.IsTickRetired(image.eviction_tick)) {
++index;
continue;
}
const bool unchanged = image.modification_tick == image.eviction_modification_tick;
if (unchanged && False(image.flags & ImageFlagBits::CpuModified)) {
const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info));
SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies,
eviction_staging[index].second.mapped_span, swizzle_data_buffer);
}
CancelEviction(image_id);
if (!unchanged) {
continue;
}
if (True(image.flags & ImageFlagBits::Tracked)) {
UntrackImage(image, image_id);
}
UnregisterImage(image_id);
DeleteImage(image_id, image.scale_tick > frame_tick + 5);
}
}
}
template <class P>
void TextureCache<P>::CancelEviction(ImageId image_id) {
slot_images[image_id].eviction_pending = false;
const auto it = std::ranges::find_if(
eviction_staging, [image_id](const auto& entry) { return entry.first == image_id; });
if (it == eviction_staging.end()) {
return;
}
async_buffers_death_ring.emplace_back(std::move(it->second));
eviction_staging.erase(it);
}
template <class P>
template <typename StagingBuffer>
void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging) {
@@ -2434,6 +2480,9 @@ void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) {
template <class P>
void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) {
ImageBase& image = slot_images[image_id];
if (image.eviction_pending) {
CancelEviction(image_id);
}
if (image.HasScaled()) {
total_used_memory -= GetScaledImageSizeBytes(image);
}
@@ -2621,6 +2670,9 @@ void TextureCache<P>::SynchronizeAliases(ImageId image_id) {
template <class P>
void TextureCache<P>::PrepareImage(ImageId image_id, bool is_modification, bool invalidate) {
Image& image = slot_images[image_id];
if (image.eviction_pending) {
CancelEviction(image_id);
}
if (invalidate) {
image.flags &= ~(ImageFlagBits::CpuModified | ImageFlagBits::GpuModified);
if (False(image.flags & ImageFlagBits::Tracked)) {
src/video_core/texture_cache/texture_cache_base.h
+8 -3
View File
@@ -310,13 +310,18 @@ private:
void RefreshContents(Image& image, ImageId image_id);
[[nodiscard]] std::optional<std::pair<size_t, u64>> ResolveUnifiedImageWindow(
const ImageBase& image);
[[nodiscard]] std::optional<u64> ResolveUnifiedImageOffset(const ImageBase& image);
bool TryUploadFromUnifiedMemory(Image& image);
bool TryDownloadToUnifiedMemory(Image& image);
bool StartEviction(ImageId image_id, Image& image);
void FinishEvictions();
void CancelEviction(ImageId image_id);
/// Upload data from guest to an image
template <typename StagingBuffer>
void UploadImageContents(Image& image, StagingBuffer& staging_buffer);
@@ -476,7 +481,6 @@ private:
bool is_swizzle;
size_t async_buffer_id;
Common::SlotId object_id;
bool is_unified = false;
};
Common::SlotVector<Image> slot_images;
@@ -494,6 +498,7 @@ private:
std::vector<AsyncBuffer> uncommitted_async_buffers;
std::deque<std::vector<AsyncBuffer>> async_buffers;
std::deque<AsyncBuffer> async_buffers_death_ring;
std::vector<std::pair<ImageId, AsyncBuffer>> eviction_staging;
struct LRUItemParams {
using ObjectType = ImageId;
src/video_core/vulkan_common/vulkan_device.h
+4
View File
@@ -920,6 +920,10 @@ FN_MAX_LIMIT_LIST
return properties.maintenance4.maxBufferSize;
}
u64 GetSparseAddressSpaceSize() const {
return properties.properties.limits.sparseAddressSpaceSize;
}
u64 GetMaxMemoryAllocationSize() const {
return properties.maintenance3.maxMemoryAllocationSize;
}
src/video_core/vulkan_common/vulkan_memory_allocator.cpp
+203 -7
View File
@@ -251,6 +251,7 @@ bool HostMemoryImport::ImportHostPointer(void *base, size_t size) {
return false;
}
window_size = candidate_window;
buffer_usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
const auto &logical = device.GetLogical();
const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties;
@@ -331,6 +332,9 @@ bool HostMemoryImport::ImportHostPointer(void *base, size_t size) {
windows.push_back(Window{
.memory = std::move(memory),
.buffer = new_buffer,
.address = 0,
.size = window_len,
.memory_type = *type_index,
});
imported_size += static_cast<size_t>(window_len);
}
@@ -361,7 +365,7 @@ bool HostMemoryImport::ImportHardwareBuffers(
window_size = hardware_buffer_window;
base_offset = hardware_buffer_base;
const auto import_all = [&](VkBufferUsageFlags usage, bool want_address) {
const auto import_all = [&](VkBufferUsageFlags usage, bool want_address, bool dedicated) {
for (size_t i = 0; i < hardware_buffers.size(); ++i) {
const size_t offset = hardware_buffer_base + i * hardware_buffer_window;
if (offset >= size) {
@@ -424,13 +428,17 @@ bool HostMemoryImport::ImportHardwareBuffers(
.image = VK_NULL_HANDLE,
.buffer = new_buffer,
};
const void *memory_next = &import_info;
if (dedicated) {
memory_next = &dedicated_info;
}
const VkMemoryAllocateFlagsInfo flags_info{
.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,
.pNext = &dedicated_info,
.pNext = memory_next,
.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT,
.deviceMask = 0,
};
const void *alloc_next = &dedicated_info;
const void *alloc_next = memory_next;
if (want_address) {
alloc_next = &flags_info;
}
@@ -457,6 +465,8 @@ bool HostMemoryImport::ImportHardwareBuffers(
.memory = std::move(memory),
.buffer = new_buffer,
.address = address,
.size = window_len,
.memory_type = *type_index,
});
imported_size += static_cast<size_t>(window_len);
}
@@ -487,14 +497,26 @@ bool HostMemoryImport::ImportHardwareBuffers(
imported_size = 0;
};
bindable = import_all(shader_usage, want_address);
if (!bindable) {
mirror_capable = SupportsMirror(shader_usage);
buffer_usage = shader_usage;
bindable = import_all(shader_usage, want_address, !mirror_capable);
if (!bindable && mirror_capable) {
mirror_capable = false;
reset_windows();
bindable = import_all(minimal_usage, want_address);
bindable = import_all(shader_usage, want_address, true);
}
if (bindable) {
mirror_usage = shader_usage;
}
if (!bindable) {
reset_windows();
import_all(TransferUsage, false);
buffer_usage = minimal_usage;
bindable = import_all(minimal_usage, want_address, true);
}
if (!bindable) {
reset_windows();
buffer_usage = TransferUsage;
import_all(TransferUsage, false, true);
}
if (windows.empty()) {
window_size = 0;
@@ -508,7 +530,181 @@ bool HostMemoryImport::ImportHardwareBuffers(
#endif
}
bool HostMemoryImport::SupportsMirror([[maybe_unused]] VkBufferUsageFlags usage) const {
#ifdef __ANDROID__
if (!device.IsSparseBindingSupported()) {
return false;
}
const auto supports = [&](VkBufferCreateFlags flags) {
const VkExternalMemoryProperties properties =
device.GetPhysical().GetExternalBufferProperties(
flags, usage,
VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID);
const VkExternalMemoryFeatureFlags features = properties.externalMemoryFeatures;
return (features & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT) != 0 &&
(features & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT) == 0;
};
return supports(0) && supports(VK_BUFFER_CREATE_SPARSE_BINDING_BIT);
#else
return false;
#endif
}
std::optional<HostMemoryImport::Range> HostMemoryImport::ResolveRange(
VkDeviceSize relative, VkDeviceSize size) const noexcept {
if (window_size == 0 || size == 0 || relative >= imported_size ||
imported_size - relative < size) {
return std::nullopt;
}
const size_t index = static_cast<size_t>(relative / window_size);
const VkDeviceSize local_offset = relative % window_size;
if (index < windows.size() && windows[index].buffer != VK_NULL_HANDLE &&
local_offset < windows[index].size && windows[index].size - local_offset >= size) {
return Range{
.buffer = windows[index].buffer,
.address = windows[index].address,
.offset = local_offset,
};
}
if (mirror_buffer != VK_NULL_HANDLE && relative < mirror_size &&
mirror_size - relative >= size) {
return Range{
.buffer = mirror_buffer,
.address = mirror_address,
.offset = relative,
};
}
return std::nullopt;
}
std::optional<HostMemoryImport::ViewMemory> HostMemoryImport::ResolveViewMemory(
VkDeviceSize relative) const noexcept {
if (!mirror_capable || !bindable || window_size == 0) {
return std::nullopt;
}
const size_t index = static_cast<size_t>(relative / window_size);
if (index >= windows.size()) {
return std::nullopt;
}
const Window &window = windows[index];
const VkDeviceSize local_offset = relative % window_size;
if (window.buffer == VK_NULL_HANDLE || local_offset >= window.size) {
return std::nullopt;
}
return ViewMemory{
.buffer = window.buffer,
.memory = *window.memory,
.offset = local_offset,
.available = window.size - local_offset,
.memory_type = window.memory_type,
};
}
void HostMemoryImport::CreateMirror(std::mutex &submit_mutex, VkDeviceSize max_size) {
if (!mirror_capable || mirror_usage == 0 || !bindable || window_size == 0) {
return;
}
size_t mirror_windows = 0;
while (mirror_windows < windows.size() && windows[mirror_windows].size == window_size) {
++mirror_windows;
}
mirror_windows = (std::min)(mirror_windows, static_cast<size_t>(max_size / window_size));
const u64 max_buffer_size = device.GetMaxBufferSize();
if (max_buffer_size != 0) {
const size_t max_windows = static_cast<size_t>(max_buffer_size / window_size);
mirror_windows = (std::min)(mirror_windows, max_windows);
}
if (mirror_windows < 2) {
return;
}
const auto &logical = device.GetLogical();
const VkDeviceSize total_size = static_cast<VkDeviceSize>(mirror_windows) * window_size;
const VkExternalMemoryBufferCreateInfo external_info{
.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
.pNext = nullptr,
.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID,
};
const VkBufferCreateInfo buffer_ci{
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
.pNext = &external_info,
.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT,
.size = total_size,
.usage = mirror_usage,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = nullptr,
};
VkBuffer buffer{};
if (logical.CreateBufferRaw(buffer_ci, &buffer) != VK_SUCCESS) {
return;
}
const VkMemoryRequirements requirements = logical.GetBufferMemoryRequirements(buffer);
bool compatible = requirements.alignment != 0 &&
(window_size % requirements.alignment) == 0 &&
requirements.size <= total_size;
for (size_t index = 0; index < mirror_windows && compatible; ++index) {
compatible = ((requirements.memoryTypeBits >> windows[index].memory_type) & 1u) != 0;
}
if (!compatible) {
logical.DestroyBufferRaw(buffer);
return;
}
std::vector<VkSparseMemoryBind> binds;
binds.reserve(mirror_windows);
for (size_t index = 0; index < mirror_windows; ++index) {
binds.push_back(VkSparseMemoryBind{
.resourceOffset = static_cast<VkDeviceSize>(index) * window_size,
.size = window_size,
.memory = *windows[index].memory,
.memoryOffset = 0,
.flags = 0,
});
}
const VkSparseBufferMemoryBindInfo buffer_bind{
.buffer = buffer,
.bindCount = static_cast<u32>(binds.size()),
.pBinds = binds.data(),
};
const VkBindSparseInfo bind_info{
.sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO,
.pNext = nullptr,
.waitSemaphoreCount = 0,
.pWaitSemaphores = nullptr,
.bufferBindCount = 1,
.pBufferBinds = &buffer_bind,
.imageOpaqueBindCount = 0,
.pImageOpaqueBinds = nullptr,
.imageBindCount = 0,
.pImageBinds = nullptr,
.signalSemaphoreCount = 0,
.pSignalSemaphores = nullptr,
};
vk::Fence fence = logical.CreateFence(VkFenceCreateInfo{
.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO,
.pNext = nullptr,
.flags = 0,
});
VkResult result = VK_ERROR_UNKNOWN;
{
std::scoped_lock lock{submit_mutex};
result = device.GetGraphicsQueue().BindSparse(bind_info, *fence);
}
if (result != VK_SUCCESS) {
logical.DestroyBufferRaw(buffer);
return;
}
fence.Wait();
mirror_buffer = buffer;
mirror_size = total_size;
if ((mirror_usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0) {
mirror_address = logical.GetBufferDeviceAddress(buffer);
}
}
HostMemoryImport::~HostMemoryImport() {
if (mirror_buffer != VK_NULL_HANDLE) {
device.GetLogical().DestroyBufferRaw(mirror_buffer);
}
for (Window &window : windows) {
if (window.buffer != VK_NULL_HANDLE) {
device.GetLogical().DestroyBufferRaw(window.buffer);
src/video_core/vulkan_common/vulkan_memory_allocator.h
+49
View File
@@ -7,6 +7,8 @@
#pragma once
#include <memory>
#include <mutex>
#include <optional>
#include <span>
#include <vector>
@@ -134,13 +136,54 @@ namespace Vulkan {
return bindable;
}
[[nodiscard]] VkBufferUsageFlags GetUsage() const noexcept {
return buffer_usage;
}
struct Range {
VkBuffer buffer{};
VkDeviceAddress address{};
VkDeviceSize offset{};
};
[[nodiscard]] std::optional<Range> ResolveRange(VkDeviceSize relative,
VkDeviceSize size) const noexcept;
struct ViewMemory {
VkBuffer buffer{};
VkDeviceMemory memory{};
VkDeviceSize offset{};
VkDeviceSize available{};
u32 memory_type{};
};
[[nodiscard]] std::optional<ViewMemory> ResolveViewMemory(
VkDeviceSize relative) const noexcept;
[[nodiscard]] VkBufferUsageFlags GetViewUsage() const noexcept {
if (!mirror_capable || !bindable) {
return 0;
}
return mirror_usage;
}
void CreateMirror(std::mutex &submit_mutex, VkDeviceSize max_size);
[[nodiscard]] VkDeviceSize GetMirrorSize() const noexcept {
return mirror_size;
}
private:
struct Window {
vk::DeviceMemory memory;
VkBuffer buffer{};
VkDeviceAddress address{};
VkDeviceSize size{};
u32 memory_type{};
};
[[nodiscard]] bool SupportsMirror(VkBufferUsageFlags usage) const;
bool ImportHostPointer(void *base, size_t size);
bool ImportHardwareBuffers(std::span<AHardwareBuffer *const> hardware_buffers,
@@ -154,6 +197,12 @@ namespace Vulkan {
size_t base_offset{};
bool foreign_ownership{};
bool bindable{};
bool mirror_capable{};
VkBufferUsageFlags buffer_usage{};
VkBufferUsageFlags mirror_usage{};
VkBuffer mirror_buffer{};
VkDeviceAddress mirror_address{};
VkDeviceSize mirror_size{};
};
/// Memory allocator container.
src/video_core/vulkan_common/vulkan_wrapper.cpp
+23
View File
@@ -316,6 +316,7 @@ bool Load(VkInstance instance, InstanceDispatch& dld) noexcept {
X(vkDestroyDebugUtilsMessengerEXT);
X(vkDestroyDebugReportCallbackEXT);
X(vkDestroySurfaceKHR);
X(vkGetPhysicalDeviceExternalBufferProperties);
X(vkGetPhysicalDeviceFeatures2);
X(vkGetPhysicalDeviceFormatProperties2);
X(vkGetPhysicalDeviceProperties2);
@@ -1038,6 +1039,28 @@ VkPhysicalDeviceMemoryProperties2 PhysicalDevice::GetMemoryProperties(
return properties;
}
VkExternalMemoryProperties PhysicalDevice::GetExternalBufferProperties(
VkBufferCreateFlags flags, VkBufferUsageFlags usage,
VkExternalMemoryHandleTypeFlagBits handle_type) const noexcept {
VkExternalBufferProperties properties{
.sType = VK_STRUCTURE_TYPE_EXTERNAL_BUFFER_PROPERTIES,
.pNext = nullptr,
.externalMemoryProperties = {},
};
if (!dld->vkGetPhysicalDeviceExternalBufferProperties) {
return properties.externalMemoryProperties;
}
const VkPhysicalDeviceExternalBufferInfo info{
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_BUFFER_INFO,
.pNext = nullptr,
.flags = flags,
.usage = usage,
.handleType = handle_type,
};
dld->vkGetPhysicalDeviceExternalBufferProperties(physical_device, &info, &properties);
return properties.externalMemoryProperties;
}
u32 AvailableVersion(const InstanceDispatch& dld) noexcept {
PFN_vkEnumerateInstanceVersion vkEnumerateInstanceVersion;
if (!Proc(vkEnumerateInstanceVersion, dld, "vkEnumerateInstanceVersion")) {
src/video_core/vulkan_common/vulkan_wrapper.h
+5
View File
@@ -178,6 +178,7 @@ struct InstanceDispatch {
PFN_vkEnumerateDeviceExtensionProperties vkEnumerateDeviceExtensionProperties{};
PFN_vkEnumeratePhysicalDevices vkEnumeratePhysicalDevices{};
PFN_vkGetDeviceProcAddr vkGetDeviceProcAddr{};
PFN_vkGetPhysicalDeviceExternalBufferProperties vkGetPhysicalDeviceExternalBufferProperties{};
PFN_vkGetPhysicalDeviceFeatures2 vkGetPhysicalDeviceFeatures2{};
PFN_vkGetPhysicalDeviceFormatProperties vkGetPhysicalDeviceFormatProperties{};
PFN_vkGetPhysicalDeviceFormatProperties2 vkGetPhysicalDeviceFormatProperties2{};
@@ -1243,6 +1244,10 @@ public:
VkPhysicalDeviceMemoryProperties2 GetMemoryProperties(
void* next_structures = nullptr) const noexcept;
VkExternalMemoryProperties GetExternalBufferProperties(
VkBufferCreateFlags flags, VkBufferUsageFlags usage,
VkExternalMemoryHandleTypeFlagBits handle_type) const noexcept;
private:
VkPhysicalDevice physical_device = nullptr;
const InstanceDispatch* dld = nullptr;