From 11d65f2ab77d7a68e5931bd2520b53f2387454f1 Mon Sep 17 00:00:00 2001 From: xbzk Date: Fri, 4 Sep 2026 13:28:23 -0300 Subject: [PATCH] [vk] map fragmented storage buffers with descriptor arrays (mhr fix) --- .../backend/spirv/emit_spirv.cpp | 3 + .../backend/spirv/emit_spirv_atomic.cpp | 65 ++--- .../backend/spirv/emit_spirv_memory.cpp | 31 +-- .../backend/spirv/spirv_emit_context.cpp | 217 ++++++++++++----- .../backend/spirv/spirv_emit_context.h | 10 +- .../frontend/maxwell/translate_program.cpp | 10 + src/shader_recompiler/host_translate_info.h | 5 + src/shader_recompiler/profile.h | 1 + src/shader_recompiler/shader_info.h | 5 + src/video_core/buffer_cache/buffer_cache.h | 222 +++++++++++++----- .../buffer_cache/buffer_cache_base.h | 25 +- .../renderer_vulkan/pipeline_helper.h | 8 + .../renderer_vulkan/vk_buffer_cache.cpp | 2 +- .../renderer_vulkan/vk_buffer_cache.h | 7 + .../renderer_vulkan/vk_compute_pipeline.cpp | 4 +- .../renderer_vulkan/vk_descriptor_pool.cpp | 2 +- .../renderer_vulkan/vk_graphics_pipeline.cpp | 3 +- .../renderer_vulkan/vk_pipeline_cache.cpp | 24 +- .../vulkan_common/vulkan_device.cpp | 1 - src/video_core/vulkan_common/vulkan_device.h | 5 + 20 files changed, 440 insertions(+), 210 deletions(-) diff --git a/src/shader_recompiler/backend/spirv/emit_spirv.cpp b/src/shader_recompiler/backend/spirv/emit_spirv.cpp index c221888c09..404c68c229 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv.cpp @@ -492,6 +492,9 @@ void SetupCapabilities(const Profile& profile, const Info& info, EmitContext& ct if (ctx.uses_nonuniform_storage_texel_buffer) { ctx.AddCapability(spv::Capability::StorageTexelBufferArrayNonUniformIndexing); } + if (ctx.uses_nonuniform_storage_buffer) { + ctx.AddCapability(spv::Capability::StorageBufferArrayNonUniformIndexing); + } } } diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp index d1c08c1131..c1b70bfa9a 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp @@ -4,8 +4,6 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later -#include - #include "shader_recompiler/backend/spirv/emit_spirv.h" #include "shader_recompiler/backend/spirv/emit_spirv_instructions.h" #include "shader_recompiler/backend/spirv/spirv_emit_context.h" @@ -23,29 +21,13 @@ Id SharedPointer(EmitContext& ctx, Id offset, u32 index_offset = 0) { : ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index); } -Id StorageIndex(EmitContext& ctx, const IR::Value& offset, size_t element_size) { - if (offset.IsImmediate()) { - const u32 imm_offset{static_cast(offset.U32() / element_size)}; - return ctx.Const(imm_offset); - } - const u32 shift{static_cast(std::countr_zero(element_size))}; - const Id index{ctx.Def(offset)}; - if (shift == 0) { - return index; - } - const Id shift_id{ctx.Const(shift)}; - return ctx.OpShiftRightLogical(ctx.U32[1], index, shift_id); -} - Id StoragePointer(EmitContext& ctx, const StorageTypeDefinition& type_def, Id StorageDefinitions::*member_ptr, const IR::Value& binding, const IR::Value& offset, size_t element_size) { if (!binding.IsImmediate()) { throw NotImplementedException("Dynamic storage buffer indexing"); } - const Id ssbo{ctx.ssbos[binding.U32()].*member_ptr}; - const Id index{StorageIndex(ctx, offset, element_size)}; - return ctx.OpAccessChain(type_def.element, ssbo, ctx.u32_zero_value, index); + return ctx.StoragePointer(binding.U32(), ctx.Def(offset), type_def, static_cast(element_size), member_ptr); } std::pair AtomicArgs(EmitContext& ctx) { @@ -216,16 +198,14 @@ Id EmitStorageAtomicUMax32(EmitContext& ctx, const IR::Value& binding, const IR: Id EmitStorageAtomicInc32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value) { - const Id ssbo{ctx.ssbos[binding.U32()].U32}; - const Id base_index{StorageIndex(ctx, offset, sizeof(u32))}; - return ctx.OpFunctionCall(ctx.U32[1], ctx.increment_cas_ssbo, base_index, value, ssbo); + const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding, offset, sizeof(u32))}; + return ctx.OpFunctionCall(ctx.U32[1], ctx.increment_cas_ssbo, pointer, value); } Id EmitStorageAtomicDec32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value) { - const Id ssbo{ctx.ssbos[binding.U32()].U32}; - const Id base_index{StorageIndex(ctx, offset, sizeof(u32))}; - return ctx.OpFunctionCall(ctx.U32[1], ctx.decrement_cas_ssbo, base_index, value, ssbo); + const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding, offset, sizeof(u32))}; + return ctx.OpFunctionCall(ctx.U32[1], ctx.decrement_cas_ssbo, pointer, value); } Id EmitStorageAtomicAnd32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, @@ -364,56 +344,49 @@ Id EmitStorageAtomicExchange32x2(EmitContext& ctx, const IR::Value& binding, Id EmitStorageAtomicAddF32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value) { - const Id ssbo{ctx.ssbos[binding.U32()].U32}; - const Id base_index{StorageIndex(ctx, offset, sizeof(u32))}; - return ctx.OpFunctionCall(ctx.F32[1], ctx.f32_add_cas, base_index, value, ssbo); + const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding, offset, sizeof(u32))}; + return ctx.OpFunctionCall(ctx.F32[1], ctx.f32_add_cas, pointer, value); } Id EmitStorageAtomicAddF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value) { - const Id ssbo{ctx.ssbos[binding.U32()].U32}; - const Id base_index{StorageIndex(ctx, offset, sizeof(u32))}; - const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_add_cas, base_index, value, ssbo)}; + const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding, offset, sizeof(u32))}; + const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_add_cas, pointer, value)}; return ctx.OpBitcast(ctx.U32[1], result); } Id EmitStorageAtomicAddF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value) { - const Id ssbo{ctx.ssbos[binding.U32()].U32}; - const Id base_index{StorageIndex(ctx, offset, sizeof(u32))}; - const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_add_cas, base_index, value, ssbo)}; + const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding, offset, sizeof(u32))}; + const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_add_cas, pointer, value)}; return ctx.OpPackHalf2x16(ctx.U32[1], result); } Id EmitStorageAtomicMinF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value) { - const Id ssbo{ctx.ssbos[binding.U32()].U32}; - const Id base_index{StorageIndex(ctx, offset, sizeof(u32))}; - const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_min_cas, base_index, value, ssbo)}; + const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding, offset, sizeof(u32))}; + const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_min_cas, pointer, value)}; return ctx.OpBitcast(ctx.U32[1], result); } Id EmitStorageAtomicMinF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value) { - const Id ssbo{ctx.ssbos[binding.U32()].U32}; - const Id base_index{StorageIndex(ctx, offset, sizeof(u32))}; - const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_min_cas, base_index, value, ssbo)}; + const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding, offset, sizeof(u32))}; + const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_min_cas, pointer, value)}; return ctx.OpPackHalf2x16(ctx.U32[1], result); } Id EmitStorageAtomicMaxF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value) { - const Id ssbo{ctx.ssbos[binding.U32()].U32}; - const Id base_index{StorageIndex(ctx, offset, sizeof(u32))}; - const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_max_cas, base_index, value, ssbo)}; + const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding, offset, sizeof(u32))}; + const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_max_cas, pointer, value)}; return ctx.OpBitcast(ctx.U32[1], result); } Id EmitStorageAtomicMaxF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value) { - const Id ssbo{ctx.ssbos[binding.U32()].U32}; - const Id base_index{StorageIndex(ctx, offset, sizeof(u32))}; - const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_max_cas, base_index, value, ssbo)}; + const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding, offset, sizeof(u32))}; + const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_max_cas, pointer, value)}; return ctx.OpPackHalf2x16(ctx.U32[1], result); } diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp index bef4421fed..493e5d38ca 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp @@ -4,46 +4,33 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later -#include - #include "shader_recompiler/backend/spirv/emit_spirv.h" #include "shader_recompiler/backend/spirv/emit_spirv_instructions.h" #include "shader_recompiler/backend/spirv/spirv_emit_context.h" namespace Shader::Backend::SPIRV { namespace { -Id StorageIndex(EmitContext& ctx, const IR::Value& offset, size_t element_size, - u32 index_offset = 0) { - if (offset.IsImmediate()) { - const u32 imm_offset{static_cast(offset.U32() / element_size) + index_offset}; - return ctx.Const(imm_offset); - } - const u32 shift{static_cast(std::countr_zero(element_size))}; - Id index{ctx.Def(offset)}; - if (shift != 0) { - const Id shift_id{ctx.Const(shift)}; - index = ctx.OpShiftRightLogical(ctx.U32[1], index, shift_id); - } +Id StorageByteOffset(EmitContext& ctx, const IR::Value& offset, size_t element_size, u32 index_offset) { + Id byte_offset{ctx.Def(offset)}; if (index_offset != 0) { - index = ctx.OpIAdd(ctx.U32[1], index, ctx.Const(index_offset)); + byte_offset = ctx.OpIAdd(ctx.U32[1], byte_offset, ctx.Const(static_cast(index_offset * element_size))); } - return index; + return byte_offset; } Id StoragePointer(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, const StorageTypeDefinition& type_def, size_t element_size, - Id StorageDefinitions::*member_ptr, u32 index_offset = 0) { + Id StorageDefinitions::* member_ptr, u32 index_offset = 0) { if (!binding.IsImmediate()) { throw NotImplementedException("Dynamic storage buffer indexing"); } - const Id ssbo{ctx.ssbos[binding.U32()].*member_ptr}; - const Id index{StorageIndex(ctx, offset, element_size, index_offset)}; - return ctx.OpAccessChain(type_def.element, ssbo, ctx.u32_zero_value, index); + const Id byte_offset{StorageByteOffset(ctx, offset, element_size, index_offset)}; + return ctx.StoragePointer(binding.U32(), byte_offset, type_def, static_cast(element_size), member_ptr); } Id LoadStorage(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id result_type, const StorageTypeDefinition& type_def, size_t element_size, - Id StorageDefinitions::*member_ptr, u32 index_offset = 0) { + Id StorageDefinitions::* member_ptr, u32 index_offset = 0) { const Id pointer{ StoragePointer(ctx, binding, offset, type_def, element_size, member_ptr, index_offset)}; return ctx.OpLoad(result_type, pointer); @@ -57,7 +44,7 @@ Id LoadStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& of void WriteStorage(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value, const StorageTypeDefinition& type_def, size_t element_size, - Id StorageDefinitions::*member_ptr, u32 index_offset = 0) { + Id StorageDefinitions::* member_ptr, u32 index_offset = 0) { const Id pointer{ StoragePointer(ctx, binding, offset, type_def, element_size, member_ptr, index_offset)}; ctx.OpStore(pointer, value); diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp index b562e82983..d6d062a416 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp @@ -299,7 +299,7 @@ void DefineConstBuffers(EmitContext& ctx, const Info& info, Id UniformDefinition } void DefineSsbos(EmitContext& ctx, StorageTypeDefinition& type_def, - Id StorageDefinitions::*member_type, const Info& info, u32 binding, Id type, + Id StorageDefinitions::* member_type, const Info& info, u32 binding, Id type, u32 stride) { const Id array_type{ctx.TypeRuntimeArray(type)}; ctx.Decorate(array_type, spv::Decoration::ArrayStride, stride); @@ -309,23 +309,27 @@ void DefineSsbos(EmitContext& ctx, StorageTypeDefinition& type_def, ctx.MemberDecorate(struct_type, 0, spv::Decoration::Offset, 0U); const Id struct_pointer{ctx.TypePointer(spv::StorageClass::StorageBuffer, struct_type)}; - type_def.array = struct_pointer; type_def.element = ctx.TypePointer(spv::StorageClass::StorageBuffer, type); u32 index{}; for (const StorageBufferDescriptor& desc : info.storage_buffers_descriptors) { - const Id id{ctx.AddGlobalVariable(struct_pointer, spv::StorageClass::StorageBuffer)}; + const Id variable_type{[&] { + if (desc.count == 1) { + return struct_pointer; + } + const Id descriptor_array{ctx.TypeArray(struct_type, ctx.Const(desc.count))}; + return ctx.TypePointer(spv::StorageClass::StorageBuffer, descriptor_array); + }()}; + const Id id{ctx.AddGlobalVariable(variable_type, spv::StorageClass::StorageBuffer)}; ctx.Decorate(id, spv::Decoration::Binding, binding); ctx.Decorate(id, spv::Decoration::DescriptorSet, 0U); ctx.Name(id, fmt::format("ssbo{}", index)); if (ctx.profile.supported_spirv >= 0x00010400) { ctx.interfaces.push_back(id); } - for (size_t i = 0; i < desc.count; ++i) { - ctx.ssbos[index + i].*member_type = id; - } - index += desc.count; - binding += desc.count; + ctx.ssbos[index].*member_type = id; + ++index; + ++binding; } } @@ -368,8 +372,7 @@ Id CasFunction(EmitContext& ctx, Operation operation, Id value_type) { return func; } -Id CasLoop(EmitContext& ctx, Operation operation, Id array_pointer, Id element_pointer, - Id value_type, Id memory_type, spv::Scope scope) { +Id CasLoop(EmitContext& ctx, Operation operation, Id element_pointer, Id value_type, Id memory_type, spv::Scope scope) { const bool is_shared{scope == spv::Scope::Workgroup}; const bool is_struct{!is_shared || ctx.uses_explicit_workgroup_layout}; const Id cas_func{CasFunction(ctx, operation, value_type)}; @@ -379,14 +382,12 @@ Id CasLoop(EmitContext& ctx, Operation operation, Id array_pointer, Id element_p const Id loop_header{ctx.OpLabel()}; const Id continue_block{ctx.OpLabel()}; const Id merge_block{ctx.OpLabel()}; - const Id func_type{is_shared - ? ctx.TypeFunction(value_type, ctx.U32[1], value_type) - : ctx.TypeFunction(value_type, ctx.U32[1], value_type, array_pointer)}; + const Id func_type{is_shared ? ctx.TypeFunction(value_type, ctx.U32[1], value_type) + : ctx.TypeFunction(value_type, element_pointer, value_type)}; const Id func{ctx.OpFunction(value_type, spv::FunctionControlMask::MaskNone, func_type)}; - const Id index{ctx.OpFunctionParameter(ctx.U32[1])}; + const Id address{ctx.OpFunctionParameter(is_shared ? ctx.U32[1] : element_pointer)}; const Id op_b{ctx.OpFunctionParameter(value_type)}; - const Id base{is_shared ? ctx.shared_memory_u32 : ctx.OpFunctionParameter(array_pointer)}; ctx.AddLabel(); ctx.OpBranch(loop_header); ctx.AddLabel(loop_header); @@ -395,8 +396,13 @@ Id CasLoop(EmitContext& ctx, Operation operation, Id array_pointer, Id element_p ctx.OpBranch(continue_block); ctx.AddLabel(continue_block); - const Id word_pointer{is_struct ? ctx.OpAccessChain(element_pointer, base, zero, index) - : ctx.OpAccessChain(element_pointer, base, index)}; + const Id word_pointer{[&] { + if (!is_shared) { + return address; + } + return is_struct ? ctx.OpAccessChain(element_pointer, ctx.shared_memory_u32, zero, address) + : ctx.OpAccessChain(element_pointer, ctx.shared_memory_u32, address); + }()}; if (value_type.value == ctx.F32[2].value) { const Id u32_value{ctx.OpLoad(ctx.U32[1], word_pointer)}; const Id value{ctx.OpUnpackHalf2x16(ctx.F32[2], u32_value)}; @@ -480,6 +486,7 @@ EmitContext::EmitContext(const Profile& profile_, const RuntimeInfo& runtime_inf DefineSharedMemoryFunctions(program); DefineConstantBuffers(program.info, uniform_binding); DefineConstantBufferIndirectFunctions(program.info); + DefineStorageBufferMappings(program.info, storage_binding); DefineStorageBuffers(program.info, storage_binding); DefineTextureBuffers(program.info, texture_binding); DefineImageBuffers(program.info, image_binding); @@ -532,6 +539,36 @@ Id EmitContext::BitOffset16(const IR::Value& offset) { return OpBitwiseAnd(U32[1], OpShiftLeftLogical(U32[1], Def(offset), Const(3u)), Const(16u)); } +Id EmitContext::StoragePointer(u32 binding, Id byte_offset, const StorageTypeDefinition& type_def, + u32 element_size, Id StorageDefinitions::* member_ptr) { + const Id ssbo{ssbos[binding].*member_ptr}; + const u32 segment_count{storage_buffer_mapping_counts[binding]}; + if (segment_count <= 1) { + const Id index{ + element_size == 1 + ? byte_offset + : OpShiftRightLogical(U32[1], byte_offset, Const(static_cast(std::countr_zero(element_size))))}; + return OpAccessChain(type_def.element, ssbo, u32_zero_value, index); + } + + const Id mapped{OpFunctionCall(U32[2], storage_buffer_map_func, + Const(storage_buffer_mapping_bases[binding]), + Const(segment_count), byte_offset)}; + const Id segment{OpCompositeExtract(U32[1], mapped, 0U)}; + const Id local_offset{OpCompositeExtract(U32[1], mapped, 1U)}; + const Id index{ + element_size == 1 + ? local_offset + : OpShiftRightLogical(U32[1], local_offset, Const(static_cast(std::countr_zero(element_size))))}; + Decorate(segment, spv::Decoration::NonUniform); + non_uniform_ids.insert(segment.value); + const Id pointer{OpAccessChain(type_def.element, ssbo, segment, u32_zero_value, index)}; + Decorate(pointer, spv::Decoration::NonUniform); + non_uniform_ids.insert(pointer.value); + uses_nonuniform_storage_buffer = true; + return pointer; +} + void EmitContext::DefineCommonTypes(const Info& info) { void_id = TypeVoid(); @@ -696,12 +733,10 @@ void EmitContext::DefineSharedMemory(const IR::Program& program) { void EmitContext::DefineSharedMemoryFunctions(const IR::Program& program) { if (program.info.uses_shared_increment) { - increment_cas_shared = CasLoop(*this, Operation::Increment, shared_memory_u32_type, - shared_u32, U32[1], U32[1], spv::Scope::Workgroup); + increment_cas_shared = CasLoop(*this, Operation::Increment, shared_u32, U32[1], U32[1], spv::Scope::Workgroup); } if (program.info.uses_shared_decrement) { - decrement_cas_shared = CasLoop(*this, Operation::Decrement, shared_memory_u32_type, - shared_u32, U32[1], U32[1], spv::Scope::Workgroup); + decrement_cas_shared = CasLoop(*this, Operation::Decrement, shared_u32, U32[1], U32[1], spv::Scope::Workgroup); } } @@ -945,8 +980,7 @@ void EmitContext::DefineGlobalMemoryFunctions(const Info& info) { } using DefPtr = Id StorageDefinitions::*; const Id zero{u32_zero_value}; - const auto define_body{[&](DefPtr ssbo_member, Id addr, Id element_pointer, u32 shift, - auto&& callback) { + const auto define_body{[&](DefPtr ssbo_member, Id addr, const StorageTypeDefinition& type_def, u32 shift, auto&& callback) { AddLabel(); const size_t num_buffers{info.storage_buffers_descriptors.size()}; for (size_t index = 0; index < num_buffers; ++index) { @@ -973,30 +1007,28 @@ void EmitContext::DefineGlobalMemoryFunctions(const Info& info) { OpSelectionMerge(else_label, spv::SelectionControlMask::MaskNone); OpBranchConditional(cond, then_label, else_label); AddLabel(then_label); - const Id ssbo_id{ssbos[index].*ssbo_member}; const Id ssbo_offset{OpUConvert(U32[1], OpISub(U64, addr, ssbo_addr))}; - const Id ssbo_index{OpShiftRightLogical(U32[1], ssbo_offset, Const(shift))}; - const Id ssbo_pointer{OpAccessChain(element_pointer, ssbo_id, zero, ssbo_index)}; + const Id ssbo_pointer{StoragePointer(static_cast(index), ssbo_offset, type_def, 1U << shift, ssbo_member)}; callback(ssbo_pointer); AddLabel(else_label); } }}; - const auto define_load{[&](DefPtr ssbo_member, Id element_pointer, Id type, u32 shift) { - const Id function_type{TypeFunction(type, U64)}; - const Id func_id{OpFunction(type, spv::FunctionControlMask::MaskNone, function_type)}; - const Id addr{OpFunctionParameter(U64)}; - define_body(ssbo_member, addr, element_pointer, shift, - [&](Id ssbo_pointer) { OpReturnValue(OpLoad(type, ssbo_pointer)); }); - OpReturnValue(ConstantNull(type)); - OpFunctionEnd(); - return func_id; - }}; - const auto define_write{[&](DefPtr ssbo_member, Id element_pointer, Id type, u32 shift) { + const auto define_load{ + [&](DefPtr ssbo_member, const StorageTypeDefinition& type_def, Id type, u32 shift) { + const Id function_type{TypeFunction(type, U64)}; + const Id func_id{OpFunction(type, spv::FunctionControlMask::MaskNone, function_type)}; + const Id addr{OpFunctionParameter(U64)}; + define_body(ssbo_member, addr, type_def, shift, [&](Id ssbo_pointer) { OpReturnValue(OpLoad(type, ssbo_pointer)); }); + OpReturnValue(ConstantNull(type)); + OpFunctionEnd(); + return func_id; + }}; + const auto define_write{[&](DefPtr ssbo_member, const StorageTypeDefinition& type_def, Id type, u32 shift) { const Id function_type{TypeFunction(void_id, U64, type)}; const Id func_id{OpFunction(void_id, spv::FunctionControlMask::MaskNone, function_type)}; const Id addr{OpFunctionParameter(U64)}; const Id data{OpFunctionParameter(type)}; - define_body(ssbo_member, addr, element_pointer, shift, [&](Id ssbo_pointer) { + define_body(ssbo_member, addr, type_def, shift, [&](Id ssbo_pointer) { OpStore(ssbo_pointer, data); OpReturn(); }); @@ -1006,10 +1038,9 @@ void EmitContext::DefineGlobalMemoryFunctions(const Info& info) { }}; const auto define{ [&](DefPtr ssbo_member, const StorageTypeDefinition& type_def, Id type, size_t size) { - const Id element_type{type_def.element}; const u32 shift{static_cast(std::countr_zero(size))}; - const Id load_func{define_load(ssbo_member, element_type, type, shift)}; - const Id write_func{define_write(ssbo_member, element_type, type, shift)}; + const Id load_func{define_load(ssbo_member, type_def, type, shift)}; + const Id write_func{define_write(ssbo_member, type_def, type, shift)}; return std::make_pair(load_func, write_func); }}; std::tie(load_global_func_u32, write_global_func_u32) = @@ -1228,6 +1259,79 @@ void EmitContext::DefineConstantBufferIndirectFunctions(const Info& info) { } } +void EmitContext::DefineStorageBufferMappings(const Info& info, u32& binding) { + if (!UsesStorageBufferMappings(info)) { + return; + } + ASSERT(profile.support_storage_buffer_array_nonuniform_indexing); + AddExtension("SPV_KHR_storage_buffer_storage_class"); + + const u32 num_entries{NumDescriptors(info.storage_buffers_descriptors)}; + const Id array_type{TypeArray(U32[1], Const(num_entries))}; + Decorate(array_type, spv::Decoration::ArrayStride, sizeof(u32)); + const Id struct_type{TypeStruct(array_type)}; + Decorate(struct_type, spv::Decoration::Block); + MemberName(struct_type, 0, "segment_sizes"); + MemberDecorate(struct_type, 0, spv::Decoration::Offset, 0U); + const Id pointer_type{TypePointer(spv::StorageClass::StorageBuffer, struct_type)}; + storage_buffer_mapping_u32 = TypePointer(spv::StorageClass::StorageBuffer, U32[1]); + storage_buffer_mapping = AddGlobalVariable(pointer_type, spv::StorageClass::StorageBuffer); + Decorate(storage_buffer_mapping, spv::Decoration::Binding, binding++); + Decorate(storage_buffer_mapping, spv::Decoration::DescriptorSet, 0U); + Name(storage_buffer_mapping, "storage_buffer_mapping"); + //Starting with version 1.4... (https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html) + if (profile.supported_spirv >= 0x00010400) { + interfaces.push_back(storage_buffer_mapping); + } + + u32 mapping_base{}; + for (u32 index = 0; index < info.storage_buffers_descriptors.size(); ++index) { + const StorageBufferDescriptor& desc = info.storage_buffers_descriptors[index]; + storage_buffer_mapping_bases[index] = mapping_base; + storage_buffer_mapping_counts[index] = desc.count; + mapping_base += desc.count; + } + + const Id function_type{TypeFunction(U32[2], U32[1], U32[1], U32[1])}; + storage_buffer_map_func = OpFunction(U32[2], spv::FunctionControlMask::MaskNone, function_type); + const Id base{OpFunctionParameter(U32[1])}; + const Id count{OpFunctionParameter(U32[1])}; + const Id byte_offset{OpFunctionParameter(U32[1])}; + const Id index_pointer_type{TypePointer(spv::StorageClass::Function, U32[1])}; + const Id loop_header{OpLabel()}; + const Id continue_block{OpLabel()}; + const Id merge_block{OpLabel()}; + AddLabel(); + const Id segment_var{AddLocalVariable(index_pointer_type, spv::StorageClass::Function)}; + const Id offset_var{AddLocalVariable(index_pointer_type, spv::StorageClass::Function)}; + OpStore(segment_var, u32_zero_value); + OpStore(offset_var, byte_offset); + OpBranch(loop_header); + + AddLabel(loop_header); + const Id segment{OpLoad(U32[1], segment_var)}; + const Id local_offset{OpLoad(U32[1], offset_var)}; + const Id mapping_index{OpIAdd(U32[1], base, segment)}; + const Id size_pointer{OpAccessChain(storage_buffer_mapping_u32, storage_buffer_mapping, u32_zero_value, mapping_index)}; + const Id segment_size{OpLoad(U32[1], size_pointer)}; + const Id fits{OpULessThan(U1, local_offset, segment_size)}; + const Id last_segment{OpISub(U32[1], count, Const(1U))}; + const Id is_last{OpIEqual(U1, segment, last_segment)}; + const Id found{OpLogicalOr(U1, fits, is_last)}; + OpLoopMerge(merge_block, continue_block, spv::LoopControlMask::MaskNone); + OpBranchConditional(found, merge_block, continue_block); + + AddLabel(continue_block); + OpStore(offset_var, OpISub(U32[1], local_offset, segment_size)); + OpStore(segment_var, OpIAdd(U32[1], segment, Const(1U))); + OpBranch(loop_header); + + AddLabel(merge_block); + OpReturnValue(OpCompositeConstruct(U32[2], segment, local_offset)); + OpFunctionEnd(); + Name(storage_buffer_map_func, "map_storage_buffer"); +} + void EmitContext::DefineStorageBuffers(const Info& info, u32& binding) { if (info.storage_buffers_descriptors.empty()) { return; @@ -1271,9 +1375,7 @@ void EmitContext::DefineStorageBuffers(const Info& info, u32& binding) { DefineSsbos(*this, storage_types.U32x4, &StorageDefinitions::U32x4, info, binding, U32[4], sizeof(u32[4])); } - for (const StorageBufferDescriptor& desc : info.storage_buffers_descriptors) { - binding += desc.count; - } + binding += static_cast(info.storage_buffers_descriptors.size()); const bool needs_function{ info.uses_global_increment || info.uses_global_decrement || info.uses_atomic_f32_add || info.uses_atomic_f16x2_add || info.uses_atomic_f16x2_min || info.uses_atomic_f16x2_max || @@ -1282,40 +1384,31 @@ void EmitContext::DefineStorageBuffers(const Info& info, u32& binding) { AddCapability(spv::Capability::VariablePointersStorageBuffer); } if (info.uses_global_increment) { - increment_cas_ssbo = CasLoop(*this, Operation::Increment, storage_types.U32.array, - storage_types.U32.element, U32[1], U32[1], spv::Scope::Device); + increment_cas_ssbo = CasLoop(*this, Operation::Increment, storage_types.U32.element, U32[1], U32[1], spv::Scope::Device); } if (info.uses_global_decrement) { - decrement_cas_ssbo = CasLoop(*this, Operation::Decrement, storage_types.U32.array, - storage_types.U32.element, U32[1], U32[1], spv::Scope::Device); + decrement_cas_ssbo = CasLoop(*this, Operation::Decrement, storage_types.U32.element, U32[1], U32[1], spv::Scope::Device); } if (info.uses_atomic_f32_add) { - f32_add_cas = CasLoop(*this, Operation::FPAdd, storage_types.U32.array, - storage_types.U32.element, F32[1], U32[1], spv::Scope::Device); + f32_add_cas = CasLoop(*this, Operation::FPAdd, storage_types.U32.element, F32[1], U32[1], spv::Scope::Device); } if (info.uses_atomic_f16x2_add) { - f16x2_add_cas = CasLoop(*this, Operation::FPAdd, storage_types.U32.array, - storage_types.U32.element, F16[2], F16[2], spv::Scope::Device); + f16x2_add_cas = CasLoop(*this, Operation::FPAdd, storage_types.U32.element, F16[2], F16[2], spv::Scope::Device); } if (info.uses_atomic_f16x2_min) { - f16x2_min_cas = CasLoop(*this, Operation::FPMin, storage_types.U32.array, - storage_types.U32.element, F16[2], F16[2], spv::Scope::Device); + f16x2_min_cas = CasLoop(*this, Operation::FPMin, storage_types.U32.element, F16[2], F16[2], spv::Scope::Device); } if (info.uses_atomic_f16x2_max) { - f16x2_max_cas = CasLoop(*this, Operation::FPMax, storage_types.U32.array, - storage_types.U32.element, F16[2], F16[2], spv::Scope::Device); + f16x2_max_cas = CasLoop(*this, Operation::FPMax, storage_types.U32.element, F16[2], F16[2], spv::Scope::Device); } if (info.uses_atomic_f32x2_add) { - f32x2_add_cas = CasLoop(*this, Operation::FPAdd, storage_types.U32.array, - storage_types.U32.element, F32[2], F32[2], spv::Scope::Device); + f32x2_add_cas = CasLoop(*this, Operation::FPAdd, storage_types.U32.element, F32[2], F32[2], spv::Scope::Device); } if (info.uses_atomic_f32x2_min) { - f32x2_min_cas = CasLoop(*this, Operation::FPMin, storage_types.U32.array, - storage_types.U32.element, F32[2], F32[2], spv::Scope::Device); + f32x2_min_cas = CasLoop(*this, Operation::FPMin, storage_types.U32.element, F32[2], F32[2], spv::Scope::Device); } if (info.uses_atomic_f32x2_max) { - f32x2_max_cas = CasLoop(*this, Operation::FPMax, storage_types.U32.array, - storage_types.U32.element, F32[2], F32[2], spv::Scope::Device); + f32x2_max_cas = CasLoop(*this, Operation::FPMax, storage_types.U32.element, F32[2], F32[2], spv::Scope::Device); } } diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.h b/src/shader_recompiler/backend/spirv/spirv_emit_context.h index 7edc104708..cd874a9f55 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.h +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.h @@ -114,7 +114,6 @@ struct UniformDefinitions { }; struct StorageTypeDefinition { - Id array{}; Id element{}; }; @@ -173,6 +172,8 @@ public: [[nodiscard]] Id BitOffset8(const IR::Value& offset); [[nodiscard]] Id BitOffset16(const IR::Value& offset); + [[nodiscard]] Id StoragePointer(u32 binding, Id byte_offset, const StorageTypeDefinition& type_def, u32 element_size, Id StorageDefinitions::*member_ptr); + Id Const(u32 value) { return Constant(U32[1], value); } @@ -257,6 +258,11 @@ public: std::array cbufs{}; std::array ssbos{}; + std::array storage_buffer_mapping_bases{}; + std::array storage_buffer_mapping_counts{}; + Id storage_buffer_mapping{}; + Id storage_buffer_mapping_u32{}; + Id storage_buffer_map_func{}; std::vector texture_buffers; std::vector image_buffers; std::vector textures; @@ -376,6 +382,7 @@ public: bool uses_nonuniform_storage_image{}; bool uses_nonuniform_uniform_texel_buffer{}; bool uses_nonuniform_storage_texel_buffer{}; + bool uses_nonuniform_storage_buffer{}; private: void DefineCommonTypes(const Info& info); @@ -386,6 +393,7 @@ private: void DefineSharedMemoryFunctions(const IR::Program& program); void DefineConstantBuffers(const Info& info, u32& binding); void DefineConstantBufferIndirectFunctions(const Info& info); + void DefineStorageBufferMappings(const Info& info, u32& binding); void DefineStorageBuffers(const Info& info, u32& binding); void DefineTextureBuffers(const Info& info, u32& binding); void DefineImageBuffers(const Info& info, u32& binding); diff --git a/src/shader_recompiler/frontend/maxwell/translate_program.cpp b/src/shader_recompiler/frontend/maxwell/translate_program.cpp index ebc5a825dd..412f21b8e1 100644 --- a/src/shader_recompiler/frontend/maxwell/translate_program.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate_program.cpp @@ -132,6 +132,14 @@ void AddNVNStorageBuffers(IR::Program& program) { } } +void ConfigureStorageBufferMappings(IR::Program& program, const HostTranslateInfo& host_info) { + // https://docs.vulkan.org/guide/latest/descriptor_arrays.html + // will be needed: descriptor array elements represent physical spans of one guest virtual buffer. + for (StorageBufferDescriptor& desc : program.info.storage_buffers_descriptors) { + desc.count = host_info.storage_buffer_segment_count; + } +} + using IR::IsLegacyAttribute; //rescoped to attribute.h to make it visible in load_store_attribute.cpp IPA std::map GenerateLegacyToGenericMappings( @@ -299,6 +307,7 @@ IR::Program TranslateProgram(ObjectPool& inst_pool, ObjectPool& inst_pool, ObjectPool #include "common/common_types.h" namespace Shader { @@ -19,6 +20,7 @@ struct HostTranslateInfo { u64 min_ssbo_alignment{}; ///< Minimum alignment supported by the device for SSBOs u32 max_per_stage_descriptor_sampled_images{}; ///< maximum sampled descriptors per stage + u32 max_per_stage_descriptor_storage_buffers{}; ///< maximum storage descriptors per stage u32 max_per_stage_resources{}; ///< maximum resources per stage u32 max_descriptor_set_samplers{}; u32 max_descriptor_set_uniform_buffers{}; @@ -38,12 +40,14 @@ struct HostTranslateInfo { ///< passthrough shaders bool support_conditional_barrier{}; ///< True when the device supports barriers in conditional ///< control flow + u32 storage_buffer_segment_count{1}; ///< Physical ranges available to each guest SSBO void ApplyDescriptorLimitPolicy() noexcept { if (min_ssbo_alignment == 0) { min_ssbo_alignment = 1; } ApplyDescriptorLimitFallback(max_per_stage_descriptor_sampled_images); + ApplyDescriptorLimitFallback(max_per_stage_descriptor_storage_buffers); ApplyDescriptorLimitFallback(max_per_stage_resources); ApplyDescriptorLimitFallback(max_descriptor_set_samplers); ApplyDescriptorLimitFallback(max_descriptor_set_uniform_buffers); @@ -53,6 +57,7 @@ struct HostTranslateInfo { ApplyDescriptorLimitFallback(max_descriptor_set_sampled_images); ApplyDescriptorLimitFallback(max_descriptor_set_storage_images); ApplyDescriptorLimitFallback(max_descriptor_set_input_attachements); + storage_buffer_segment_count = (std::max)(storage_buffer_segment_count, 1U); } private: diff --git a/src/shader_recompiler/profile.h b/src/shader_recompiler/profile.h index d2ac480ab8..61449ba401 100644 --- a/src/shader_recompiler/profile.h +++ b/src/shader_recompiler/profile.h @@ -64,6 +64,7 @@ struct Profile { bool support_storage_image_array_nonuniform_indexing{}; bool support_uniform_texel_buffer_array_nonuniform_indexing{}; bool support_storage_texel_buffer_array_nonuniform_indexing{}; + bool support_storage_buffer_array_nonuniform_indexing{}; bool warp_size_potentially_larger_than_guest{}; diff --git a/src/shader_recompiler/shader_info.h b/src/shader_recompiler/shader_info.h index 71309e040e..64411832f4 100644 --- a/src/shader_recompiler/shader_info.h +++ b/src/shader_recompiler/shader_info.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -340,6 +341,10 @@ struct Info { ImageDescriptors image_descriptors; }; +[[nodiscard]] inline bool UsesStorageBufferMappings(const Info& info) noexcept { + return std::ranges::any_of(info.storage_buffers_descriptors, [](const auto& desc) { return desc.count > 1; }); +} + template u32 NumDescriptors(const Descriptors& descriptors) { u32 num{}; diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e40aad1fb5..517ed3a58f 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -7,6 +7,8 @@ #pragma once #include +#include +#include #include #include @@ -423,7 +425,7 @@ void BufferCache

::UnbindGraphicsStorageBuffers(size_t stage) { template bool BufferCache

::BindGraphicsStorageBuffer(size_t stage, size_t ssbo_index, u32 cbuf_index, - u32 cbuf_offset, bool is_written) { + u32 cbuf_offset, bool is_written, u32 descriptor_count) { const bool already_enabled = ((channel_state->enabled_storage_buffers[stage] >> ssbo_index) & 1U) != 0; if constexpr (requires { runtime.ShouldLimitDynamicStorageBuffers(); }) { @@ -448,8 +450,8 @@ bool BufferCache

::BindGraphicsStorageBuffer(size_t stage, size_t ssbo_index, const auto& cbufs = maxwell3d->state.shader_stages[stage]; const GPUVAddr ssbo_addr = cbufs.const_buffers[cbuf_index].address + cbuf_offset; channel_state->storage_buffers[stage][ssbo_index] = - StorageBufferBinding(ssbo_addr, cbuf_index, is_written); - return (channel_state->storage_buffers[stage][ssbo_index].buffer_id != NULL_BUFFER_ID); + StorageBufferBinding(ssbo_addr, cbuf_index, is_written, descriptor_count); + return channel_state->storage_buffers[stage][ssbo_index].gpu_addr != 0; } template @@ -487,7 +489,7 @@ void BufferCache

::UnbindComputeStorageBuffers() { template void BufferCache

::BindComputeStorageBuffer(size_t ssbo_index, u32 cbuf_index, u32 cbuf_offset, - bool is_written) { + bool is_written, u32 descriptor_count) { if (ssbo_index >= channel_state->compute_storage_buffers.size()) [[unlikely]] { LOG_ERROR(HW_GPU, "Storage buffer index {} exceeds maximum storage buffer count", ssbo_index); @@ -524,7 +526,7 @@ void BufferCache

::BindComputeStorageBuffer(size_t ssbo_index, u32 cbuf_index, const auto& cbufs = launch_desc.const_buffer_config; const GPUVAddr ssbo_addr = cbufs[cbuf_index].Address() + cbuf_offset; channel_state->compute_storage_buffers[ssbo_index] = - StorageBufferBinding(ssbo_addr, cbuf_index, is_written); + StorageBufferBinding(ssbo_addr, cbuf_index, is_written, descriptor_count); } template @@ -1000,27 +1002,52 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 template void BufferCache

::BindHostGraphicsStorageBuffers(size_t stage) { + boost::container::small_vector segment_sizes; + bool uses_mapping{}; + ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { + const StorageBufferBindingInfo& binding = channel_state->storage_buffers[stage][index]; + uses_mapping |= binding.descriptor_count > 1; + for (u32 segment = 0; segment < binding.descriptor_count; ++segment) { + segment_sizes.push_back(segment < binding.segments.size() ? binding.segments[segment].size : 0); + } + }); + if (uses_mapping) { + const u32 mapping_size = static_cast(segment_sizes.size() * sizeof(u32)); + if constexpr (!IS_OPENGL) { + const std::span mapped = runtime.BindMappedStorageBuffer(mapping_size); + std::memcpy(mapped.data(), segment_sizes.data(), mapping_size); + } + } + u32 binding_index = 0; ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { - const Binding& binding = channel_state->storage_buffers[stage][index]; - Buffer& buffer = slot_buffers[binding.buffer_id]; - TouchBuffer(buffer, binding.buffer_id); - const u32 size = binding.size; - SynchronizeBuffer(buffer, binding.device_addr, size); - - const u32 offset = buffer.Offset(binding.device_addr); - buffer.MarkUsage(offset, size); + const StorageBufferBindingInfo& storage = channel_state->storage_buffers[stage][index]; const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; - - if (is_written) { - MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); - } - - if constexpr (NEEDS_BIND_STORAGE_INDEX) { - runtime.BindStorageBuffer(stage, binding_index, buffer, offset, size, is_written); - ++binding_index; - } else { - runtime.BindStorageBuffer(buffer, offset, size, is_written); + for (u32 segment = 0; segment < storage.descriptor_count; ++segment) { + Buffer* buffer = &slot_buffers[NULL_BUFFER_ID]; + u32 offset{}; + u32 size{IS_OPENGL ? 0U : static_cast(sizeof(u32))}; + const bool is_actual_segment = segment < storage.segments.size(); + // shall be safe enough if the segment is not actual, use the last available segment or nullptr if none exist. + const Binding* binding = is_actual_segment ? &storage.segments[segment] : storage.segments.empty() ? nullptr : &storage.segments.back(); + if (binding) { + buffer = &slot_buffers[binding->buffer_id]; + size = binding->size; + offset = buffer->Offset(binding->device_addr); + if (is_actual_segment) { + TouchBuffer(*buffer, binding->buffer_id); + SynchronizeBuffer(*buffer, binding->device_addr, size); + buffer->MarkUsage(offset, size); + if (is_written) { + MarkWrittenBuffer(binding->buffer_id, binding->device_addr, size); + } + } + } + if constexpr (NEEDS_BIND_STORAGE_INDEX) { + runtime.BindStorageBuffer(stage, binding_index++, *buffer, offset, size, is_written); + } else { + runtime.BindStorageBuffer(*buffer, offset, size, is_written); + } } }); } @@ -1136,28 +1163,53 @@ void BufferCache

::BindHostComputeUniformBuffers() { template void BufferCache

::BindHostComputeStorageBuffers() { + boost::container::small_vector segment_sizes; + bool uses_mapping{}; + ForEachEnabledBit(channel_state->enabled_compute_storage_buffers, [&](u32 index) { + const StorageBufferBindingInfo& binding = channel_state->compute_storage_buffers[index]; + uses_mapping |= binding.descriptor_count > 1; + for (u32 segment = 0; segment < binding.descriptor_count; ++segment) { + segment_sizes.push_back(segment < binding.segments.size() ? binding.segments[segment].size : 0); + } + }); + if (uses_mapping) { + const u32 mapping_size = static_cast(segment_sizes.size() * sizeof(u32)); + if constexpr (!IS_OPENGL) { + const std::span mapped = runtime.BindMappedStorageBuffer(mapping_size); + std::memcpy(mapped.data(), segment_sizes.data(), mapping_size); + } + } + u32 binding_index = 0; ForEachEnabledBit(channel_state->enabled_compute_storage_buffers, [&](u32 index) { - const Binding& binding = channel_state->compute_storage_buffers[index]; - Buffer& buffer = slot_buffers[binding.buffer_id]; - TouchBuffer(buffer, binding.buffer_id); - const u32 size = binding.size; - SynchronizeBuffer(buffer, binding.device_addr, size); - - const u32 offset = buffer.Offset(binding.device_addr); - buffer.MarkUsage(offset, size); + const StorageBufferBindingInfo& storage = channel_state->compute_storage_buffers[index]; const bool is_written = ((channel_state->written_compute_storage_buffers >> index) & 1) != 0; - - if (is_written) { - MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); - } - - if constexpr (NEEDS_BIND_STORAGE_INDEX) { - runtime.BindComputeStorageBuffer(binding_index, buffer, offset, size, is_written); - ++binding_index; - } else { - runtime.BindStorageBuffer(buffer, offset, size, is_written); + for (u32 segment = 0; segment < storage.descriptor_count; ++segment) { + Buffer* buffer = &slot_buffers[NULL_BUFFER_ID]; + u32 offset{}; + u32 size{IS_OPENGL ? 0U : static_cast(sizeof(u32))}; + const bool is_actual_segment = segment < storage.segments.size(); + //same fallback logic + const Binding* binding = is_actual_segment ? &storage.segments[segment] : storage.segments.empty() ? nullptr : &storage.segments.back(); + if (binding) { + buffer = &slot_buffers[binding->buffer_id]; + size = binding->size; + offset = buffer->Offset(binding->device_addr); + if (is_actual_segment) { + TouchBuffer(*buffer, binding->buffer_id); + SynchronizeBuffer(*buffer, binding->device_addr, size); + buffer->MarkUsage(offset, size); + if (is_written) { + MarkWrittenBuffer(binding->buffer_id, binding->device_addr, size); + } + } + } + if constexpr (NEEDS_BIND_STORAGE_INDEX) { + runtime.BindComputeStorageBuffer(binding_index++, *buffer, offset, size, is_written); + } else { + runtime.BindStorageBuffer(*buffer, offset, size, is_written); + } } }); } @@ -1350,10 +1402,7 @@ void BufferCache

::UpdateUniformBuffers(size_t stage) { template void BufferCache

::UpdateStorageBuffers(size_t stage) { ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { - // Resolve buffer - Binding& binding = channel_state->storage_buffers[stage][index]; - const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size); - binding.buffer_id = buffer_id; + UpdateStorageBuffer(channel_state->storage_buffers[stage][index]); }); } @@ -1414,12 +1463,54 @@ void BufferCache

::UpdateComputeUniformBuffers() { template void BufferCache

::UpdateComputeStorageBuffers() { ForEachEnabledBit(channel_state->enabled_compute_storage_buffers, [&](u32 index) { - // Resolve buffer - Binding& binding = channel_state->compute_storage_buffers[index]; - binding.buffer_id = FindBuffer(binding.device_addr, binding.size); + UpdateStorageBuffer(channel_state->compute_storage_buffers[index]); }); } +template +void BufferCache

::UpdateStorageBuffer(StorageBufferBindingInfo& binding) { + binding.segments.clear(); + if (binding.gpu_addr == 0 || binding.size == 0) { return;} + if (binding.descriptor_count == 1) { + // for safety gotta preserve the legacy path on possible hosts without storage-buffer descriptor indexing. + const std::optional device_addr = gpu_memory->GpuToCpuAddress(binding.gpu_addr); + if (device_addr) { + binding.segments.push_back(Binding{ + .device_addr = *device_addr, + .size = binding.size, + .buffer_id = FindBuffer(*device_addr, binding.size), + }); + } + return; + } + + const auto ranges = gpu_memory->GetSubmappedRange(binding.gpu_addr, binding.size); + const size_t mapped_size = + std::accumulate(ranges.begin(), ranges.end(), size_t{}, [](size_t total, const auto& range) { return total + range.second; }); + if (mapped_size != binding.size) { + LOG_ERROR(HW_GPU, "Storage buffer range {:#x}+{:#x} is not fully mapped", binding.gpu_addr, binding.size); + return; + } + if (ranges.size() > binding.descriptor_count) { + LOG_ERROR(HW_GPU, "Storage buffer range {:#x}+{:#x} has {} physical segments, exceeding host capacity {}", + binding.gpu_addr, binding.size, ranges.size(), binding.descriptor_count); + return; + } + for (const auto& [gpu_addr, size] : ranges) { + const std::optional device_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + if (!device_addr || size > (std::numeric_limits::max)()) { + binding.segments.clear(); + return; + } + const u32 segment_size = static_cast(size); + binding.segments.push_back(Binding{ + .device_addr = *device_addr, + .size = segment_size, + .buffer_id = FindBuffer(*device_addr, segment_size), + }); + } +} + template void BufferCache

::UpdateComputeTextureBuffers() { ForEachEnabledBit(channel_state->enabled_compute_texture_buffers, [&](u32 index) { @@ -1841,6 +1932,11 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { const auto replace = [scalar_replace](std::span bindings) { std::ranges::for_each(bindings, scalar_replace); }; + const auto storage_replace = [scalar_replace](std::span bindings) { + for (StorageBufferBindingInfo& binding : bindings) { + std::ranges::for_each(binding.segments, scalar_replace); + } + }; if (channel_state->index_buffer.buffer_id == buffer_id) { channel_state->index_buffer.buffer_id = BufferId{}; @@ -1856,10 +1952,10 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { } } std::ranges::for_each(channel_state->uniform_buffers, replace); - std::ranges::for_each(channel_state->storage_buffers, replace); + std::ranges::for_each(channel_state->storage_buffers, storage_replace); replace(channel_state->transform_feedback_buffers); replace(channel_state->compute_uniform_buffers); - replace(channel_state->compute_storage_buffers); + storage_replace(channel_state->compute_storage_buffers); // Mark the whole buffer as CPU written to stop tracking CPU writes if (!do_not_mark) { @@ -1896,12 +1992,14 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { } template -Binding BufferCache

::StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, - bool is_written) const { +StorageBufferBindingInfo BufferCache

::StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, + bool is_written, u32 descriptor_count) const { + // time to get rid of these null bindings + ASSERT(descriptor_count > 0); // shant happen const GPUVAddr gpu_addr = gpu_memory->Read(ssbo_addr); if (gpu_addr == 0) { - return NULL_BINDING; + return {.descriptor_count = descriptor_count}; } const auto size = [&]() { @@ -1923,21 +2021,17 @@ Binding BufferCache

::StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, const GPUVAddr aligned_gpu_addr = Common::AlignDown(gpu_addr, alignment); const u32 aligned_size = static_cast(gpu_addr - aligned_gpu_addr) + size; - const std::optional aligned_device_addr = gpu_memory->GpuToCpuAddress(aligned_gpu_addr); - if (!aligned_device_addr || size == 0) { + if (!gpu_memory->GpuToCpuAddress(aligned_gpu_addr) || size == 0) { LOG_DEBUG(HW_GPU, "Failed to find storage buffer for cbuf index {}", cbuf_index); - return NULL_BINDING; + return {.descriptor_count = descriptor_count}; } - const std::optional device_addr = gpu_memory->GpuToCpuAddress(gpu_addr); - ASSERT_MSG(device_addr, "Unaligned storage buffer address not found for cbuf index {}", - cbuf_index); // The end address used for size calculation does not need to be aligned - const DAddr cpu_end = Common::AlignUp(*device_addr + size, Core::DEVICE_PAGESIZE); + const GPUVAddr gpu_end = Common::AlignUp(gpu_addr + size, Core::DEVICE_PAGESIZE); - const Binding binding{ - .device_addr = *aligned_device_addr, - .size = is_written ? aligned_size : static_cast(cpu_end - *aligned_device_addr), - .buffer_id = BufferId{}, + const StorageBufferBindingInfo binding{ + .gpu_addr = aligned_gpu_addr, + .size = is_written ? aligned_size : static_cast(gpu_end - aligned_gpu_addr), + .descriptor_count = descriptor_count, }; return binding; } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 0a521192c6..62bfc2f1ab 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -89,6 +89,15 @@ struct TextureBufferBinding : Binding { PixelFormat format; }; +struct StorageBufferBindingInfo { + // another good one: guest SSBO is a virtual interval and may span discontiguous device-memory ranges. + // exact case of missing character frames (high sample lane) + GPUVAddr gpu_addr{}; + u32 size{}; + u32 descriptor_count{1}; + boost::container::small_vector segments; +}; + static constexpr Binding NULL_BINDING{ .device_addr = 0, .size = 0, @@ -115,14 +124,15 @@ public: Binding index_buffer; std::array vertex_buffers; std::array, NUM_STAGES> uniform_buffers; - std::array, NUM_STAGES> storage_buffers; + std::array, NUM_STAGES> + storage_buffers; std::array, NUM_STAGES> texture_buffers; std::array transform_feedback_buffers; Binding count_buffer_binding; Binding indirect_buffer_binding; std::array compute_uniform_buffers; - std::array compute_storage_buffers; + std::array compute_storage_buffers; std::array compute_texture_buffers; std::array enabled_uniform_buffer_masks{}; @@ -249,7 +259,7 @@ public: void UnbindGraphicsStorageBuffers(size_t stage); bool BindGraphicsStorageBuffer(size_t stage, size_t ssbo_index, u32 cbuf_index, u32 cbuf_offset, - bool is_written); + bool is_written, u32 descriptor_count = 1); void UnbindGraphicsTextureBuffers(size_t stage); @@ -259,7 +269,7 @@ public: void UnbindComputeStorageBuffers(); void BindComputeStorageBuffer(size_t ssbo_index, u32 cbuf_index, u32 cbuf_offset, - bool is_written); + bool is_written, u32 descriptor_count = 1); void UnbindComputeTextureBuffers(); @@ -400,6 +410,8 @@ private: void UpdateStorageBuffers(size_t stage); + void UpdateStorageBuffer(StorageBufferBindingInfo& binding); + void UpdateTextureBuffers(size_t stage); void UpdateTransformFeedbackBuffers(); @@ -449,8 +461,9 @@ private: void DeleteBuffer(BufferId buffer_id, bool do_not_mark = false); - [[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, - bool is_written) const; + [[nodiscard]] StorageBufferBindingInfo StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, + bool is_written, + u32 descriptor_count) const; [[nodiscard]] TextureBufferBinding GetTextureBufferBinding(GPUVAddr gpu_addr, u32 size, PixelFormat format); diff --git a/src/video_core/renderer_vulkan/pipeline_helper.h b/src/video_core/renderer_vulkan/pipeline_helper.h index 07de25f18f..94e3b86f16 100644 --- a/src/video_core/renderer_vulkan/pipeline_helper.h +++ b/src/video_core/renderer_vulkan/pipeline_helper.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include @@ -136,6 +137,7 @@ inline void WriteDescriptorBuffer(const Device& device, const DescriptorBufferLa [[nodiscard]] inline u32 NumDescriptorEntries(const Shader::Info& info) { return Shader::NumDescriptors(info.constant_buffer_descriptors) + + static_cast(Shader::UsesStorageBufferMappings(info)) + Shader::NumDescriptors(info.storage_buffers_descriptors) + Shader::NumDescriptors(info.texture_buffer_descriptors) + Shader::NumDescriptors(info.image_buffer_descriptors) + @@ -268,6 +270,12 @@ public: is_compute |= (stage & VK_SHADER_STAGE_COMPUTE_BIT) != 0; Add(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, stage, info.constant_buffer_descriptors); + // for extra implicit storage-buffer binding required by mapped-storage-buffer support + if (Shader::UsesStorageBufferMappings(info)) { + struct Descriptor { u32 count; }; + const std::array descriptors{Descriptor{1}}; + Add(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, stage, descriptors); + } Add(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, stage, info.storage_buffers_descriptors); Add(VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER, stage, info.texture_buffer_descriptors); Add(VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, stage, info.image_buffer_descriptors); diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index a734bd049c..e308ec46ec 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -695,7 +695,7 @@ vk::Buffer BufferCacheRuntime::CreateNullBuffer() { .flags = 0, .size = 4, .usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE, .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 9b3dd50eaa..152d16c661 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -155,6 +155,13 @@ public: return ref.mapped_span; } + // compute/graphics new binding + std::span BindMappedStorageBuffer(u32 size) { + const StagingBufferRef ref = staging_pool.Request(size, MemoryUsage::Upload); + guest_descriptor_queue.AddBuffer(ref.buffer, ref.device_address, static_cast(ref.offset), size); + return ref.mapped_span; + } + void BindUniformBuffer(const Buffer& buffer, u32 offset, u32 size) { BindBuffer(buffer, offset, size); } diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp index 29356cfcec..853b88138a 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp @@ -157,9 +157,7 @@ bool ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, buffer_cache.UnbindComputeStorageBuffers(); size_t ssbo_index{}; for (const auto& desc : info.storage_buffers_descriptors) { - ASSERT(desc.count == 1); - buffer_cache.BindComputeStorageBuffer(ssbo_index, desc.cbuf_index, desc.cbuf_offset, - desc.is_written); + buffer_cache.BindComputeStorageBuffer(ssbo_index, desc.cbuf_index, desc.cbuf_offset, desc.is_written, desc.count); ++ssbo_index; } diff --git a/src/video_core/renderer_vulkan/vk_descriptor_pool.cpp b/src/video_core/renderer_vulkan/vk_descriptor_pool.cpp index b020d99b93..3084988f2d 100644 --- a/src/video_core/renderer_vulkan/vk_descriptor_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_descriptor_pool.cpp @@ -47,7 +47,7 @@ static DescriptorBankInfo MakeBankInfo(std::span infos) { DescriptorBankInfo bank; for (const Shader::Info& info : infos) { bank.uniform_buffers += Accumulate(info.constant_buffer_descriptors); - bank.storage_buffers += Accumulate(info.storage_buffers_descriptors); + bank.storage_buffers += Accumulate(info.storage_buffers_descriptors) + static_cast(Shader::UsesStorageBufferMappings(info)); bank.texture_buffers += Accumulate(info.texture_buffer_descriptors); bank.image_buffers += Accumulate(info.image_buffer_descriptors); bank.textures += Accumulate(info.texture_descriptors); diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 8b5b0bc9c5..6d11d9a444 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -367,9 +367,8 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { if constexpr (Spec::has_storage_buffers) { size_t ssbo_index{}; for (const auto& desc : info.storage_buffers_descriptors) { - ASSERT(desc.count == 1); buffer_cache.BindGraphicsStorageBuffer(stage, ssbo_index, desc.cbuf_index, - desc.cbuf_offset, desc.is_written); + desc.cbuf_offset, desc.is_written, desc.count); ++ssbo_index; } } diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index 642b9fa223..65aa1ee258 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -62,7 +62,12 @@ using VideoCommon::FileEnvironment; using VideoCommon::GenericEnvironment; using VideoCommon::GraphicsEnvironment; -constexpr u32 CACHE_VERSION = 18; +// SPIR-V descriptor arrays require a fixed pipeline-layout count. +// Exploration ceiling; buffer-cache telemetry records the actual physical-range demand. +// Keep this modest because every mapped SSBO binds the full fixed array on each update. +constexpr u32 MAX_MAPPED_STORAGE_BUFFER_DESCRIPTORS = 32; + +constexpr u32 CACHE_VERSION = 19; constexpr size_t VULKAN_CACHE_FLUSH_PIPELINES = 128; constexpr size_t VULKAN_CACHE_FLUSH_MIN_SECONDS = 30; constexpr std::array VULKAN_CACHE_MAGIC_NUMBER{'y', 'u', 'z', 'u', 'v', 'k', 'c', 'h'}; @@ -432,6 +437,8 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, device.IsUniformTexelBufferArrayNonUniformIndexingSupported(), .support_storage_texel_buffer_array_nonuniform_indexing = device.IsStorageTexelBufferArrayNonUniformIndexingSupported(), + .support_storage_buffer_array_nonuniform_indexing = + device.IsStorageBufferArrayNonUniformIndexingSupported(), .warp_size_potentially_larger_than_guest = device.IsWarpSizePotentiallyBiggerThanGuest(), @@ -460,6 +467,7 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, host_info = Shader::HostTranslateInfo{ .min_ssbo_alignment = device.GetStorageBufferAlignment(), .max_per_stage_descriptor_sampled_images = device.GetMaxPerStageDescriptorSampledImages(), + .max_per_stage_descriptor_storage_buffers = device.GetMaxPerStageDescriptorStorageBuffers(), .max_per_stage_resources = device.GetMaxPerStageResources(), .max_descriptor_set_samplers = device.GetMaxDescriptorSetSamplers(), .max_descriptor_set_uniform_buffers = device.GetMaxDescriptorSetUniformBuffers(), @@ -479,6 +487,20 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, .support_viewport_index_layer = device.IsExtShaderViewportIndexLayerSupported(), .support_geometry_shader_passthrough = device.IsNvGeometryShaderPassthroughSupported(), .support_conditional_barrier = device.SupportsConditionalBarriers(), + .storage_buffer_segment_count = [&] { + if (!device.IsStorageBufferArrayNonUniformIndexingSupported()) { + return 1U; + } + constexpr u32 MaxGraphicsStages = static_cast(Maxwell::MaxShaderStage); + const auto reserve = [](u32 limit, u32 count) { + return limit > count ? limit - count : 0U; + }; + const u32 per_stage = reserve(device.GetMaxPerStageDescriptorStorageBuffers(), 1) / static_cast(Shader::Info::MAX_SSBOS); + const u32 per_set = reserve(device.GetMaxDescriptorSetStorageBuffers(), MaxGraphicsStages) / (static_cast(Shader::Info::MAX_SSBOS) * MaxGraphicsStages); + const u32 resources = reserve(device.GetMaxPerStageResources(), 1) / (static_cast(Shader::Info::MAX_SSBOS) * 2); + // ensure at least one storage buffer segment is available per stage. max is still arbitrary + return (std::max)(1U, (std::min)({MAX_MAPPED_STORAGE_BUFFER_DESCRIPTORS, per_stage, per_set, resources})); + }(), }; host_info.ApplyDescriptorLimitPolicy(); diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index fb3a9c4e7a..d7f484fbc7 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -708,7 +708,6 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR descriptor_indexing.shaderUniformTexelBufferArrayDynamicIndexing = false; descriptor_indexing.shaderStorageTexelBufferArrayDynamicIndexing = false; descriptor_indexing.shaderUniformBufferArrayNonUniformIndexing = false; - descriptor_indexing.shaderStorageBufferArrayNonUniformIndexing = false; descriptor_indexing.shaderInputAttachmentArrayNonUniformIndexing = false; descriptor_indexing.descriptorBindingUniformBufferUpdateAfterBind = false; descriptor_indexing.descriptorBindingSampledImageUpdateAfterBind = false; diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index c53bd6443e..2bb1ca4141 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -360,6 +360,7 @@ public: #define FN_MAX_LIMIT_LIST \ FN_MAX_LIMIT_ELEM(ComputeSharedMemorySize) \ FN_MAX_LIMIT_ELEM(PerStageDescriptorSampledImages) \ + FN_MAX_LIMIT_ELEM(PerStageDescriptorStorageBuffers) \ FN_MAX_LIMIT_ELEM(PerStageResources) \ FN_MAX_LIMIT_ELEM(DescriptorSetSamplers) \ FN_MAX_LIMIT_ELEM(DescriptorSetUniformBuffers) \ @@ -415,6 +416,10 @@ FN_MAX_LIMIT_LIST return features.descriptor_indexing.shaderStorageTexelBufferArrayNonUniformIndexing; } + bool IsStorageBufferArrayNonUniformIndexingSupported() const { + return features.descriptor_indexing.shaderStorageBufferArrayNonUniformIndexing; + } + /// Returns true if the device supports float64 natively. bool IsFloat64Supported() const { return features.features.shaderFloat64;