From 08917afe0e29b3bb7ddeb3b405b76832841cc96d Mon Sep 17 00:00:00 2001 From: CamilleLaVey Date: Tue, 29 Sep 2026 21:57:46 -0400 Subject: [PATCH] God has forsaken me to BDA --- .../backend/glasm/emit_glasm_memory.cpp | 8 +- .../backend/glsl/glsl_emit_context.cpp | 3 +- .../backend/spirv/emit_spirv.h | 5 + .../backend/spirv/spirv_emit_context.cpp | 183 ++++++++++++++---- .../backend/spirv/spirv_emit_context.h | 2 + .../global_memory_to_storage_buffer_pass.cpp | 18 +- src/shader_recompiler/profile.h | 1 + src/shader_recompiler/shader_info.h | 1 + src/video_core/buffer_cache/buffer_cache.h | 72 +++++++ .../buffer_cache/buffer_cache_base.h | 19 ++ .../renderer_vulkan/pipeline_helper.h | 8 +- .../renderer_vulkan/vk_compute_pipeline.cpp | 14 +- .../renderer_vulkan/vk_graphics_pipeline.cpp | 15 +- .../renderer_vulkan/vk_graphics_pipeline.h | 3 + .../renderer_vulkan/vk_pipeline_cache.cpp | 1 + 15 files changed, 301 insertions(+), 52 deletions(-) diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp b/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp index ee39ec2a47..905c8d2bb8 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp +++ b/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp @@ -41,17 +41,17 @@ void GlobalStorageOp(EmitContext& ctx, Register address, bool pointer_based, std } const u64 ssbo_align_mask{~(ctx.profile.min_ssbo_alignment - 1U)}; ctx.Add("LDC.U64 DC.x,c{}[{}];" // unaligned_ssbo_addr - "AND.U64 DC.x,DC.x,{};" // ssbo_addr = unaligned_ssbo_addr & ssbo_align_mask "LDC.U32 RC.x,c{}[{}];" // ssbo_size_u32 "CVT.U64.U32 DC.y,RC.x;" // ssbo_size = ssbo_size_u32 - "ADD.U64 DC.y,DC.y,DC.x;" // ssbo_end = ssbo_addr + ssbo_size + "ADD.U64 DC.y,DC.y,DC.x;" // ssbo_end = unaligned_ssbo_addr + ssbo_size + "AND.U64 DC.x,DC.x,{};" // ssbo_addr = unaligned_ssbo_addr & ssbo_align_mask "SGE.U64 RC.x,{}.x,DC.x;" // a = input_addr >= ssbo_addr ? -1 : 0 "SLT.U64 RC.y,{}.x,DC.y;" // b = input_addr < ssbo_end ? -1 : 0 "AND.U.CC RC.x,RC.x,RC.y;" // cond = a && b "IF NE.x;" // if cond "SUB.U64 DC.x,{}.x,DC.x;", // offset = input_addr - ssbo_addr - ssbo.cbuf_index, ssbo.cbuf_offset, ssbo_align_mask, ssbo.cbuf_index, - ssbo.cbuf_offset + 8, address, address, address); + ssbo.cbuf_index, ssbo.cbuf_offset, ssbo.cbuf_index, ssbo.cbuf_offset + 8, + ssbo_align_mask, address, address, address); if (pointer_based) { ctx.Add("PK64.U DC.y,c[{}];" // host_ssbo = cbuf "ADD.U64 DC.x,DC.x,DC.y;" // host_addr = host_ssbo + offset diff --git a/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp b/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp index 4f4d420bd7..25887792d7 100644 --- a/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp +++ b/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp @@ -618,7 +618,8 @@ std::string EmitContext::DefineGlobalMemoryFunctions() { const auto size_vec{fmt::format("uvec2({},{})", size_xy[0], size_xy[1])}; const auto comp_lhs{fmt::format("(addr>={})", ssbo_addr)}; - const auto comp_rhs{fmt::format("(addr<({}+uint64_t({})))", ssbo_addr, size_vec)}; + const auto comp_rhs{fmt::format("(addr<(packUint2x32(uvec2({},{}))+uint64_t({})))", + addr_xy[0], addr_xy[1], size_vec)}; const auto comparison{fmt::format("if({}&&{}){{", comp_lhs, comp_rhs)}; func += comparison; diff --git a/src/shader_recompiler/backend/spirv/emit_spirv.h b/src/shader_recompiler/backend/spirv/emit_spirv.h index c88f07c9ce..53e4e7dbb9 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv.h +++ b/src/shader_recompiler/backend/spirv/emit_spirv.h @@ -29,6 +29,11 @@ struct RescalingLayout { struct RenderAreaLayout { std::array render_area; }; +struct GlobalPointerLayout { + std::array table; + u32 count; +}; +constexpr u32 GLOBAL_POINTER_LAYOUT_OFFSET = 64; constexpr u32 RESCALING_LAYOUT_WORDS_OFFSET = offsetof(RescalingLayout, rescaling_textures); constexpr u32 RESCALING_LAYOUT_DOWN_FACTOR_OFFSET = offsetof(RescalingLayout, down_factor); constexpr u32 RENDERAREA_LAYOUT_OFFSET = offsetof(RenderAreaLayout, render_area); diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp index f174251625..6906f42d09 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp @@ -486,9 +486,9 @@ EmitContext::EmitContext(const Profile& profile_, const RuntimeInfo& runtime_inf DefineTextures(program.info, texture_binding, bindings.texture_scaling_index); DefineImages(program.info, image_binding, bindings.image_scaling_index); DefineAttributeMemAccess(program.info); - DefineGlobalMemoryFunctions(program); DefineRescalingInput(program.info); DefineRenderArea(program.info); + DefineGlobalMemoryFunctions(program); } EmitContext::~EmitContext() = default; @@ -911,12 +911,48 @@ void EmitContext::DefineGlobalMemoryFunctions(const IR::Program& program) { if (!info.uses_global_memory || !profile.support_int64) { return; } + struct Access { + const StorageDefinitions* ssbo; + Id word; + Id address; + }; const Id zero{u32_zero_value}; const Id scope{Const(static_cast(spv::Scope::Device))}; const Id align_mask{Const(~(static_cast(profile.min_ssbo_alignment) - 1U))}; - const auto word_pointer{[&](Id ssbo, Id word, u32 element) { - return OpAccessChain(storage_types.U32.element, ssbo, zero, - OpIAdd(U32[1], word, Const(element))); + Id physical_u32{}; + Id physical_u32x4{}; + if (uses_global_pointers) { + AddCapability(spv::Capability::PhysicalStorageBufferAddresses); + AddExtension("SPV_KHR_physical_storage_buffer"); + SetMemoryModel(spv::AddressingModel::PhysicalStorageBuffer64, spv::MemoryModel::GLSL450); + physical_u32 = TypePointer(spv::StorageClass::PhysicalStorageBuffer, U32[1]); + physical_u32x4 = TypePointer(spv::StorageClass::PhysicalStorageBuffer, U32[4]); + } + const auto word_pointer{[&](const Access& access, u32 element) { + if (access.ssbo) { + return OpAccessChain(storage_types.U32.element, access.ssbo->U32, zero, + OpIAdd(U32[1], access.word, Const(element))); + } + return OpConvertUToPtr(physical_u32, + OpIAdd(U64, access.address, Constant(U64, u64{element} * 4))); + }}; + const auto load_word{[&](const Access& access, u32 element) { + if (access.ssbo) { + return OpLoad(U32[1], word_pointer(access, element)); + } + return OpLoad(U32[1], word_pointer(access, element), spv::MemoryAccessMask::Aligned, 4U); + }}; + const auto store_word{[&](const Access& access, u32 element, Id value) { + if (access.ssbo) { + OpStore(word_pointer(access, element), value); + return; + } + OpStore(word_pointer(access, element), value, spv::MemoryAccessMask::Aligned, 4U); + }}; + const auto load_entry{[&](Id address) { + const Id entry{OpLoad(U32[4], OpConvertUToPtr(physical_u32x4, address), + spv::MemoryAccessMask::Aligned, 16U)}; + return std::pair{entry, OpBitcast(U64, OpVectorShuffle(U32[2], entry, entry, 0U, 1U))}; }}; const auto cbuf_word{[&](u32 index, u32 offset) { if (profile.support_descriptor_aliasing) { @@ -938,7 +974,19 @@ void EmitContext::DefineGlobalMemoryFunctions(const IR::Program& program) { const Id addr{OpFunctionParameter(U64)}; const Id value{OpFunctionParameter(value_type)}; const bool returns_value{result_type.value != void_id.value}; + const auto finish{[&](Id result) { + if (returns_value) { + OpReturnValue(result); + } else { + OpReturn(); + } + }}; AddLabel(); + Id entry_index{}; + if (uses_global_pointers) { + entry_index = AddLocalVariable(TypePointer(spv::StorageClass::Function, U32[1]), + spv::StorageClass::Function, zero); + } const Id addr_words{OpBitcast(U32[2], addr)}; const Id addr_low{OpCompositeExtract(U32[1], addr_words, 0U)}; const Id addr_high{OpCompositeExtract(U32[1], addr_words, 1U)}; @@ -947,8 +995,7 @@ void EmitContext::DefineGlobalMemoryFunctions(const IR::Program& program) { if (!desc.is_global_fallback) { continue; } - const Id ssbo_low{ - OpBitwiseAnd(U32[1], cbuf_word(desc.cbuf_index, desc.cbuf_offset), align_mask)}; + const Id ssbo_low{cbuf_word(desc.cbuf_index, desc.cbuf_offset)}; const Id ssbo_high{cbuf_word(desc.cbuf_index, desc.cbuf_offset + 4)}; const Id ssbo_size{cbuf_word(desc.cbuf_index, desc.cbuf_offset + 8)}; const Id offset{OpISub(U32[1], addr_low, ssbo_low)}; @@ -961,15 +1008,55 @@ void EmitContext::DefineGlobalMemoryFunctions(const IR::Program& program) { OpSelectionMerge(else_label, spv::SelectionControlMask::MaskNone); OpBranchConditional(cond, then_label, else_label); AddLabel(then_label); - const Id word{OpShiftRightLogical(U32[1], offset, Const(2U))}; - const Id result{callback(ssbos[index], word, offset, value)}; - if (returns_value) { - OpReturnValue(result); - } else { - OpReturn(); - } + const Id aligned_offset{ + OpISub(U32[1], addr_low, OpBitwiseAnd(U32[1], ssbo_low, align_mask))}; + const Id word{OpShiftRightLogical(U32[1], aligned_offset, Const(2U))}; + finish(callback(Access{&ssbos[index], word, Id{}}, aligned_offset, value)); AddLabel(else_label); } + if (uses_global_pointers) { + const Id table_pointer{ + OpAccessChain(TypePointer(spv::StorageClass::PushConstant, U32[2]), + rescaling_push_constants, Const(global_pointer_member_index))}; + const Id count_pointer{ + OpAccessChain(TypePointer(spv::StorageClass::PushConstant, U32[1]), + rescaling_push_constants, Const(global_pointer_member_index + 1))}; + const Id table{OpBitcast(U64, OpLoad(U32[2], table_pointer))}; + const Id count{OpLoad(U32[1], count_pointer)}; + const Id header_label{OpLabel()}; + const Id body_label{OpLabel()}; + const Id hit_label{OpLabel()}; + const Id skip_label{OpLabel()}; + const Id continue_label{OpLabel()}; + const Id merge_label{OpLabel()}; + OpBranch(header_label); + AddLabel(header_label); + const Id entry{OpLoad(U32[1], entry_index)}; + const Id in_table{OpULessThan(U1, entry, count)}; + OpLoopMerge(merge_label, continue_label, spv::LoopControlMask::MaskNone); + OpBranchConditional(in_table, body_label, merge_label); + AddLabel(body_label); + const Id entry_address{OpIAdd( + U64, table, OpUConvert(U64, OpShiftLeftLogical(U32[1], entry, Const(5U))))}; + const auto [guest, guest_base]{load_entry(entry_address)}; + const Id guest_offset{OpISub(U64, addr, guest_base)}; + const Id guest_size{OpUConvert(U64, OpCompositeExtract(U32[1], guest, 2U))}; + const Id hit{OpULessThan(U1, guest_offset, guest_size)}; + OpSelectionMerge(skip_label, spv::SelectionControlMask::MaskNone); + OpBranchConditional(hit, hit_label, skip_label); + AddLabel(hit_label); + const Id host_base{ + load_entry(OpIAdd(U64, entry_address, Constant(U64, u64{16}))).second}; + const Id target{OpBitwiseAnd(U64, OpIAdd(U64, host_base, guest_offset), + Constant(U64, ~u64{3}))}; + finish(callback(Access{nullptr, Id{}, target}, addr_low, value)); + AddLabel(skip_label); + OpBranch(continue_label); + AddLabel(continue_label); + OpStore(entry_index, OpIAdd(U32[1], entry, Const(1U))); + OpBranch(header_label); + AddLabel(merge_label); + } if (returns_value) { OpReturnValue(ConstantNull(result_type)); } else { @@ -987,13 +1074,13 @@ void EmitContext::DefineGlobalMemoryFunctions(const IR::Program& program) { OpShiftRightLogical(U32[1], word, Const(2U))); }}; const auto load{[&](Id type, u32 count) { - return [&, type, count](const StorageDefinitions& ssbo, Id word, Id, Id) { - if (count > 1 && profile.support_descriptor_aliasing) { - return OpLoad(type, vector_pointer(ssbo, word, count)); + return [&, type, count](const Access& access, Id, Id) { + if (count > 1 && access.ssbo && profile.support_descriptor_aliasing) { + return OpLoad(type, vector_pointer(*access.ssbo, access.word, count)); } std::array words{}; for (u32 element = 0; element < count; ++element) { - words[element] = OpLoad(U32[1], word_pointer(ssbo.U32, word, element)); + words[element] = load_word(access, element); } if (count == 1) { return words[0]; @@ -1002,25 +1089,24 @@ void EmitContext::DefineGlobalMemoryFunctions(const IR::Program& program) { }; }}; const auto store{[&](u32 count) { - return [&, count](const StorageDefinitions& ssbo, Id word, Id, Id value) { - if (count > 1 && profile.support_descriptor_aliasing) { - OpStore(vector_pointer(ssbo, word, count), value); + return [&, count](const Access& access, Id, Id value) { + if (count > 1 && access.ssbo && profile.support_descriptor_aliasing) { + OpStore(vector_pointer(*access.ssbo, access.word, count), value); return Id{}; } if (count == 1) { - OpStore(word_pointer(ssbo.U32, word, 0), value); + store_word(access, 0, value); return Id{}; } for (u32 element = 0; element < count; ++element) { - OpStore(word_pointer(ssbo.U32, word, element), - OpCompositeExtract(U32[1], value, element)); + store_word(access, element, OpCompositeExtract(U32[1], value, element)); } return Id{}; }; }}; const auto extract{[&](bool is_signed, u32 count) { - return [&, is_signed, count](const StorageDefinitions& ssbo, Id word, Id offset, Id) { - const Id loaded{OpLoad(U32[1], word_pointer(ssbo.U32, word, 0))}; + return [&, is_signed, count](const Access& access, Id offset, Id) { + const Id loaded{load_word(access, 0)}; if (is_signed) { return OpBitFieldSExtract(U32[1], loaded, bits(offset, count), Const(count)); } @@ -1028,28 +1114,36 @@ void EmitContext::DefineGlobalMemoryFunctions(const IR::Program& program) { }; }}; const auto insert{[&](u32 count) { - return [&, count](const StorageDefinitions& ssbo, Id word, Id offset, Id value) { - AtomicBitFieldInsert(word_pointer(ssbo.U32, word, 0), value, bits(offset, count), + return [&, count](const Access& access, Id offset, Id value) { + AtomicBitFieldInsert(word_pointer(access, 0), value, bits(offset, count), Const(count)); return Id{}; }; }}; const auto atomic{[&](Id (Sirit::Module::*func)(Id, Id, Id, Id, Id)) { - return [&, func](const StorageDefinitions& ssbo, Id word, Id, Id value) { - return (this->*func)(U32[1], word_pointer(ssbo.U32, word, 0), scope, zero, value); + return [&, func](const Access& access, Id, Id value) { + return (this->*func)(U32[1], word_pointer(access, 0), scope, zero, value); }; }}; const auto cas{[&](Id type, Id helper) { - return [&, type, helper](const StorageDefinitions& ssbo, Id word, Id, Id value) { - return OpFunctionCall(type, helper, word, value, ssbo.U32); + return [&, type, helper](const Access& access, Id, Id value) { + if (!access.ssbo) { + return ConstantNull(type); + } + return OpFunctionCall(type, helper, access.word, value, access.ssbo->U32); }; }}; const auto packed{[&](bool is_half, Id helper) { - return [&, is_half, helper](const StorageDefinitions& ssbo, Id word, Id, Id value) { - if (is_half) { - return OpBitcast(U32[1], OpFunctionCall(F16[2], helper, word, value, ssbo.U32)); + return [&, is_half, helper](const Access& access, Id, Id value) { + if (!access.ssbo) { + return ConstantNull(U32[1]); } - return OpPackHalf2x16(U32[1], OpFunctionCall(F32[2], helper, word, value, ssbo.U32)); + const Id ssbo{access.ssbo->U32}; + if (is_half) { + return OpBitcast(U32[1], OpFunctionCall(F16[2], helper, access.word, value, ssbo)); + } + return OpPackHalf2x16(U32[1], + OpFunctionCall(F32[2], helper, access.word, value, ssbo)); }; }}; for (const IR::Block* const block : program.post_order_blocks) { @@ -1159,7 +1253,9 @@ void EmitContext::DefineGlobalMemoryFunctions(const IR::Program& program) { } void EmitContext::DefineRescalingInput(const Info& info) { - if (!info.uses_rescaling_uniform) { + uses_global_pointers = info.uses_global_pointers && profile.support_buffer_device_address && + profile.support_int64 && profile.unified_descriptor_binding; + if (!info.uses_rescaling_uniform && !uses_global_pointers) { return; } if (profile.unified_descriptor_binding) { @@ -1170,7 +1266,7 @@ void EmitContext::DefineRescalingInput(const Info& info) { } void EmitContext::DefineRescalingInputPushConstant() { - boost::container::static_vector members{}; + boost::container::static_vector members{}; u32 member_index{0}; rescaling_textures_type = TypeArray(U32[1], Const(4u)); @@ -1187,6 +1283,11 @@ void EmitContext::DefineRescalingInputPushConstant() { members.push_back(F32[1]); rescaling_downfactor_member_index = member_index++; } + if (uses_global_pointers) { + members.push_back(U32[2]); + members.push_back(U32[1]); + global_pointer_member_index = member_index; + } const Id push_constant_struct{TypeStruct(std::span(members.data(), members.size()))}; Decorate(push_constant_struct, spv::Decoration::Block); Name(push_constant_struct, "ResolutionInfo"); @@ -1205,6 +1306,14 @@ void EmitContext::DefineRescalingInputPushConstant() { static_cast(offsetof(RescalingLayout, down_factor))); MemberName(push_constant_struct, rescaling_downfactor_member_index, "down_factor"); } + if (uses_global_pointers) { + MemberDecorate(push_constant_struct, global_pointer_member_index, spv::Decoration::Offset, + GLOBAL_POINTER_LAYOUT_OFFSET); + MemberDecorate(push_constant_struct, global_pointer_member_index + 1, + spv::Decoration::Offset, + GLOBAL_POINTER_LAYOUT_OFFSET + + static_cast(offsetof(GlobalPointerLayout, count))); + } const Id pointer_type{TypePointer(spv::StorageClass::PushConstant, push_constant_struct)}; rescaling_push_constants = AddGlobalVariable(pointer_type, spv::StorageClass::PushConstant); Name(rescaling_push_constants, "rescaling_push_constants"); diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.h b/src/shader_recompiler/backend/spirv/spirv_emit_context.h index 68761e1f1a..904686e2c7 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.h +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.h @@ -307,6 +307,8 @@ public: u32 rescaling_textures_member_index{}; u32 rescaling_images_member_index{}; u32 rescaling_downfactor_member_index{}; + u32 global_pointer_member_index{}; + bool uses_global_pointers{}; u32 texture_rescaling_index{}; u32 image_rescaling_index{}; diff --git a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp index bf40b0695b..339b6aabd1 100644 --- a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp +++ b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp @@ -56,6 +56,7 @@ struct StorageInfo { StorageBufferSet set; StorageInstVector to_replace; StorageWritesSet writes; + bool pointers{}; }; /// Returns true when the instruction is a global memory instruction @@ -335,7 +336,8 @@ std::optional TrackLowAddress(IR::Inst* inst) { } /// Tries to track the storage buffer address used by a global memory instruction -StorageBufferSet Track(const IR::Value& value, const Bias* bias, const LocalStores& local_stores) { +StorageBufferSet Track(const IR::Value& value, const Bias* bias, const LocalStores& local_stores, + bool& from_memory) { const auto pred{[bias](const IR::Inst* inst) -> std::optional { if (inst->GetOpcode() != IR::Opcode::GetCbufU32 && inst->GetOpcode() != IR::Opcode::GetCbufU32x2) { @@ -386,6 +388,8 @@ StorageBufferSet Track(const IR::Value& value, const Bias* bias, const LocalStor } switch (inst->GetOpcode()) { case IR::Opcode::LoadLocal: + from_memory |= + !inst->Arg(0).IsImmediate() || !local_stores.contains(inst->Arg(0).U32()); if (inst->Arg(0).IsImmediate()) { const auto [begin, end]{local_stores.equal_range(inst->Arg(0).U32())}; for (auto it = begin; it != end; ++it) { @@ -405,6 +409,7 @@ StorageBufferSet Track(const IR::Value& value, const Bias* bias, const LocalStor case IR::Opcode::GetCbufU32: case IR::Opcode::GetCbufF32: case IR::Opcode::GetCbufU32x2: + continue; case IR::Opcode::LoadSharedU8: case IR::Opcode::LoadSharedS8: case IR::Opcode::LoadSharedU16: @@ -412,11 +417,13 @@ StorageBufferSet Track(const IR::Value& value, const Bias* bias, const LocalStor case IR::Opcode::LoadSharedU32: case IR::Opcode::LoadSharedU64: case IR::Opcode::LoadSharedU128: + from_memory = true; continue; default: break; } if (IsGlobalMemory(*inst) || inst->MayHaveSideEffects()) { + from_memory = true; continue; } for (size_t arg = 0; arg < inst->NumArgs(); ++arg) { @@ -457,18 +464,21 @@ void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info, const std::optional low_addr_info{TrackLowAddress(&inst)}; if (!low_addr_info) { // Failed to track the low address, use NVN fallbacks + info.pointers = true; return; } // First try to find storage buffers in the NVN address const IR::U32 low_addr{low_addr_info->value}; - StorageBufferSet candidates{Track(low_addr, &nvn_bias, local_stores)}; + bool from_memory{}; + StorageBufferSet candidates{Track(low_addr, &nvn_bias, local_stores, from_memory)}; if (candidates.empty()) { // If it fails, track without a bias - candidates = Track(low_addr, nullptr, local_stores); + candidates = Track(low_addr, nullptr, local_stores, from_memory); } if (candidates.size() != 1) { // If that also fails, use NVN fallbacks LOG_WARNING(Shader, "Storage buffer failed to track, using global memory fallbacks"); + info.pointers |= from_memory; return; } const StorageBufferAddr storage_buffer{*candidates.begin()}; @@ -615,6 +625,7 @@ void GlobalMemoryToStorageBufferPass(IR::Program& program, const HostTranslateIn .is_written = info.writes.contains(storage_buffer), }); } + program.info.uses_global_pointers = info.pointers; for (const StorageInst& storage_inst : info.to_replace) { const StorageBufferAddr storage_buffer{storage_inst.storage_buffer}; const auto it{info.set.find(storage_inst.storage_buffer)}; @@ -628,6 +639,7 @@ void GlobalMemoryToStorageBufferPass(IR::Program& program, const HostTranslateIn } void JoinStorageInfo(Info& base, Info& source) { + base.uses_global_pointers |= source.uses_global_pointers; auto& descriptors = base.storage_buffers_descriptors; for (auto& desc : source.storage_buffers_descriptors) { auto it{std::ranges::find_if(descriptors, [&desc](const auto& existing) { diff --git a/src/shader_recompiler/profile.h b/src/shader_recompiler/profile.h index 8e9c83f888..3b5b159384 100644 --- a/src/shader_recompiler/profile.h +++ b/src/shader_recompiler/profile.h @@ -47,6 +47,7 @@ struct Profile { bool support_typeless_image_loads{}; bool support_demote_to_helper_invocation{}; bool support_int64_atomics{}; + bool support_buffer_device_address{}; bool support_shared_int64_atomics{}; bool support_derivative_control{}; bool support_geometry_shader_passthrough{}; diff --git a/src/shader_recompiler/shader_info.h b/src/shader_recompiler/shader_info.h index b8d760a30e..6729b338b5 100644 --- a/src/shader_recompiler/shader_info.h +++ b/src/shader_recompiler/shader_info.h @@ -312,6 +312,7 @@ struct Info { bool uses_atomic_s32_max{}; bool uses_int64_bit_atomics{}; bool uses_global_memory{}; + bool uses_global_pointers{}; bool uses_atomic_image_u32{}; bool uses_shadow_lod{}; bool uses_rescaling_uniform{}; diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 087d7ac1f8..5ab5a6f4f3 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -123,6 +123,8 @@ void BufferCache

::TickFrame() { if (total_used_memory >= minimum_memory || heap_pressure) { RunGarbageCollector(); } + std::erase_if(pointer_ranges, + [this](const PointerRange& range) { return range.frame + 2 < frame_tick; }); ++frame_tick; delayed_destruction_ring.Tick(); @@ -1144,6 +1146,7 @@ void BufferCache

::BindHostGraphicsStorageBuffers(size_t stage) { if (is_written) { MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); + RecordPointerRange(binding); } if constexpr (NEEDS_BIND_STORAGE_INDEX) { @@ -1284,6 +1287,7 @@ void BufferCache

::BindHostComputeStorageBuffers() { if (is_written) { MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); + RecordPointerRange(binding); } if constexpr (NEEDS_BIND_STORAGE_INDEX) { @@ -1341,6 +1345,7 @@ void BufferCache

::DoUpdateGraphicsBuffers(bool is_indexed) { if (current_draw_indirect) { UpdateDrawIndirect(); } + UpdatePointerRanges(); }); } @@ -1351,6 +1356,7 @@ void BufferCache

::DoUpdateComputeBuffers() { UpdateComputeUniformBuffers(); UpdateComputeStorageBuffers(); UpdateComputeTextureBuffers(); + UpdatePointerRanges(); }); } @@ -1496,6 +1502,72 @@ void BufferCache

::UpdateStorageBuffers(size_t stage) { }); } +template +void BufferCache

::UpdatePointerRanges() { + if (!pointer_table_requested) { + return; + } + for (PointerRange& range : pointer_ranges) { + range.buffer_id = NULL_BUFFER_ID; + if (gpu_memory->GpuToCpuAddress(range.gpu_addr) == range.device_addr) { + range.buffer_id = FindBuffer(range.device_addr, range.size, false); + } + } +} + +template +void BufferCache

::RecordPointerRange(const Binding& binding) { + if (!record_pointer_ranges) { + return; + } + const auto it = std::ranges::find(pointer_ranges, binding.gpu_addr, &PointerRange::gpu_addr); + if (it != pointer_ranges.end()) { + it->device_addr = binding.device_addr; + it->size = (std::max)(it->size, binding.size); + it->frame = frame_tick; + return; + } + if (pointer_ranges.size() < MAX_POINTER_RANGES) { + pointer_ranges.push_back({binding.gpu_addr, binding.device_addr, binding.size, frame_tick, + NULL_BUFFER_ID}); + } +} + +template +void BufferCache

::RequestPointerTable(bool enable) noexcept { + pointer_table_requested = enable; + record_pointer_ranges |= enable; +} + +template +std::array BufferCache

::BindHostPointerTable(bool is_written) { + std::array table{}; + if constexpr (!IS_OPENGL) { + const size_t capacity = (std::max)(pointer_ranges.size(), size_t{1}); + const auto upload = runtime.UploadStagingBuffer(capacity * sizeof(std::array)); + u32 count = 0; + for (const PointerRange& range : pointer_ranges) { + if (range.buffer_id == NULL_BUFFER_ID) { + continue; + } + Buffer& buffer = slot_buffers[range.buffer_id]; + TouchBuffer(buffer, range.buffer_id); + SynchronizeBuffer(buffer, range.device_addr, range.size); + if (is_written) { + MarkWrittenBuffer(range.buffer_id, range.device_addr, range.size); + } + const u64 host_address = buffer.DeviceAddress() + buffer.Offset(range.device_addr); + const std::array entry{range.gpu_addr, range.size, host_address, 0}; + std::memcpy(upload.mapped_span.data() + count * sizeof(entry), entry.data(), + sizeof(entry)); + ++count; + } + const u64 address = upload.device_address + upload.offset; + table = {static_cast(address), static_cast(address >> 32), count}; + } + return table; +} + template void BufferCache

::UpdateTextureBuffers(size_t stage) { ForEachEnabledBit(channel_state->enabled_texture_buffers[stage], [&](u32 index) { diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index d1f9ff5f97..b7802d5f76 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -270,6 +270,10 @@ public: void BindHostComputeBuffers(); + void RequestPointerTable(bool enable) noexcept; + + [[nodiscard]] std::array BindHostPointerTable(bool is_written); + void SetUniformBuffersState(const std::array& mask, const UniformBufferSizes* sizes); @@ -431,6 +435,10 @@ private: void UpdateStorageBuffers(size_t stage); + void UpdatePointerRanges(); + + void RecordPointerRange(const Binding& binding); + void UpdateTextureBuffers(size_t stage); void UpdateTransformFeedbackBuffers(); @@ -563,6 +571,17 @@ private: std::vector graphics_segments; std::vector compute_segments; u64 frame_tick = 0; + struct PointerRange { + GPUVAddr gpu_addr; + DAddr device_addr; + u32 size; + u64 frame; + BufferId buffer_id; + }; + static constexpr size_t MAX_POINTER_RANGES = 256; + std::vector pointer_ranges; + bool record_pointer_ranges = false; + bool pointer_table_requested = false; u64 total_used_memory = 0; u64 device_local_memory = 0; u64 minimum_memory = 0; diff --git a/src/video_core/renderer_vulkan/pipeline_helper.h b/src/video_core/renderer_vulkan/pipeline_helper.h index 07de25f18f..3e4a4e40be 100644 --- a/src/video_core/renderer_vulkan/pipeline_helper.h +++ b/src/video_core/renderer_vulkan/pipeline_helper.h @@ -243,15 +243,13 @@ public: } vk::PipelineLayout CreatePipelineLayout(VkDescriptorSetLayout descriptor_set_layout) const { - using Shader::Backend::SPIRV::RenderAreaLayout; - using Shader::Backend::SPIRV::RescalingLayout; - const u32 size_offset = is_compute ? sizeof(RescalingLayout::down_factor) : 0u; + using Shader::Backend::SPIRV::GLOBAL_POINTER_LAYOUT_OFFSET; + using Shader::Backend::SPIRV::GlobalPointerLayout; const VkPushConstantRange range{ .stageFlags = static_cast( is_compute ? VK_SHADER_STAGE_COMPUTE_BIT : VK_SHADER_STAGE_ALL_GRAPHICS), .offset = 0, - .size = static_cast(sizeof(RescalingLayout)) - size_offset + - static_cast(sizeof(RenderAreaLayout)), + .size = GLOBAL_POINTER_LAYOUT_OFFSET + static_cast(sizeof(GlobalPointerLayout)), }; return device->GetLogical().CreatePipelineLayout({ .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp index 840ff5897b..755c6611be 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp @@ -28,6 +28,7 @@ namespace Vulkan { using Shader::ImageBufferDescriptor; +using Shader::Backend::SPIRV::GLOBAL_POINTER_LAYOUT_OFFSET; using Shader::Backend::SPIRV::RESCALING_LAYOUT_WORDS_OFFSET; using Tegra::Texture::TexturePair; @@ -245,8 +246,14 @@ bool ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, std::ranges::for_each(info.texture_buffer_descriptors, add_buffer); std::ranges::for_each(info.image_buffer_descriptors, add_buffer); + const bool uses_pointers{info.uses_global_pointers && device.IsBufferDeviceAddressSupported()}; + buffer_cache.RequestPointerTable(uses_pointers); buffer_cache.UpdateComputeBuffers(); buffer_cache.BindHostComputeBuffers(); + std::array pointer_table{}; + if (uses_pointers) { + pointer_table = buffer_cache.BindHostPointerTable(info.stores_global_memory); + } RescalingPushConstant rescaling; const VideoCommon::SamplerId* samplers_it{samplers.data()}; @@ -288,7 +295,7 @@ bool ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, const bool is_rescaling = !info.texture_descriptors.empty() || !info.image_descriptors.empty(); scheduler.Record([this, descriptor_data, is_rescaling, descriptor_buffer_offset, - descriptor_buffer_chunk, bind_descriptor_buffer, + descriptor_buffer_chunk, bind_descriptor_buffer, uses_pointers, pointer_table, rescaling_data = rescaling.Data()](vk::CommandBuffer cmdbuf) { if (bind_descriptor_buffer) { const VkDescriptorBufferBindingInfoEXT binding_info{ @@ -299,6 +306,11 @@ bool ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, return; } cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); + if (uses_pointers) { + cmdbuf.PushConstants(*pipeline_layout, VK_SHADER_STAGE_COMPUTE_BIT, + GLOBAL_POINTER_LAYOUT_OFFSET, sizeof(pointer_table), + pointer_table.data()); + } if (!descriptor_set_layout) { return; } diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 0a8a2af973..354bcfa06f 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -41,6 +41,7 @@ namespace { using boost::container::small_vector; using boost::container::static_vector; using Shader::ImageBufferDescriptor; +using Shader::Backend::SPIRV::GLOBAL_POINTER_LAYOUT_OFFSET; using Shader::Backend::SPIRV::RENDERAREA_LAYOUT_OFFSET; using Shader::Backend::SPIRV::RESCALING_LAYOUT_DOWN_FACTOR_OFFSET; using Shader::Backend::SPIRV::RESCALING_LAYOUT_WORDS_OFFSET; @@ -301,6 +302,9 @@ GraphicsPipeline::GraphicsPipeline( num_image_elements += Shader::NumDescriptors(info->texture_descriptors); num_image_elements += Shader::NumDescriptors(info->image_descriptors); num_descriptor_entries += NumDescriptorEntries(*info); + uses_global_pointers |= + info->uses_global_pointers && device.IsBufferDeviceAddressSupported(); + stores_global_pointers |= info->stores_global_memory; } fragment_has_color0_output = stage_infos[NUM_STAGES - 1].stores_frag_color[0]; @@ -536,6 +540,7 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { scheduler.RequestOutsideRenderPassOperationContext(); } + buffer_cache.RequestPointerTable(uses_global_pointers); buffer_cache.UpdateGraphicsBuffers(is_indexed); buffer_cache.BindHostGeometryBuffers(is_indexed); @@ -571,6 +576,9 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { if constexpr (Spec::enabled_stages[4]) { prepare_stage(4); } + if (uses_global_pointers) { + pointer_table = buffer_cache.BindHostPointerTable(stores_global_pointers); + } texture_cache.UpdateRenderTargets(false); texture_cache.CheckFeedbackLoop(std::span{views.data(), views.size()}); @@ -656,7 +664,8 @@ bool GraphicsPipeline::ConfigureDraw(const RescalingPushConstant& rescaling, descriptor_buffer_offset, descriptor_buffer_chunk, bind_descriptor_buffer, rescaling_data = rescaling.Data(), is_rescaling, update_rescaling, uses_render_area = render_area.uses_render_area, - render_area_data = render_area.words](vk::CommandBuffer cmdbuf) { + render_area_data = render_area.words, + table = pointer_table](vk::CommandBuffer cmdbuf) { if (bind_descriptor_buffer) { const VkDescriptorBufferBindingInfoEXT binding_info{ descriptor_buffer_ring.BindingInfo(descriptor_buffer_chunk)}; @@ -683,6 +692,10 @@ bool GraphicsPipeline::ConfigureDraw(const RescalingPushConstant& rescaling, RENDERAREA_LAYOUT_OFFSET, sizeof(render_area_data), &render_area_data); } + if (uses_global_pointers) { + cmdbuf.PushConstants(*pipeline_layout, VK_SHADER_STAGE_ALL_GRAPHICS, + GLOBAL_POINTER_LAYOUT_OFFSET, sizeof(table), table.data()); + } if (!descriptor_set_layout) { return; } diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h index 1371e8ee06..26c28d536e 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h @@ -168,6 +168,9 @@ private: size_t num_image_elements{}; u32 num_textures{}; bool fragment_has_color0_output{}; + bool uses_global_pointers{}; + bool stores_global_pointers{}; + std::array pointer_table{}; vk::DescriptorSetLayout descriptor_set_layout; DescriptorAllocator descriptor_allocator; diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index dd55b576e8..3d0905d221 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -421,6 +421,7 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, .support_demote_to_helper_invocation = device.IsExtShaderDemoteToHelperInvocationSupported(), .support_int64_atomics = device.IsExtShaderAtomicInt64Supported(), + .support_buffer_device_address = device.IsBufferDeviceAddressSupported(), .support_shared_int64_atomics = device.IsSharedInt64AtomicsSupported(), .support_derivative_control = true, .support_geometry_shader_passthrough = device.IsNvGeometryShaderPassthroughSupported(),