shader_recompiler: complete geometry-stage compaction fallback (#4356)

#4330's gl_PrimitiveId scatter only rewrote the elected lane's atomic.
Every other invocation still got its index through a subgroup shuffle,
which the fallback resolves to its own unchanged local value, so all
non-elected qualifying primitives in a subgroup collide into the same
slot. Extend the same gl_PrimitiveId scatter to the non-elected path,
using each lane's own already-local stride (no cross-lane read needed).

Separately, the predicate gating whether an invocation writes its
compacted data is dead code for this shader's own emitted geometry: a
working reference compile of the same guest shader eliminates it and
its inputs entirely and writes unconditionally. Under our fallback it
is not dead, so invocations where it evaluates false simply never
write their slot. Neutralize the predicate feeding the matched ballot
to match that known-correct unconditional behavior.

Confirmed fixing residual missing/incomplete compacted meshes in NieR
Automata beyond what #4330 alone resolved, no regressions observed on
vendor Qualcomm.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4356

src/shader_recompiler/CMakeLists.txt

@@ -217,6 +217,7 @@ add_library(shader_recompiler STATIC
     frontend/maxwell/translate_program.h
     host_translate_info.h
     ir_opt/collect_shader_info_pass.cpp
+    ir_opt/compaction_fallback_pass.cpp
     ir_opt/conditional_barrier_pass.cpp
     ir_opt/constant_propagation_pass.cpp
     ir_opt/dead_code_elimination_pass.cpp

src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp

@@ -217,6 +217,10 @@ Id EmitShuffleIndex(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id cla
 
 Id EmitShuffleUp(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
                  Id segmentation_mask) {
+    if (!StageSupportsSubgroups(ctx)) {
+        SetInBoundsFlag(inst, ctx.false_value);
+        return value;
+    }
     const Id thread_id{EmitLaneId(ctx)};
     const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)};
     Id src_thread_id{ctx.OpISub(ctx.U32[1], thread_id, index)};
@@ -232,6 +236,10 @@ Id EmitShuffleUp(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
 
 Id EmitShuffleDown(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
                    Id segmentation_mask) {
+    if (!StageSupportsSubgroups(ctx)) {
+        SetInBoundsFlag(inst, ctx.false_value);
+        return value;
+    }
     const Id thread_id{EmitLaneId(ctx)};
     const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)};
     Id src_thread_id{ctx.OpIAdd(ctx.U32[1], thread_id, index)};
@@ -247,6 +255,10 @@ Id EmitShuffleDown(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clam
 
 Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
                         Id segmentation_mask) {
+    if (!StageSupportsSubgroups(ctx)) {
+        SetInBoundsFlag(inst, ctx.false_value);
+        return value;
+    }
     const Id thread_id{EmitLaneId(ctx)};
     const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)};
     Id src_thread_id{ctx.OpBitwiseXor(ctx.U32[1], thread_id, index)};

src/shader_recompiler/frontend/maxwell/translate_program.cpp

@@ -299,6 +299,7 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo
     Optimization::PositionPass(env, program);
 
     Optimization::GlobalMemoryToStorageBufferPass(program, normalized_host_info);
+    Optimization::CompactionFallbackPass(program, normalized_host_info);
     Optimization::TexturePass(env, program, normalized_host_info);
 
     if (Settings::values.resolution_info.active || Settings::values.rescale_hack.GetValue()) {

src/shader_recompiler/host_translate_info.h

@@ -38,6 +38,7 @@ struct HostTranslateInfo {
                                                 ///< passthrough shaders
     bool support_conditional_barrier{}; ///< True when the device supports barriers in conditional
                                         ///< control flow
+    bool support_subgroup_in_geometry_stage{};
 
     void ApplyDescriptorLimitPolicy() noexcept {
         if (min_ssbo_alignment == 0) {

src/shader_recompiler/ir_opt/compaction_fallback_pass.cpp

@@ -0,0 +1,172 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "shader_recompiler/frontend/ir/ir_emitter.h"
+#include "shader_recompiler/frontend/ir/program.h"
+#include "shader_recompiler/host_translate_info.h"
+#include "shader_recompiler/ir_opt/passes.h"
+
+namespace Shader::Optimization {
+namespace {
+
+constexpr int MAX_BALLOT_SEARCH_DEPTH = 8;
+
+bool DependsOnSubgroupBallot(const IR::Value& value, int depth) {
+    if (depth > MAX_BALLOT_SEARCH_DEPTH || value.IsImmediate()) {
+        return false;
+    }
+    IR::Inst* const producer{value.Inst()};
+    if (producer->GetOpcode() == IR::Opcode::SubgroupBallot) {
+        return true;
+    }
+    if (producer->GetOpcode() == IR::Opcode::Phi) {
+        return false;
+    }
+    const size_t num_args{producer->NumArgs()};
+    for (size_t index = 0; index < num_args; ++index) {
+        if (DependsOnSubgroupBallot(producer->Arg(index), depth + 1)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+IR::Inst* FindSubgroupBallot(const IR::Value& value, int depth) {
+    if (depth > MAX_BALLOT_SEARCH_DEPTH || value.IsImmediate()) {
+        return nullptr;
+    }
+    IR::Inst* const producer{value.Inst()};
+    if (producer->GetOpcode() == IR::Opcode::SubgroupBallot) {
+        return producer;
+    }
+    if (producer->GetOpcode() == IR::Opcode::Phi) {
+        return nullptr;
+    }
+    const size_t num_args{producer->NumArgs()};
+    for (size_t index = 0; index < num_args; ++index) {
+        if (IR::Inst* const found{FindSubgroupBallot(producer->Arg(index), depth + 1)}) {
+            return found;
+        }
+    }
+    return nullptr;
+}
+
+IR::Value UnwrapIdentity(IR::Value value) {
+    for (; !value.IsImmediate() && value.Inst()->GetOpcode() == IR::Opcode::Identity;
+         value = value.Inst()->Arg(0))
+        ;
+    return value;
+}
+
+bool IsShuffleFamily(IR::Opcode op) {
+    switch (op) {
+    case IR::Opcode::ShuffleIndex:
+    case IR::Opcode::ShuffleUp:
+    case IR::Opcode::ShuffleDown:
+    case IR::Opcode::ShuffleButterfly:
+        return true;
+    default:
+        return false;
+    }
+}
+
+void FixNonElectedPath(IR::Block& elected_block, IR::Inst& atomic) {
+    for (IR::Block* const successor : elected_block.ImmSuccessors()) {
+        for (IR::Inst& phi : successor->Instructions()) {
+            if (phi.GetOpcode() != IR::Opcode::Phi) {
+                continue;
+            }
+            const size_t num_args{phi.NumArgs()};
+            size_t elected_index = static_cast<size_t>(-1);
+            for (size_t i = 0; i < num_args; ++i) {
+                if (phi.PhiBlock(i) == &elected_block) {
+                    elected_index = i;
+                    break;
+                }
+            }
+            if (elected_index == size_t(-1)) {
+                continue;
+            }
+            const IR::Value elected_value{UnwrapIdentity(phi.Arg(elected_index))};
+            if (elected_value.IsImmediate() || elected_value.Inst() != &atomic) {
+                continue;
+            }
+            for (size_t i = 0; i < num_args; ++i) {
+                IR::Block* const pred{phi.PhiBlock(i)};
+                if (pred == &elected_block) {
+                    continue;
+                }
+                const IR::Value shuffle_value{UnwrapIdentity(phi.Arg(i))};
+                if (shuffle_value.IsImmediate()) {
+                    continue;
+                }
+                IR::Inst* const producer{shuffle_value.Inst()};
+                if (!IsShuffleFamily(producer->GetOpcode())) {
+                    continue;
+                }
+                const IR::U32 local_amount{UnwrapIdentity(producer->Arg(0))};
+                const auto insert_point{IR::Block::InstructionList::s_iterator_to(*producer)};
+                IR::IREmitter ir{*pred, insert_point};
+
+                const IR::U32 primitive_id{ir.GetAttributeU32(IR::Attribute::PrimitiveId)};
+                const IR::U32 new_index{ir.IMul(primitive_id, local_amount)};
+
+                producer->ReplaceUsesWith(new_index);
+            }
+        }
+    }
+}
+
+void NeutralizeCullPredicate(IR::Inst& ballot) {
+    const IR::Value pred{UnwrapIdentity(ballot.Arg(0))};
+    if (pred.IsImmediate()) {
+        return;
+    }
+    pred.Inst()->ReplaceUsesWith(IR::Value{true});
+}
+
+void RewriteAtomic(IR::Block& block, IR::Inst& inst) {
+    const auto insert_point{IR::Block::InstructionList::s_iterator_to(inst)};
+    IR::IREmitter ir{block, insert_point};
+
+    const IR::U32 amount{inst.Arg(2)};
+    const IR::U32 primitive_id{ir.GetAttributeU32(IR::Attribute::PrimitiveId)};
+    const IR::U32 new_index{ir.IMul(primitive_id, amount)};
+    const IR::U32 new_atomic_value{ir.IAdd(new_index, amount)};
+
+    const IR::Value umax_result{&*block.PrependNewInst(
+        insert_point, IR::Opcode::StorageAtomicUMax32,
+        {inst.Arg(0), inst.Arg(1), new_atomic_value})};
+    static_cast<void>(umax_result);
+
+    inst.ReplaceUsesWith(new_index);
+}
+
+} // Anonymous namespace
+
+void CompactionFallbackPass(IR::Program& program, const HostTranslateInfo& host_info) {
+    if (program.stage != Stage::Geometry) {
+        return;
+    }
+    if (host_info.support_subgroup_in_geometry_stage) {
+        return;
+    }
+    for (IR::Block* const block : program.post_order_blocks) {
+        for (IR::Inst& inst : block->Instructions()) {
+            if (inst.GetOpcode() != IR::Opcode::StorageAtomicIAdd32) {
+                continue;
+            }
+            if (!DependsOnSubgroupBallot(inst.Arg(2), 0)) {
+                continue;
+            }
+            FixNonElectedPath(*block, inst);
+            IR::Inst* const ballot{FindSubgroupBallot(inst.Arg(2), 0)};
+            RewriteAtomic(*block, inst);
+            if (ballot) {
+                NeutralizeCullPredicate(*ballot);
+            }
+        }
+    }
+}
+
+} // namespace Shader::Optimization

src/shader_recompiler/ir_opt/passes.h

@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -13,6 +16,7 @@ struct HostTranslateInfo;
 namespace Shader::Optimization {
 
 void CollectShaderInfoPass(Environment& env, IR::Program& program);
+void CompactionFallbackPass(IR::Program& program, const HostTranslateInfo& host_info);
 void ConditionalBarrierPass(IR::Program& program);
 void ConstantPropagationPass(Environment& env, IR::Program& program);
 void DeadCodeEliminationPass(IR::Program& program);

src/video_core/renderer_opengl/gl_shader_cache.cpp

@@ -269,6 +269,8 @@ ShaderCache::ShaderCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
         .support_viewport_index_layer = device.HasVertexViewportLayer(),
         .support_geometry_shader_passthrough = device.HasGeometryShaderPassthrough(),
         .support_conditional_barrier = device.SupportsConditionalBarriers(),
+        .support_subgroup_in_geometry_stage =
+            profile.SupportsSubgroupStage(Shader::Stage::Geometry),
       } {
     host_info.ApplyDescriptorLimitPolicy();
     if (use_asynchronous_shaders) {

src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

@@ -360,9 +360,12 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
     const VkDriverId driver_id{device.GetDriverID()};
     const VkShaderStageFlags subgroup_stages{device.GetSubgroupSupportedStages()};
     const auto subgroup_stage_bit{[subgroup_stages](VkShaderStageFlags flag, Shader::Stage stage) {
-        return (subgroup_stages & flag) != 0 ? (1u << static_cast<u32>(stage)) : 0u;
+        if ((subgroup_stages & flag) == 0) {
+            return 0u;
+        }
+        return 1u << static_cast<u32>(stage);
     }};
-    const u32 supported_subgroup_stages{
+    u32 supported_subgroup_stages{
         subgroup_stage_bit(VK_SHADER_STAGE_VERTEX_BIT, Shader::Stage::VertexA) |
         subgroup_stage_bit(VK_SHADER_STAGE_VERTEX_BIT, Shader::Stage::VertexB) |
         subgroup_stage_bit(VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT,
@@ -372,6 +375,11 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
         subgroup_stage_bit(VK_SHADER_STAGE_GEOMETRY_BIT, Shader::Stage::Geometry) |
         subgroup_stage_bit(VK_SHADER_STAGE_FRAGMENT_BIT, Shader::Stage::Fragment) |
         subgroup_stage_bit(VK_SHADER_STAGE_COMPUTE_BIT, Shader::Stage::Compute)};
+    if (!device.AreSubgroupFeaturesSupported(VK_SUBGROUP_FEATURE_VOTE_BIT |
+                                             VK_SUBGROUP_FEATURE_BALLOT_BIT |
+                                             VK_SUBGROUP_FEATURE_SHUFFLE_BIT)) {
+        supported_subgroup_stages = 0u;
+    }
     profile = Shader::Profile{
         .supported_spirv = device.SupportedSpirvVersion(),
         .unified_descriptor_binding = true,
@@ -479,6 +487,7 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
         .support_viewport_index_layer = device.IsExtShaderViewportIndexLayerSupported(),
         .support_geometry_shader_passthrough = device.IsNvGeometryShaderPassthroughSupported(),
         .support_conditional_barrier = device.SupportsConditionalBarriers(),
+        .support_subgroup_in_geometry_stage = profile.SupportsSubgroupStage(Shader::Stage::Geometry),
     };
     host_info.ApplyDescriptorLimitPolicy();
 

src/video_core/vulkan_common/vulkan_device.h

@@ -479,6 +479,11 @@ FN_MAX_LIMIT_LIST
         return properties.subgroup_properties.supportedStages;
     }
 
+    /// Returns true if every requested subgroup operation is supported by the device.
+    bool AreSubgroupFeaturesSupported(VkSubgroupFeatureFlags features) const {
+        return (properties.subgroup_properties.supportedOperations & features) == features;
+    }
+
     /// Returns the maximum number of push descriptors.
     u32 MaxPushDescriptors() const {
         return properties.push_descriptor.maxPushDescriptors;