[vulkan] Adjustments on MSAA and BlitHelpers (#4287)

PR/Commit Owner: CamilleLaVey

This PR contains changes complementary to the MSAA refactor from some weeks ago, adds proper shader convert to depth, stencil for msaa and non msaa convertions; removes redundant helpers and unify paths on the resolve functions, just to make the readability and maintainability better; includes fixes for the blit operations on Nvidia, Intel/Windows (previously banned) and ensures Linux retain the fix without extra burden; fixes the QCOM driver bug on resolution upscaling on any games above x1 (includes turnip on A8xx series) and fixes crashes/ resolution upscaled bugs on certain games that used to have wrong viewporting effect on screen or directly device loss on Vulkan. Adds fixes for regressions caused by previous MSAA refactor on games as Fire Emblem: Engage.

Special Thanks

1.- Big Smolio (@Gidoly)

Co-authored-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4287
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-by: Samuel <lizzie@eden-emu.dev>

src/video_core/host_shaders/CMakeLists.txt

@@ -18,6 +18,7 @@ set(SHADER_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_float.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d.comp
     ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_msaa.frag
+    ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_msaa.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_stencil_msaa.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d.comp
@@ -30,8 +31,12 @@ set(SHADER_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_float_to_depth.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_msaa_to_non_msaa.comp
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_msaa_to_non_msaa.frag
+    ${CMAKE_CURRENT_SOURCE_DIR}/convert_msaa_to_non_msaa_depth.frag
+    ${CMAKE_CURRENT_SOURCE_DIR}/convert_msaa_to_non_msaa_depth_stencil.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa.comp
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa.frag
+    ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa_depth.frag
+    ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa_depth_stencil.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_s8d24_to_abgr8.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/full_screen_triangle.vert
     ${CMAKE_CURRENT_SOURCE_DIR}/fxaa.frag
@@ -163,6 +168,44 @@ foreach(SOURCE_FILE IN ITEMS ${SHADER_FILES})
     endif()
 endforeach()
 
+# Integer variants of the MSAA conversion shaders. They only differ from the float
+# source in the sampler and output types, so they are generated from it via defines.
+set(SHADER_TYPE_VARIANTS
+    "convert_msaa_to_non_msaa.frag|sint|isampler2DMS|ivec4"
+    "convert_msaa_to_non_msaa.frag|uint|usampler2DMS|uvec4"
+    "convert_non_msaa_to_msaa.frag|sint|isampler2D|ivec4"
+    "convert_non_msaa_to_msaa.frag|uint|usampler2D|uvec4"
+)
+
+foreach(VARIANT IN ITEMS ${SHADER_TYPE_VARIANTS})
+    string(REPLACE "|" ";" VARIANT_PARTS ${VARIANT})
+    list(GET VARIANT_PARTS 0 VARIANT_FILENAME)
+    list(GET VARIANT_PARTS 1 VARIANT_SUFFIX)
+    list(GET VARIANT_PARTS 2 VARIANT_SAMPLER)
+    list(GET VARIANT_PARTS 3 VARIANT_TEXEL)
+
+    set(VARIANT_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/${VARIANT_FILENAME})
+    get_filename_component(VARIANT_STEM ${VARIANT_FILENAME} NAME_WE)
+    get_filename_component(VARIANT_EXT ${VARIANT_FILENAME} EXT)
+    string(REPLACE "." "" VARIANT_EXT ${VARIANT_EXT})
+    set(VARIANT_NAME ${VARIANT_STEM}_${VARIANT_SUFFIX}_${VARIANT_EXT})
+
+    string(TOUPPER ${VARIANT_NAME}_SPV VARIANT_VARIABLE_NAME)
+    set(VARIANT_HEADER_FILE ${SHADER_DIR}/${VARIANT_NAME}_spv.h)
+    add_custom_command(
+        OUTPUT
+            ${VARIANT_HEADER_FILE}
+        COMMAND
+            ${GLSLANGVALIDATOR} -V ${QUIET_FLAG} -I"${FIDELITYFX_INCLUDE_DIR}" ${GLSL_FLAGS}
+            -DSAMPLER_TYPE=${VARIANT_SAMPLER} -DTEXEL_TYPE=${VARIANT_TEXEL}
+            --variable-name ${VARIANT_VARIABLE_NAME} -o ${VARIANT_HEADER_FILE} ${VARIANT_SOURCE}
+            --target-env ${SPIR_V_VERSION}
+        MAIN_DEPENDENCY
+            ${VARIANT_SOURCE}
+    )
+    set(SHADER_HEADERS ${SHADER_HEADERS} ${VARIANT_HEADER_FILE})
+endforeach()
+
 foreach(FILEPATH IN ITEMS ${FIDELITYFX_FILES})
     get_filename_component(FILENAME ${FILEPATH} NAME)
     string(REPLACE "." "_" HEADER_NAME ${FILENAME})

src/video_core/host_shaders/blit_depth.frag

@@ -0,0 +1,12 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#version 450 core
+
+layout(binding = 0) uniform sampler2D depth_tex;
+
+layout(location = 0) in vec2 texcoord;
+
+void main() {
+    gl_FragDepth = textureLod(depth_tex, texcoord, 0).r;
+}

src/video_core/host_shaders/blit_depth_msaa.frag

@@ -8,5 +8,5 @@ layout(binding = 0) uniform sampler2DMS depth_tex;
 layout(location = 0) in vec2 texcoord;
 
 void main() {
-    gl_FragDepth = texelFetch(depth_tex, ivec2(texcoord), 0).r;
+    gl_FragDepth = texelFetch(depth_tex, ivec2(texcoord), gl_SampleID).r;
 }

src/video_core/host_shaders/blit_depth_stencil_msaa.frag

@@ -10,6 +10,6 @@ layout(binding = 1) uniform usampler2DMS stencil_tex;
 layout(location = 0) in vec2 texcoord;
 
 void main() {
-    gl_FragDepth = texelFetch(depth_tex, ivec2(texcoord), 0).r;
-    gl_FragStencilRefARB = int(texelFetch(stencil_tex, ivec2(texcoord), 0).r);
+    gl_FragDepth = texelFetch(depth_tex, ivec2(texcoord), gl_SampleID).r;
+    gl_FragStencilRefARB = int(texelFetch(stencil_tex, ivec2(texcoord), gl_SampleID).r);
 }

src/video_core/host_shaders/convert_msaa_to_non_msaa.frag

@@ -3,7 +3,14 @@
 
 #version 450 core
 
-layout(binding = 0) uniform sampler2DMS msaa_in;
+#ifndef SAMPLER_TYPE
+#define SAMPLER_TYPE sampler2DMS
+#endif
+#ifndef TEXEL_TYPE
+#define TEXEL_TYPE vec4
+#endif
+
+layout(binding = 0) uniform SAMPLER_TYPE msaa_in;
 
 layout(push_constant) uniform PushConstants {
     ivec2 dst_offset;
@@ -11,7 +18,7 @@ layout(push_constant) uniform PushConstants {
     ivec2 scale;
 };
 
-layout(location = 0) out vec4 frag_color;
+layout(location = 0) out TEXEL_TYPE frag_color;
 
 void main() {
     const ivec2 coord = ivec2(gl_FragCoord.xy) - dst_offset + src_offset;

src/video_core/host_shaders/convert_msaa_to_non_msaa_depth.frag

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#version 450 core
+
+layout(binding = 0) uniform sampler2DMS msaa_in;
+
+layout(push_constant) uniform PushConstants {
+    ivec2 dst_offset;
+    ivec2 src_offset;
+    ivec2 scale;
+};
+
+void main() {
+    const ivec2 coord = ivec2(gl_FragCoord.xy) - dst_offset + src_offset;
+    const ivec2 msaa_coord = coord / scale;
+    const ivec2 sample_offset = coord % scale;
+    const int sample_id = sample_offset.x + scale.x * sample_offset.y;
+    gl_FragDepth = texelFetch(msaa_in, msaa_coord, sample_id).r;
+}

src/video_core/host_shaders/convert_msaa_to_non_msaa_depth_stencil.frag

@@ -0,0 +1,23 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#version 450 core
+#extension GL_ARB_shader_stencil_export : require
+
+layout(binding = 0) uniform sampler2DMS depth_tex;
+layout(binding = 1) uniform usampler2DMS stencil_tex;
+
+layout(push_constant) uniform PushConstants {
+    ivec2 dst_offset;
+    ivec2 src_offset;
+    ivec2 scale;
+};
+
+void main() {
+    const ivec2 coord = ivec2(gl_FragCoord.xy) - dst_offset + src_offset;
+    const ivec2 msaa_coord = coord / scale;
+    const ivec2 sample_offset = coord % scale;
+    const int sample_id = sample_offset.x + scale.x * sample_offset.y;
+    gl_FragDepth = texelFetch(depth_tex, msaa_coord, sample_id).r;
+    gl_FragStencilRefARB = int(texelFetch(stencil_tex, msaa_coord, sample_id).r);
+}

src/video_core/host_shaders/convert_non_msaa_to_msaa.frag

@@ -3,7 +3,14 @@
 
 #version 450 core
 
-layout(binding = 0) uniform sampler2D img_in;
+#ifndef SAMPLER_TYPE
+#define SAMPLER_TYPE sampler2D
+#endif
+#ifndef TEXEL_TYPE
+#define TEXEL_TYPE vec4
+#endif
+
+layout(binding = 0) uniform SAMPLER_TYPE img_in;
 
 layout(push_constant) uniform PushConstants {
     ivec2 dst_offset;
@@ -11,7 +18,7 @@ layout(push_constant) uniform PushConstants {
     ivec2 scale;
 };
 
-layout(location = 0) out vec4 frag_color;
+layout(location = 0) out TEXEL_TYPE frag_color;
 
 void main() {
     const ivec2 msaa_coord = ivec2(gl_FragCoord.xy) - dst_offset;

src/video_core/host_shaders/convert_non_msaa_to_msaa_depth.frag

@@ -0,0 +1,19 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#version 450 core
+
+layout(binding = 0) uniform sampler2D img_in;
+
+layout(push_constant) uniform PushConstants {
+    ivec2 dst_offset;
+    ivec2 src_offset;
+    ivec2 scale;
+};
+
+void main() {
+    const ivec2 msaa_coord = ivec2(gl_FragCoord.xy) - dst_offset;
+    const ivec2 sample_offset = ivec2(gl_SampleID % scale.x, gl_SampleID / scale.x);
+    const ivec2 coord = msaa_coord * scale + sample_offset + src_offset;
+    gl_FragDepth = texelFetch(img_in, coord, 0).r;
+}

src/video_core/host_shaders/convert_non_msaa_to_msaa_depth_stencil.frag

@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#version 450 core
+#extension GL_ARB_shader_stencil_export : require
+
+layout(binding = 0) uniform sampler2D depth_tex;
+layout(binding = 1) uniform usampler2D stencil_tex;
+
+layout(push_constant) uniform PushConstants {
+    ivec2 dst_offset;
+    ivec2 src_offset;
+    ivec2 scale;
+};
+
+void main() {
+    const ivec2 msaa_coord = ivec2(gl_FragCoord.xy) - dst_offset;
+    const ivec2 sample_offset = ivec2(gl_SampleID % scale.x, gl_SampleID / scale.x);
+    const ivec2 coord = msaa_coord * scale + sample_offset + src_offset;
+    gl_FragDepth = texelFetch(depth_tex, coord, 0).r;
+    gl_FragStencilRefARB = int(texelFetch(stencil_tex, coord, 0).r);
+}