[vulkan] Adjustments on MSAA and BlitHelpers (#4287)

PR/Commit Owner: CamilleLaVey

This PR contains changes complementary to the MSAA refactor from some weeks ago, adds proper shader convert to depth, stencil for msaa and non msaa convertions; removes redundant helpers and unify paths on the resolve functions, just to make the readability and maintainability better; includes fixes for the blit operations on Nvidia, Intel/Windows (previously banned) and ensures Linux retain the fix without extra burden; fixes the QCOM driver bug on resolution upscaling on any games above x1 (includes turnip on A8xx series) and fixes crashes/ resolution upscaled bugs on certain games that used to have wrong viewporting effect on screen or directly device loss on Vulkan. Adds fixes for regressions caused by previous MSAA refactor on games as Fire Emblem: Engage.

Special Thanks

1.- Big Smolio (@Gidoly)

Co-authored-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4287
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-by: Samuel <lizzie@eden-emu.dev>

10 files changed, 160 insertions, 7 deletions

diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt
index 5b8f47a7a6..c4e7ac3f6f 100644
--- a/src/video_core/host_shaders/CMakeLists.txt
+++ b/src/video_core/host_shaders/CMakeLists.txt
@@ -18,6 +18,7 @@ set(SHADER_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_float.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d.comp
     ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_msaa.frag
+    ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_msaa.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_stencil_msaa.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d.comp
@@ -30,8 +31,12 @@ set(SHADER_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_float_to_depth.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_msaa_to_non_msaa.comp
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_msaa_to_non_msaa.frag
+    ${CMAKE_CURRENT_SOURCE_DIR}/convert_msaa_to_non_msaa_depth.frag
+    ${CMAKE_CURRENT_SOURCE_DIR}/convert_msaa_to_non_msaa_depth_stencil.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa.comp
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa.frag
+    ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa_depth.frag
+    ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa_depth_stencil.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/convert_s8d24_to_abgr8.frag
     ${CMAKE_CURRENT_SOURCE_DIR}/full_screen_triangle.vert
     ${CMAKE_CURRENT_SOURCE_DIR}/fxaa.frag
@@ -163,6 +168,44 @@ foreach(SOURCE_FILE IN ITEMS ${SHADER_FILES})
     endif()
 endforeach()
 
+# Integer variants of the MSAA conversion shaders. They only differ from the float
+# source in the sampler and output types, so they are generated from it via defines.
+set(SHADER_TYPE_VARIANTS
+    "convert_msaa_to_non_msaa.frag|sint|isampler2DMS|ivec4"
+    "convert_msaa_to_non_msaa.frag|uint|usampler2DMS|uvec4"
+    "convert_non_msaa_to_msaa.frag|sint|isampler2D|ivec4"
+    "convert_non_msaa_to_msaa.frag|uint|usampler2D|uvec4"
+)
+
+foreach(VARIANT IN ITEMS ${SHADER_TYPE_VARIANTS})
+    string(REPLACE "|" ";" VARIANT_PARTS ${VARIANT})
+    list(GET VARIANT_PARTS 0 VARIANT_FILENAME)
+    list(GET VARIANT_PARTS 1 VARIANT_SUFFIX)
+    list(GET VARIANT_PARTS 2 VARIANT_SAMPLER)
+    list(GET VARIANT_PARTS 3 VARIANT_TEXEL)
+
+    set(VARIANT_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/${VARIANT_FILENAME})
+    get_filename_component(VARIANT_STEM ${VARIANT_FILENAME} NAME_WE)
+    get_filename_component(VARIANT_EXT ${VARIANT_FILENAME} EXT)
+    string(REPLACE "." "" VARIANT_EXT ${VARIANT_EXT})
+    set(VARIANT_NAME ${VARIANT_STEM}_${VARIANT_SUFFIX}_${VARIANT_EXT})
+
+    string(TOUPPER ${VARIANT_NAME}_SPV VARIANT_VARIABLE_NAME)
+    set(VARIANT_HEADER_FILE ${SHADER_DIR}/${VARIANT_NAME}_spv.h)
+    add_custom_command(
+        OUTPUT
+            ${VARIANT_HEADER_FILE}
+        COMMAND
+            ${GLSLANGVALIDATOR} -V ${QUIET_FLAG} -I"${FIDELITYFX_INCLUDE_DIR}" ${GLSL_FLAGS}
+            -DSAMPLER_TYPE=${VARIANT_SAMPLER} -DTEXEL_TYPE=${VARIANT_TEXEL}
+            --variable-name ${VARIANT_VARIABLE_NAME} -o ${VARIANT_HEADER_FILE} ${VARIANT_SOURCE}
+            --target-env ${SPIR_V_VERSION}
+        MAIN_DEPENDENCY
+            ${VARIANT_SOURCE}
+    )
+    set(SHADER_HEADERS ${SHADER_HEADERS} ${VARIANT_HEADER_FILE})
+endforeach()
+
 foreach(FILEPATH IN ITEMS ${FIDELITYFX_FILES})
     get_filename_component(FILENAME ${FILEPATH} NAME)
     string(REPLACE "." "_" HEADER_NAME ${FILENAME})
diff --git a/src/video_core/host_shaders/blit_depth.frag b/src/video_core/host_shaders/blit_depth.frag
new file mode 100644
index 0000000000..9e19c767aa
--- /dev/null
+++ b/src/video_core/host_shaders/blit_depth.frag
@@ -0,0 +1,12 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#version 450 core
+
+layout(binding = 0) uniform sampler2D depth_tex;
+
+layout(location = 0) in vec2 texcoord;
+
+void main() {
+    gl_FragDepth = textureLod(depth_tex, texcoord, 0).r;
+}
diff --git a/src/video_core/host_shaders/blit_depth_msaa.frag b/src/video_core/host_shaders/blit_depth_msaa.frag
index 064289ad04..9e955d471e 100644
--- a/src/video_core/host_shaders/blit_depth_msaa.frag
+++ b/src/video_core/host_shaders/blit_depth_msaa.frag
@@ -8,5 +8,5 @@ layout(binding = 0) uniform sampler2DMS depth_tex;
 layout(location = 0) in vec2 texcoord;
 
 void main() {
-    gl_FragDepth = texelFetch(depth_tex, ivec2(texcoord), 0).r;
+    gl_FragDepth = texelFetch(depth_tex, ivec2(texcoord), gl_SampleID).r;
 }
diff --git a/src/video_core/host_shaders/blit_depth_stencil_msaa.frag b/src/video_core/host_shaders/blit_depth_stencil_msaa.frag
index d4264f2d14..7d4555bfd4 100644
--- a/src/video_core/host_shaders/blit_depth_stencil_msaa.frag
+++ b/src/video_core/host_shaders/blit_depth_stencil_msaa.frag
@@ -10,6 +10,6 @@ layout(binding = 1) uniform usampler2DMS stencil_tex;
 layout(location = 0) in vec2 texcoord;
 
 void main() {
-    gl_FragDepth = texelFetch(depth_tex, ivec2(texcoord), 0).r;
-    gl_FragStencilRefARB = int(texelFetch(stencil_tex, ivec2(texcoord), 0).r);
+    gl_FragDepth = texelFetch(depth_tex, ivec2(texcoord), gl_SampleID).r;
+    gl_FragStencilRefARB = int(texelFetch(stencil_tex, ivec2(texcoord), gl_SampleID).r);
 }
diff --git a/src/video_core/host_shaders/convert_msaa_to_non_msaa.frag b/src/video_core/host_shaders/convert_msaa_to_non_msaa.frag
index 6c4aac3c82..8f7ec34d1a 100644
--- a/src/video_core/host_shaders/convert_msaa_to_non_msaa.frag
+++ b/src/video_core/host_shaders/convert_msaa_to_non_msaa.frag
@@ -3,7 +3,14 @@
 
 #version 450 core
 
-layout(binding = 0) uniform sampler2DMS msaa_in;
+#ifndef SAMPLER_TYPE
+#define SAMPLER_TYPE sampler2DMS
+#endif
+#ifndef TEXEL_TYPE
+#define TEXEL_TYPE vec4
+#endif
+
+layout(binding = 0) uniform SAMPLER_TYPE msaa_in;
 
 layout(push_constant) uniform PushConstants {
     ivec2 dst_offset;
@@ -11,7 +18,7 @@ layout(push_constant) uniform PushConstants {
     ivec2 scale;
 };
 
-layout(location = 0) out vec4 frag_color;
+layout(location = 0) out TEXEL_TYPE frag_color;
 
 void main() {
     const ivec2 coord = ivec2(gl_FragCoord.xy) - dst_offset + src_offset;
diff --git a/src/video_core/host_shaders/convert_msaa_to_non_msaa_depth.frag b/src/video_core/host_shaders/convert_msaa_to_non_msaa_depth.frag
new file mode 100644
index 0000000000..26fdd48b36
--- /dev/null
+++ b/src/video_core/host_shaders/convert_msaa_to_non_msaa_depth.frag
@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#version 450 core
+
+layout(binding = 0) uniform sampler2DMS msaa_in;
+
+layout(push_constant) uniform PushConstants {
+    ivec2 dst_offset;
+    ivec2 src_offset;
+    ivec2 scale;
+};
+
+void main() {
+    const ivec2 coord = ivec2(gl_FragCoord.xy) - dst_offset + src_offset;
+    const ivec2 msaa_coord = coord / scale;
+    const ivec2 sample_offset = coord % scale;
+    const int sample_id = sample_offset.x + scale.x * sample_offset.y;
+    gl_FragDepth = texelFetch(msaa_in, msaa_coord, sample_id).r;
+}
diff --git a/src/video_core/host_shaders/convert_msaa_to_non_msaa_depth_stencil.frag b/src/video_core/host_shaders/convert_msaa_to_non_msaa_depth_stencil.frag
new file mode 100644
index 0000000000..cd34a1a025
--- /dev/null
+++ b/src/video_core/host_shaders/convert_msaa_to_non_msaa_depth_stencil.frag
@@ -0,0 +1,23 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#version 450 core
+#extension GL_ARB_shader_stencil_export : require
+
+layout(binding = 0) uniform sampler2DMS depth_tex;
+layout(binding = 1) uniform usampler2DMS stencil_tex;
+
+layout(push_constant) uniform PushConstants {
+    ivec2 dst_offset;
+    ivec2 src_offset;
+    ivec2 scale;
+};
+
+void main() {
+    const ivec2 coord = ivec2(gl_FragCoord.xy) - dst_offset + src_offset;
+    const ivec2 msaa_coord = coord / scale;
+    const ivec2 sample_offset = coord % scale;
+    const int sample_id = sample_offset.x + scale.x * sample_offset.y;
+    gl_FragDepth = texelFetch(depth_tex, msaa_coord, sample_id).r;
+    gl_FragStencilRefARB = int(texelFetch(stencil_tex, msaa_coord, sample_id).r);
+}
diff --git a/src/video_core/host_shaders/convert_non_msaa_to_msaa.frag b/src/video_core/host_shaders/convert_non_msaa_to_msaa.frag
index 19e23849d2..b5a93068a9 100644
--- a/src/video_core/host_shaders/convert_non_msaa_to_msaa.frag
+++ b/src/video_core/host_shaders/convert_non_msaa_to_msaa.frag
@@ -3,7 +3,14 @@
 
 #version 450 core
 
-layout(binding = 0) uniform sampler2D img_in;
+#ifndef SAMPLER_TYPE
+#define SAMPLER_TYPE sampler2D
+#endif
+#ifndef TEXEL_TYPE
+#define TEXEL_TYPE vec4
+#endif
+
+layout(binding = 0) uniform SAMPLER_TYPE img_in;
 
 layout(push_constant) uniform PushConstants {
     ivec2 dst_offset;
@@ -11,7 +18,7 @@ layout(push_constant) uniform PushConstants {
     ivec2 scale;
 };
 
-layout(location = 0) out vec4 frag_color;
+layout(location = 0) out TEXEL_TYPE frag_color;
 
 void main() {
     const ivec2 msaa_coord = ivec2(gl_FragCoord.xy) - dst_offset;
diff --git a/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth.frag b/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth.frag
new file mode 100644
index 0000000000..f883b3196f
--- /dev/null
+++ b/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth.frag
@@ -0,0 +1,19 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#version 450 core
+
+layout(binding = 0) uniform sampler2D img_in;
+
+layout(push_constant) uniform PushConstants {
+    ivec2 dst_offset;
+    ivec2 src_offset;
+    ivec2 scale;
+};
+
+void main() {
+    const ivec2 msaa_coord = ivec2(gl_FragCoord.xy) - dst_offset;
+    const ivec2 sample_offset = ivec2(gl_SampleID % scale.x, gl_SampleID / scale.x);
+    const ivec2 coord = msaa_coord * scale + sample_offset + src_offset;
+    gl_FragDepth = texelFetch(img_in, coord, 0).r;
+}
diff --git a/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth_stencil.frag b/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth_stencil.frag
new file mode 100644
index 0000000000..d9a308f5c3
--- /dev/null
+++ b/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth_stencil.frag
@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#version 450 core
+#extension GL_ARB_shader_stencil_export : require
+
+layout(binding = 0) uniform sampler2D depth_tex;
+layout(binding = 1) uniform usampler2D stencil_tex;
+
+layout(push_constant) uniform PushConstants {
+    ivec2 dst_offset;
+    ivec2 src_offset;
+    ivec2 scale;
+};
+
+void main() {
+    const ivec2 msaa_coord = ivec2(gl_FragCoord.xy) - dst_offset;
+    const ivec2 sample_offset = ivec2(gl_SampleID % scale.x, gl_SampleID / scale.x);
+    const ivec2 coord = msaa_coord * scale + sample_offset + src_offset;
+    gl_FragDepth = texelFetch(depth_tex, coord, 0).r;
+    gl_FragStencilRefARB = int(texelFetch(stencil_tex, coord, 0).r);
+}