[am, renderer_vulkan] Fix overlay darkening and SGSR black screen on applets (#4475)

- [x] I have read and followed the [Contribution Guidelines](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/CONTRIBUTING.md#code-contributions).
- [x] I have read and followed the [AI Policy](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/docs/policies/AI.md)
- [x] I have read and followed the [Coding Guidelines](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/docs/policies/Coding.md) to the best of my ability.

-------------------
- Overlay applet: use IsOverlayOpenLocked to pick the Z-index, so the darkening background is only layered above the game while the overlay is open.
- Vulkan: skip the SGSR pass for applet layers to avoid presenting a black frame.
- Partial revert fix crashes in games on UE with the overlay applet enabled.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4475
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-by: lizzie <lizzie@eden-emu.dev>

1 files changed, 8 insertions, 5 deletions

diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp
index 6d3e7721f6..9f4aaf390f 100644
--- a/src/video_core/renderer_vulkan/present/layer.cpp
+++ b/src/video_core/renderer_vulkan/present/layer.cpp
@@ -94,13 +94,13 @@ void Layer::ConfigureDraw(const Device& device, PresentPushConstants* out_push_c
     const u32 scaled_width = texture_info ? texture_info->scaled_width : texture_width;
     const u32 scaled_height = texture_info ? texture_info->scaled_height : texture_height;
     const bool use_accelerated = texture_info.has_value();
+    const bool is_applet =
+        (framebuffer.layer_stack_mask & Service::Nvnflinger::LayerStackBit(
+                                            Service::Nvnflinger::LayerStackId::Recording)) == 0;
 
     RefreshResources(device, framebuffer);
     SetAntiAliasPass(device);
 #ifdef HAS_RESHADE
-    const bool is_applet =
-        (framebuffer.layer_stack_mask & Service::Nvnflinger::LayerStackBit(
-                                            Service::Nvnflinger::LayerStackId::Recording)) == 0;
     SetPostProcessPass(device, is_applet);
 #endif
 
@@ -141,8 +141,11 @@ void Layer::ConfigureDraw(const Device& device, PresentPushConstants* out_push_c
         source_image_view = fsr->Draw(device, scheduler, image_index, source_image, source_image_view, render_extent, crop_rect);
         crop_rect = {0, 0, 1, 1};
     } else if (auto* sgsr = std::get_if<SGSR>(&sr_filter)) {
-        source_image_view = sgsr->Draw(device, scheduler, image_index, source_image, source_image_view, render_extent, crop_rect);
-        crop_rect = {0, 0, 1, 1};
+        if (!is_applet) {
+            source_image_view = sgsr->Draw(device, scheduler, image_index, source_image,
+                                           source_image_view, render_extent, crop_rect);
+            crop_rect = {0, 0, 1, 1};
+        }
     }
 
     SetMatrixData(device, *out_push_constants, layout);