[vk, ogl/IR, dynarmic/IR] friendlier IR identity pointer chasing, inline AA passes (#3250)

Only merge once we find out wth is causing the crashes [we know it's fsr]

This reverts commit 82f9d489e7916318b56dac59fc52efc46fe0df7e.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3250
Reviewed-by: DraVee <dravee@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

5 files changed, 70 insertions, 86 deletions

diff --git a/src/video_core/renderer_vulkan/present/anti_alias_pass.h b/src/video_core/renderer_vulkan/present/anti_alias_pass.h
index 1f20fbd7f0..4990e87502 100644
--- a/src/video_core/renderer_vulkan/present/anti_alias_pass.h
+++ b/src/video_core/renderer_vulkan/present/anti_alias_pass.h
@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -16,10 +19,4 @@ public:
                       VkImageView* inout_image_view) = 0;
 };
 
-class NoAA final : public AntiAliasPass {
-public:
-    void Draw(Scheduler& scheduler, size_t image_index, VkImage* inout_image,
-              VkImageView* inout_image_view) override {}
-};
-
 } // namespace Vulkan
diff --git a/src/video_core/renderer_vulkan/present/filters.cpp b/src/video_core/renderer_vulkan/present/filters.cpp
index 0a28ea6349..6203b41218 100644
--- a/src/video_core/renderer_vulkan/present/filters.cpp
+++ b/src/video_core/renderer_vulkan/present/filters.cpp
@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2024 Torzu Emulator Project
@@ -30,32 +30,19 @@
 
 namespace Vulkan {
 
-namespace {
-
-vk::ShaderModule SelectScaleForceShader(const Device& device) {
-    if (device.IsFloat16Supported()) {
-        return BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP16_FRAG_SPV);
-    } else {
-        return BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP32_FRAG_SPV);
-    }
-}
-
-} // Anonymous namespace
-
 std::unique_ptr<WindowAdaptPass> MakeNearestNeighbor(const Device& device, VkFormat frame_format) {
-    return std::make_unique<WindowAdaptPass>(device, frame_format,
-                                             CreateNearestNeighborSampler(device),
-                                             BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
+    return std::make_unique<WindowAdaptPass>(device, frame_format, CreateNearestNeighborSampler(device),
+        BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeBilinear(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
+        BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeSpline1(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             BuildShader(device, PRESENT_SPLINE1_FRAG_SPV));
+        BuildShader(device, PRESENT_SPLINE1_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeBicubic(const Device& device, VkFormat frame_format, VkCubicFilterWeightsQCOM qcom_weights) {
@@ -84,22 +71,26 @@ std::unique_ptr<WindowAdaptPass> MakeBicubic(const Device& device, VkFormat fram
 
 std::unique_ptr<WindowAdaptPass> MakeGaussian(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             BuildShader(device, PRESENT_GAUSSIAN_FRAG_SPV));
+        BuildShader(device, PRESENT_GAUSSIAN_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeLanczos(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             BuildShader(device, PRESENT_LANCZOS_FRAG_SPV));
+        BuildShader(device, PRESENT_LANCZOS_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeScaleForce(const Device& device, VkFormat frame_format) {
-    return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             SelectScaleForceShader(device));
+    auto const select_fn = [&]() {
+        return device.IsFloat16Supported()
+            ? BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP16_FRAG_SPV)
+            : BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP32_FRAG_SPV);
+    };
+    return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device), select_fn());
 }
 
 std::unique_ptr<WindowAdaptPass> MakeArea(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             BuildShader(device, PRESENT_AREA_FRAG_SPV));
+        BuildShader(device, PRESENT_AREA_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeMmpx(const Device& device, VkFormat frame_format) {
diff --git a/src/video_core/renderer_vulkan/present/fsr.cpp b/src/video_core/renderer_vulkan/present/fsr.cpp
index 3f708be704..ba6252ed95 100644
--- a/src/video_core/renderer_vulkan/present/fsr.cpp
+++ b/src/video_core/renderer_vulkan/present/fsr.cpp
@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -41,25 +44,18 @@ FSR::FSR(const Device& device, MemoryAllocator& memory_allocator, size_t image_c
 void FSR::CreateImages() {
     m_dynamic_images.resize(m_image_count);
     for (auto& images : m_dynamic_images) {
-        images.images[Easu] =
-            CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
-        images.images[Rcas] =
-            CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
-        images.image_views[Easu] =
-            CreateWrappedImageView(m_device, images.images[Easu], VK_FORMAT_R16G16B16A16_SFLOAT);
-        images.image_views[Rcas] =
-            CreateWrappedImageView(m_device, images.images[Rcas], VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.images[Easu] = CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.images[Rcas] = CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.image_views[Easu] = CreateWrappedImageView(m_device, images.images[Easu], VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.image_views[Rcas] = CreateWrappedImageView(m_device, images.images[Rcas], VK_FORMAT_R16G16B16A16_SFLOAT);
     }
 }
 
 void FSR::CreateRenderPasses() {
     m_renderpass = CreateWrappedRenderPass(m_device, VK_FORMAT_R16G16B16A16_SFLOAT);
-
     for (auto& images : m_dynamic_images) {
-        images.framebuffers[Easu] =
-            CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Easu], m_extent);
-        images.framebuffers[Rcas] =
-            CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Rcas], m_extent);
+        images.framebuffers[Easu] = CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Easu], m_extent);
+        images.framebuffers[Rcas] = CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Rcas], m_extent);
     }
 }
 
@@ -87,16 +83,13 @@ void FSR::CreateDescriptorPool() {
 }
 
 void FSR::CreateDescriptorSetLayout() {
-    m_descriptor_set_layout =
-        CreateWrappedDescriptorSetLayout(m_device, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER});
+    m_descriptor_set_layout = CreateWrappedDescriptorSetLayout(m_device, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER});
 }
 
 void FSR::CreateDescriptorSets() {
     std::vector<VkDescriptorSetLayout> layouts(MaxFsrStage, *m_descriptor_set_layout);
-
-    for (auto& images : m_dynamic_images) {
+    for (auto& images : m_dynamic_images)
         images.descriptor_sets = CreateWrappedDescriptorSets(m_descriptor_pool, layouts);
-    }
 }
 
 void FSR::CreatePipelineLayouts() {
@@ -128,31 +121,25 @@ void FSR::CreatePipelines() {
 void FSR::UpdateDescriptorSets(VkImageView image_view, size_t image_index) {
     Images& images = m_dynamic_images[image_index];
     std::vector<VkDescriptorImageInfo> image_infos;
-    std::vector<VkWriteDescriptorSet> updates;
     image_infos.reserve(2);
-
-    updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, image_view,
-                                               images.descriptor_sets[Easu], 0));
-    updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, *images.image_views[Easu],
-                                               images.descriptor_sets[Rcas], 0));
-
+    std::vector<VkWriteDescriptorSet> updates{
+        CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, images.descriptor_sets[Easu], 0),
+        CreateWriteDescriptorSet(image_infos, *m_sampler, *images.image_views[Easu], images.descriptor_sets[Rcas], 0)
+    };
     m_device.GetLogical().UpdateDescriptorSets(updates, {});
 }
 
 void FSR::UploadImages(Scheduler& scheduler) {
-    if (m_images_ready) {
-        return;
+    if (!m_images_ready) {
+        m_images_ready = true;
+        scheduler.Record([&](vk::CommandBuffer cmdbuf) {
+            for (auto& image : m_dynamic_images) {
+                ClearColorImage(cmdbuf, *image.images[Easu]);
+                ClearColorImage(cmdbuf, *image.images[Rcas]);
+            }
+        });
+        scheduler.Finish();
     }
-
-    scheduler.Record([&](vk::CommandBuffer cmdbuf) {
-        for (auto& image : m_dynamic_images) {
-            ClearColorImage(cmdbuf, *image.images[Easu]);
-            ClearColorImage(cmdbuf, *image.images[Rcas]);
-        }
-    });
-    scheduler.Finish();
-
-    m_images_ready = true;
 }
 
 VkImageView FSR::Draw(Scheduler& scheduler, size_t image_index, VkImage source_image,
diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp
index 5676dfe62a..b462c672cc 100644
--- a/src/video_core/renderer_vulkan/present/layer.cpp
+++ b/src/video_core/renderer_vulkan/present/layer.cpp
@@ -1,10 +1,15 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include <variant>
 #include "video_core/present.h"
+#include "video_core/renderer_vulkan/present/anti_alias_pass.h"
+/* X11 defines */
+#undef Success
+#undef BadValue
 #include "video_core/renderer_vulkan/vk_rasterizer.h"
 
 #include "common/settings.h"
@@ -58,7 +63,7 @@ Layer::Layer(const Device& device_, MemoryAllocator& memory_allocator_, Schedule
     CreateDescriptorPool();
     CreateDescriptorSets(layout);
     if (filters.get_scaling_filter() == Settings::ScalingFilter::Fsr) {
-        CreateFSR(output_size);
+        fsr.emplace(device, memory_allocator, image_count, output_size);
     }
 }
 
@@ -97,7 +102,11 @@ void Layer::ConfigureDraw(PresentPushConstants* out_push_constants,
     VkImageView source_image_view =
         texture_info ? texture_info->image_view : *raw_image_views[image_index];
 
-    anti_alias->Draw(scheduler, image_index, &source_image, &source_image_view);
+    if (auto* fxaa = std::get_if<FXAA>(&anti_alias)) {
+        fxaa->Draw(scheduler, image_index, &source_image, &source_image_view);
+    } else if (auto* smaa = std::get_if<SMAA>(&anti_alias)) {
+        smaa->Draw(scheduler, image_index, &source_image, &source_image_view);
+    }
 
     auto crop_rect = Tegra::NormalizeCrop(framebuffer, texture_width, texture_height);
     const VkExtent2D render_extent{
@@ -106,8 +115,7 @@ void Layer::ConfigureDraw(PresentPushConstants* out_push_constants,
     };
 
     if (fsr) {
-        source_image_view = fsr->Draw(scheduler, image_index, source_image, source_image_view,
-                                      render_extent, crop_rect);
+        source_image_view = fsr->Draw(scheduler, image_index, source_image, source_image_view, render_extent, crop_rect);
         crop_rect = {0, 0, 1, 1};
     }
 
@@ -156,10 +164,6 @@ void Layer::CreateRawImages(const Tegra::FramebufferConfig& framebuffer) {
     }
 }
 
-void Layer::CreateFSR(VkExtent2D output_size) {
-    fsr = std::make_unique<FSR>(device, memory_allocator, image_count, output_size);
-}
-
 void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
     if (framebuffer.width == raw_width && framebuffer.height == raw_height &&
         framebuffer.pixel_format == pixel_format && !raw_images.empty()) {
@@ -169,7 +173,7 @@ void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
     raw_width = framebuffer.width;
     raw_height = framebuffer.height;
     pixel_format = framebuffer.pixel_format;
-    anti_alias.reset();
+    anti_alias.emplace<std::monostate>();
 
     ReleaseRawImages();
     CreateStagingBuffer(framebuffer);
@@ -177,9 +181,8 @@ void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
 }
 
 void Layer::SetAntiAliasPass() {
-    if (anti_alias && anti_alias_setting == filters.get_anti_aliasing()) {
+    if (!std::holds_alternative<std::monostate>(anti_alias) && anti_alias_setting == filters.get_anti_aliasing())
         return;
-    }
 
     anti_alias_setting = filters.get_anti_aliasing();
 
@@ -190,13 +193,13 @@ void Layer::SetAntiAliasPass() {
 
     switch (anti_alias_setting) {
     case Settings::AntiAliasing::Fxaa:
-        anti_alias = std::make_unique<FXAA>(device, memory_allocator, image_count, render_area);
+        anti_alias.emplace<FXAA>(device, memory_allocator, image_count, render_area);
         break;
     case Settings::AntiAliasing::Smaa:
-        anti_alias = std::make_unique<SMAA>(device, memory_allocator, image_count, render_area);
+        anti_alias.emplace<SMAA>(device, memory_allocator, image_count, render_area);
         break;
     default:
-        anti_alias = std::make_unique<NoAA>();
+        anti_alias.emplace<std::monostate>();
         break;
     }
 }
diff --git a/src/video_core/renderer_vulkan/present/layer.h b/src/video_core/renderer_vulkan/present/layer.h
index f5effdcd7f..d38b81823e 100644
--- a/src/video_core/renderer_vulkan/present/layer.h
+++ b/src/video_core/renderer_vulkan/present/layer.h
@@ -1,11 +1,20 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #pragma once
 
+#include <optional>
+#include <variant>
+
 #include "common/math_util.h"
 #include "video_core/host1x/gpu_device_memory_manager.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
+#include "video_core/renderer_vulkan/present/fsr.h"
+#include "video_core/renderer_vulkan/present/fxaa.h"
+#include "video_core/renderer_vulkan/present/smaa.h"
 
 namespace Layout {
 struct FramebufferLayout;
@@ -29,7 +38,6 @@ namespace Vulkan {
 
 class AntiAliasPass;
 class Device;
-class FSR;
 class MemoryAllocator;
 struct PresentPushConstants;
 class RasterizerVulkan;
@@ -54,7 +62,6 @@ private:
     void CreateDescriptorSets(VkDescriptorSetLayout layout);
     void CreateStagingBuffer(const Tegra::FramebufferConfig& framebuffer);
     void CreateRawImages(const Tegra::FramebufferConfig& framebuffer);
-    void CreateFSR(VkExtent2D output_size);
 
     void RefreshResources(const Tegra::FramebufferConfig& framebuffer);
     void SetAntiAliasPass();
@@ -87,9 +94,8 @@ private:
     Service::android::PixelFormat pixel_format{};
 
     Settings::AntiAliasing anti_alias_setting{};
-    std::unique_ptr<AntiAliasPass> anti_alias{};
-
-    std::unique_ptr<FSR> fsr{};
+    std::variant<std::monostate, FXAA, SMAA> anti_alias{};
+    std::optional<FSR> fsr{};
     std::vector<u64> resource_ticks{};
 };