Revert "[vk, ogl/IR, dynarmic/IR] friendlier IR identity pointer chasing, inline AA passes (#2565)" (#3249)

FUCK. Fixes crash on Linux and SteamDeck
.
This reverts commit 46b32b76886b5cd0cfc8dfee1a02c3e83fb1e7d7.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3249
Reviewed-by: DraVee <dravee@eden-emu.dev>
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

5 files changed, 83 insertions, 67 deletions

diff --git a/src/video_core/renderer_vulkan/present/anti_alias_pass.h b/src/video_core/renderer_vulkan/present/anti_alias_pass.h
index b7d3e2de15..1f20fbd7f0 100644
--- a/src/video_core/renderer_vulkan/present/anti_alias_pass.h
+++ b/src/video_core/renderer_vulkan/present/anti_alias_pass.h
@@ -1,6 +1,3 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
-// SPDX-License-Identifier: GPL-3.0-or-later
-
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -19,4 +16,10 @@ public:
                       VkImageView* inout_image_view) = 0;
 };
 
+class NoAA final : public AntiAliasPass {
+public:
+    void Draw(Scheduler& scheduler, size_t image_index, VkImage* inout_image,
+              VkImageView* inout_image_view) override {}
+};
+
 } // namespace Vulkan
diff --git a/src/video_core/renderer_vulkan/present/filters.cpp b/src/video_core/renderer_vulkan/present/filters.cpp
index b6145c4102..0a28ea6349 100644
--- a/src/video_core/renderer_vulkan/present/filters.cpp
+++ b/src/video_core/renderer_vulkan/present/filters.cpp
@@ -30,19 +30,32 @@
 
 namespace Vulkan {
 
+namespace {
+
+vk::ShaderModule SelectScaleForceShader(const Device& device) {
+    if (device.IsFloat16Supported()) {
+        return BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP16_FRAG_SPV);
+    } else {
+        return BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP32_FRAG_SPV);
+    }
+}
+
+} // Anonymous namespace
+
 std::unique_ptr<WindowAdaptPass> MakeNearestNeighbor(const Device& device, VkFormat frame_format) {
-    return std::make_unique<WindowAdaptPass>(device, frame_format, CreateNearestNeighborSampler(device),
-        BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
+    return std::make_unique<WindowAdaptPass>(device, frame_format,
+                                             CreateNearestNeighborSampler(device),
+                                             BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeBilinear(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-        BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
+                                             BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeSpline1(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-        BuildShader(device, PRESENT_SPLINE1_FRAG_SPV));
+                                             BuildShader(device, PRESENT_SPLINE1_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeBicubic(const Device& device, VkFormat frame_format, VkCubicFilterWeightsQCOM qcom_weights) {
@@ -71,26 +84,22 @@ std::unique_ptr<WindowAdaptPass> MakeBicubic(const Device& device, VkFormat fram
 
 std::unique_ptr<WindowAdaptPass> MakeGaussian(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-        BuildShader(device, PRESENT_GAUSSIAN_FRAG_SPV));
+                                             BuildShader(device, PRESENT_GAUSSIAN_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeLanczos(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-        BuildShader(device, PRESENT_LANCZOS_FRAG_SPV));
+                                             BuildShader(device, PRESENT_LANCZOS_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeScaleForce(const Device& device, VkFormat frame_format) {
-    auto const select_fn = [&]() {
-        return device.IsFloat16Supported()
-            ? BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP16_FRAG_SPV)
-            : BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP32_FRAG_SPV);
-    };
-    return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device), select_fn());
+    return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
+                                             SelectScaleForceShader(device));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeArea(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-        BuildShader(device, PRESENT_AREA_FRAG_SPV));
+                                             BuildShader(device, PRESENT_AREA_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeMmpx(const Device& device, VkFormat frame_format) {
diff --git a/src/video_core/renderer_vulkan/present/fsr.cpp b/src/video_core/renderer_vulkan/present/fsr.cpp
index 24fc61e813..3f708be704 100644
--- a/src/video_core/renderer_vulkan/present/fsr.cpp
+++ b/src/video_core/renderer_vulkan/present/fsr.cpp
@@ -1,6 +1,3 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
-// SPDX-License-Identifier: GPL-3.0-or-later
-
 // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -44,18 +41,25 @@ FSR::FSR(const Device& device, MemoryAllocator& memory_allocator, size_t image_c
 void FSR::CreateImages() {
     m_dynamic_images.resize(m_image_count);
     for (auto& images : m_dynamic_images) {
-        images.images[Easu] = CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
-        images.images[Rcas] = CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
-        images.image_views[Easu] = CreateWrappedImageView(m_device, images.images[Easu], VK_FORMAT_R16G16B16A16_SFLOAT);
-        images.image_views[Rcas] = CreateWrappedImageView(m_device, images.images[Rcas], VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.images[Easu] =
+            CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.images[Rcas] =
+            CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.image_views[Easu] =
+            CreateWrappedImageView(m_device, images.images[Easu], VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.image_views[Rcas] =
+            CreateWrappedImageView(m_device, images.images[Rcas], VK_FORMAT_R16G16B16A16_SFLOAT);
     }
 }
 
 void FSR::CreateRenderPasses() {
     m_renderpass = CreateWrappedRenderPass(m_device, VK_FORMAT_R16G16B16A16_SFLOAT);
+
     for (auto& images : m_dynamic_images) {
-        images.framebuffers[Easu] = CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Easu], m_extent);
-        images.framebuffers[Rcas] = CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Rcas], m_extent);
+        images.framebuffers[Easu] =
+            CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Easu], m_extent);
+        images.framebuffers[Rcas] =
+            CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Rcas], m_extent);
     }
 }
 
@@ -83,13 +87,16 @@ void FSR::CreateDescriptorPool() {
 }
 
 void FSR::CreateDescriptorSetLayout() {
-    m_descriptor_set_layout = CreateWrappedDescriptorSetLayout(m_device, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER});
+    m_descriptor_set_layout =
+        CreateWrappedDescriptorSetLayout(m_device, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER});
 }
 
 void FSR::CreateDescriptorSets() {
     std::vector<VkDescriptorSetLayout> layouts(MaxFsrStage, *m_descriptor_set_layout);
-    for (auto& images : m_dynamic_images)
+
+    for (auto& images : m_dynamic_images) {
         images.descriptor_sets = CreateWrappedDescriptorSets(m_descriptor_pool, layouts);
+    }
 }
 
 void FSR::CreatePipelineLayouts() {
@@ -121,24 +128,31 @@ void FSR::CreatePipelines() {
 void FSR::UpdateDescriptorSets(VkImageView image_view, size_t image_index) {
     Images& images = m_dynamic_images[image_index];
     std::vector<VkDescriptorImageInfo> image_infos;
-    std::vector<VkWriteDescriptorSet> updates{
-        CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, images.descriptor_sets[Easu], 0),
-        CreateWriteDescriptorSet(image_infos, *m_sampler, *images.image_views[Easu], images.descriptor_sets[Rcas], 0)
-    };
+    std::vector<VkWriteDescriptorSet> updates;
+    image_infos.reserve(2);
+
+    updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, image_view,
+                                               images.descriptor_sets[Easu], 0));
+    updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, *images.image_views[Easu],
+                                               images.descriptor_sets[Rcas], 0));
+
     m_device.GetLogical().UpdateDescriptorSets(updates, {});
 }
 
 void FSR::UploadImages(Scheduler& scheduler) {
-    if (!m_images_ready) {
-        m_images_ready = true;
-        scheduler.Record([&](vk::CommandBuffer cmdbuf) {
-            for (auto& image : m_dynamic_images) {
-                ClearColorImage(cmdbuf, *image.images[Easu]);
-                ClearColorImage(cmdbuf, *image.images[Rcas]);
-            }
-        });
-        scheduler.Finish();
+    if (m_images_ready) {
+        return;
     }
+
+    scheduler.Record([&](vk::CommandBuffer cmdbuf) {
+        for (auto& image : m_dynamic_images) {
+            ClearColorImage(cmdbuf, *image.images[Easu]);
+            ClearColorImage(cmdbuf, *image.images[Rcas]);
+        }
+    });
+    scheduler.Finish();
+
+    m_images_ready = true;
 }
 
 VkImageView FSR::Draw(Scheduler& scheduler, size_t image_index, VkImage source_image,
diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp
index 5bf190bc82..5676dfe62a 100644
--- a/src/video_core/renderer_vulkan/present/layer.cpp
+++ b/src/video_core/renderer_vulkan/present/layer.cpp
@@ -4,12 +4,7 @@
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include <variant>
 #include "video_core/present.h"
-#include "video_core/renderer_vulkan/present/anti_alias_pass.h"
-/* X11 defines */
-#undef Success
-#undef BadValue
 #include "video_core/renderer_vulkan/vk_rasterizer.h"
 
 #include "common/settings.h"
@@ -63,7 +58,7 @@ Layer::Layer(const Device& device_, MemoryAllocator& memory_allocator_, Schedule
     CreateDescriptorPool();
     CreateDescriptorSets(layout);
     if (filters.get_scaling_filter() == Settings::ScalingFilter::Fsr) {
-        fsr.emplace(device, memory_allocator, image_count, output_size);
+        CreateFSR(output_size);
     }
 }
 
@@ -102,11 +97,7 @@ void Layer::ConfigureDraw(PresentPushConstants* out_push_constants,
     VkImageView source_image_view =
         texture_info ? texture_info->image_view : *raw_image_views[image_index];
 
-    if (std::holds_alternative<FXAA>(anti_alias)) {
-        std::get<FXAA>(anti_alias).Draw(scheduler, image_index, &source_image, &source_image_view);
-    } else if (std::holds_alternative<SMAA>(anti_alias)) {
-        std::get<SMAA>(anti_alias).Draw(scheduler, image_index, &source_image, &source_image_view);
-    }
+    anti_alias->Draw(scheduler, image_index, &source_image, &source_image_view);
 
     auto crop_rect = Tegra::NormalizeCrop(framebuffer, texture_width, texture_height);
     const VkExtent2D render_extent{
@@ -165,6 +156,10 @@ void Layer::CreateRawImages(const Tegra::FramebufferConfig& framebuffer) {
     }
 }
 
+void Layer::CreateFSR(VkExtent2D output_size) {
+    fsr = std::make_unique<FSR>(device, memory_allocator, image_count, output_size);
+}
+
 void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
     if (framebuffer.width == raw_width && framebuffer.height == raw_height &&
         framebuffer.pixel_format == pixel_format && !raw_images.empty()) {
@@ -174,7 +169,7 @@ void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
     raw_width = framebuffer.width;
     raw_height = framebuffer.height;
     pixel_format = framebuffer.pixel_format;
-    anti_alias.emplace<std::monostate>();
+    anti_alias.reset();
 
     ReleaseRawImages();
     CreateStagingBuffer(framebuffer);
@@ -182,8 +177,9 @@ void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
 }
 
 void Layer::SetAntiAliasPass() {
-    if (!std::holds_alternative<std::monostate>(anti_alias) && anti_alias_setting == filters.get_anti_aliasing())
+    if (anti_alias && anti_alias_setting == filters.get_anti_aliasing()) {
         return;
+    }
 
     anti_alias_setting = filters.get_anti_aliasing();
 
@@ -194,13 +190,13 @@ void Layer::SetAntiAliasPass() {
 
     switch (anti_alias_setting) {
     case Settings::AntiAliasing::Fxaa:
-        anti_alias.emplace<FXAA>(device, memory_allocator, image_count, render_area);
+        anti_alias = std::make_unique<FXAA>(device, memory_allocator, image_count, render_area);
         break;
     case Settings::AntiAliasing::Smaa:
-        anti_alias.emplace<SMAA>(device, memory_allocator, image_count, render_area);
+        anti_alias = std::make_unique<SMAA>(device, memory_allocator, image_count, render_area);
         break;
     default:
-        anti_alias.emplace<std::monostate>();
+        anti_alias = std::make_unique<NoAA>();
         break;
     }
 }
diff --git a/src/video_core/renderer_vulkan/present/layer.h b/src/video_core/renderer_vulkan/present/layer.h
index 062b7792c4..f5effdcd7f 100644
--- a/src/video_core/renderer_vulkan/present/layer.h
+++ b/src/video_core/renderer_vulkan/present/layer.h
@@ -1,20 +1,11 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
-// SPDX-License-Identifier: GPL-3.0-or-later
-
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #pragma once
 
-#include <optional>
-#include <variant>
-
 #include "common/math_util.h"
 #include "video_core/host1x/gpu_device_memory_manager.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
-#include "video_core/renderer_vulkan/present/fsr.h"
-#include "video_core/renderer_vulkan/present/fxaa.h"
-#include "video_core/renderer_vulkan/present/smaa.h"
 
 namespace Layout {
 struct FramebufferLayout;
@@ -38,6 +29,7 @@ namespace Vulkan {
 
 class AntiAliasPass;
 class Device;
+class FSR;
 class MemoryAllocator;
 struct PresentPushConstants;
 class RasterizerVulkan;
@@ -62,6 +54,7 @@ private:
     void CreateDescriptorSets(VkDescriptorSetLayout layout);
     void CreateStagingBuffer(const Tegra::FramebufferConfig& framebuffer);
     void CreateRawImages(const Tegra::FramebufferConfig& framebuffer);
+    void CreateFSR(VkExtent2D output_size);
 
     void RefreshResources(const Tegra::FramebufferConfig& framebuffer);
     void SetAntiAliasPass();
@@ -94,8 +87,9 @@ private:
     Service::android::PixelFormat pixel_format{};
 
     Settings::AntiAliasing anti_alias_setting{};
-    std::variant<std::monostate, FXAA, SMAA> anti_alias{};
-    std::optional<FSR> fsr{};
+    std::unique_ptr<AntiAliasPass> anti_alias{};
+
+    std::unique_ptr<FSR> fsr{};
     std::vector<u64> resource_ticks{};
 };