[vk, ogl/IR, dynarmic/IR] friendlier IR identity pointer chasing, inline AA passes (#3250)

Only merge once we find out wth is causing the crashes [we know it's fsr]

This reverts commit 82f9d489e7916318b56dac59fc52efc46fe0df7e.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3250
Reviewed-by: DraVee <dravee@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

1 files changed, 18 insertions, 15 deletions

diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp
index 5676dfe62a..b462c672cc 100644
--- a/src/video_core/renderer_vulkan/present/layer.cpp
+++ b/src/video_core/renderer_vulkan/present/layer.cpp
@@ -1,10 +1,15 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include <variant>
 #include "video_core/present.h"
+#include "video_core/renderer_vulkan/present/anti_alias_pass.h"
+/* X11 defines */
+#undef Success
+#undef BadValue
 #include "video_core/renderer_vulkan/vk_rasterizer.h"
 
 #include "common/settings.h"
@@ -58,7 +63,7 @@ Layer::Layer(const Device& device_, MemoryAllocator& memory_allocator_, Schedule
     CreateDescriptorPool();
     CreateDescriptorSets(layout);
     if (filters.get_scaling_filter() == Settings::ScalingFilter::Fsr) {
-        CreateFSR(output_size);
+        fsr.emplace(device, memory_allocator, image_count, output_size);
     }
 }
 
@@ -97,7 +102,11 @@ void Layer::ConfigureDraw(PresentPushConstants* out_push_constants,
     VkImageView source_image_view =
         texture_info ? texture_info->image_view : *raw_image_views[image_index];
 
-    anti_alias->Draw(scheduler, image_index, &source_image, &source_image_view);
+    if (auto* fxaa = std::get_if<FXAA>(&anti_alias)) {
+        fxaa->Draw(scheduler, image_index, &source_image, &source_image_view);
+    } else if (auto* smaa = std::get_if<SMAA>(&anti_alias)) {
+        smaa->Draw(scheduler, image_index, &source_image, &source_image_view);
+    }
 
     auto crop_rect = Tegra::NormalizeCrop(framebuffer, texture_width, texture_height);
     const VkExtent2D render_extent{
@@ -106,8 +115,7 @@ void Layer::ConfigureDraw(PresentPushConstants* out_push_constants,
     };
 
     if (fsr) {
-        source_image_view = fsr->Draw(scheduler, image_index, source_image, source_image_view,
-                                      render_extent, crop_rect);
+        source_image_view = fsr->Draw(scheduler, image_index, source_image, source_image_view, render_extent, crop_rect);
         crop_rect = {0, 0, 1, 1};
     }
 
@@ -156,10 +164,6 @@ void Layer::CreateRawImages(const Tegra::FramebufferConfig& framebuffer) {
     }
 }
 
-void Layer::CreateFSR(VkExtent2D output_size) {
-    fsr = std::make_unique<FSR>(device, memory_allocator, image_count, output_size);
-}
-
 void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
     if (framebuffer.width == raw_width && framebuffer.height == raw_height &&
         framebuffer.pixel_format == pixel_format && !raw_images.empty()) {
@@ -169,7 +173,7 @@ void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
     raw_width = framebuffer.width;
     raw_height = framebuffer.height;
     pixel_format = framebuffer.pixel_format;
-    anti_alias.reset();
+    anti_alias.emplace<std::monostate>();
 
     ReleaseRawImages();
     CreateStagingBuffer(framebuffer);
@@ -177,9 +181,8 @@ void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
 }
 
 void Layer::SetAntiAliasPass() {
-    if (anti_alias && anti_alias_setting == filters.get_anti_aliasing()) {
+    if (!std::holds_alternative<std::monostate>(anti_alias) && anti_alias_setting == filters.get_anti_aliasing())
         return;
-    }
 
     anti_alias_setting = filters.get_anti_aliasing();
 
@@ -190,13 +193,13 @@ void Layer::SetAntiAliasPass() {
 
     switch (anti_alias_setting) {
     case Settings::AntiAliasing::Fxaa:
-        anti_alias = std::make_unique<FXAA>(device, memory_allocator, image_count, render_area);
+        anti_alias.emplace<FXAA>(device, memory_allocator, image_count, render_area);
         break;
     case Settings::AntiAliasing::Smaa:
-        anti_alias = std::make_unique<SMAA>(device, memory_allocator, image_count, render_area);
+        anti_alias.emplace<SMAA>(device, memory_allocator, image_count, render_area);
         break;
     default:
-        anti_alias = std::make_unique<NoAA>();
+        anti_alias.emplace<std::monostate>();
         break;
     }
 }