[vk, ogl/IR, dynarmic/IR] friendlier IR identity pointer chasing, inline AA passes (#3250)

Only merge once we find out wth is causing the crashes [we know it's fsr]

This reverts commit 82f9d489e7916318b56dac59fc52efc46fe0df7e.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3250
Reviewed-by: DraVee <dravee@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

2 files changed, 21 insertions, 18 deletions

diff --git a/src/video_core/renderer_opengl/present/layer.cpp b/src/video_core/renderer_opengl/present/layer.cpp
index 6c7092d229..d0a8e1186e 100644
--- a/src/video_core/renderer_opengl/present/layer.cpp
+++ b/src/video_core/renderer_opengl/present/layer.cpp
@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -54,12 +57,12 @@ GLuint Layer::ConfigureDraw(std::array<GLfloat, 3 * 2>& out_matrix,
         switch (anti_aliasing) {
         case Settings::AntiAliasing::Fxaa:
             CreateFXAA();
-            texture = fxaa->Draw(program_manager, info.display_texture);
+            texture = std::get<FXAA>(anti_alias).Draw(program_manager, info.display_texture);
             break;
         case Settings::AntiAliasing::Smaa:
         default:
             CreateSMAA();
-            texture = smaa->Draw(program_manager, info.display_texture);
+            texture = std::get<SMAA>(anti_alias).Draw(program_manager, info.display_texture);
             break;
         }
     }
@@ -68,7 +71,7 @@ GLuint Layer::ConfigureDraw(std::array<GLfloat, 3 * 2>& out_matrix,
 
     if (filters.get_scaling_filter() == Settings::ScalingFilter::Fsr) {
         if (!fsr || fsr->NeedsRecreation(layout.screen)) {
-            fsr = std::make_unique<FSR>(layout.screen.GetWidth(), layout.screen.GetHeight());
+            fsr.emplace(layout.screen.GetWidth(), layout.screen.GetHeight());
         }
 
         texture = fsr->Draw(program_manager, texture, info.scaled_width, info.scaled_height, crop);
@@ -199,23 +202,20 @@ void Layer::ConfigureFramebufferTexture(const Tegra::FramebufferConfig& framebuf
     glTextureStorage2D(framebuffer_texture.resource.handle, 1, internal_format,
                        framebuffer_texture.width, framebuffer_texture.height);
 
-    fxaa.reset();
-    smaa.reset();
+    anti_alias.emplace<std::monostate>();
 }
 
 void Layer::CreateFXAA() {
-    smaa.reset();
-    if (!fxaa) {
-        fxaa = std::make_unique<FXAA>(
+    if (!std::holds_alternative<FXAA>(anti_alias)) {
+        anti_alias.emplace<FXAA>(
             Settings::values.resolution_info.ScaleUp(framebuffer_texture.width),
             Settings::values.resolution_info.ScaleUp(framebuffer_texture.height));
     }
 }
 
 void Layer::CreateSMAA() {
-    fxaa.reset();
-    if (!smaa) {
-        smaa = std::make_unique<SMAA>(
+    if (!std::holds_alternative<SMAA>(anti_alias)) {
+        anti_alias.emplace<SMAA>(
             Settings::values.resolution_info.ScaleUp(framebuffer_texture.width),
             Settings::values.resolution_info.ScaleUp(framebuffer_texture.height));
     }
diff --git a/src/video_core/renderer_opengl/present/layer.h b/src/video_core/renderer_opengl/present/layer.h
index 5b15b730fc..7a58aee7b9 100644
--- a/src/video_core/renderer_opengl/present/layer.h
+++ b/src/video_core/renderer_opengl/present/layer.h
@@ -1,13 +1,20 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #pragma once
 
-#include <memory>
+#include <optional>
+#include <variant>
 #include <vector>
 
 #include "video_core/host1x/gpu_device_memory_manager.h"
 #include "video_core/renderer_opengl/gl_resource_manager.h"
+#include "video_core/renderer_opengl/present/smaa.h"
+#include "video_core/renderer_opengl/present/fxaa.h"
+#include "video_core/renderer_opengl/present/fsr.h"
 
 namespace Layout {
 struct FramebufferLayout;
@@ -26,11 +33,8 @@ struct FramebufferConfig;
 namespace OpenGL {
 
 struct FramebufferTextureInfo;
-class FSR;
-class FXAA;
 class ProgramManager;
 class RasterizerOpenGL;
-class SMAA;
 
 /// Structure used for storing information about the textures for the Switch screen
 struct TextureInfo {
@@ -76,9 +80,8 @@ private:
     /// Display information for Switch screen
     TextureInfo framebuffer_texture;
 
-    std::unique_ptr<FSR> fsr;
-    std::unique_ptr<FXAA> fxaa;
-    std::unique_ptr<SMAA> smaa;
+    std::optional<FSR> fsr;
+    std::variant<std::monostate, FXAA, SMAA> anti_alias;
 };
 
 } // namespace OpenGL