[vk, ogl/IR, dynarmic/IR] friendlier IR identity pointer chasing, inline AA passes (#2565)

- use std::optional instead of std::unique_ptr for the Antialias (FXAA, etc) passes to avoid the extra deref
- use a pattern for deferencing the IR pointer chasing loop as suggested on the intel optimization manual
- this also removes std::vector<> overhead by using boost::container::small_vector<> (not a silver bullet but in the case of this function reduces access times)

Signed-off-by: lizzie <lizzie@eden-emu.dev>
Co-authored-by: Caio Oliveira <caiooliveirafarias0@gmail.com>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/2565
Reviewed-by: Maufeat <sahyno1996@gmail.com>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

7 files changed, 88 insertions, 101 deletions

diff --git a/src/video_core/renderer_opengl/present/layer.cpp b/src/video_core/renderer_opengl/present/layer.cpp
index 6c7092d229..f18600e692 100644
--- a/src/video_core/renderer_opengl/present/layer.cpp
+++ b/src/video_core/renderer_opengl/present/layer.cpp
@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -54,12 +57,12 @@ GLuint Layer::ConfigureDraw(std::array<GLfloat, 3 * 2>& out_matrix,
         switch (anti_aliasing) {
         case Settings::AntiAliasing::Fxaa:
             CreateFXAA();
-            texture = fxaa->Draw(program_manager, info.display_texture);
+            texture = std::get<FXAA>(anti_alias).Draw(program_manager, info.display_texture);
             break;
         case Settings::AntiAliasing::Smaa:
         default:
             CreateSMAA();
-            texture = smaa->Draw(program_manager, info.display_texture);
+            texture = std::get<SMAA>(anti_alias).Draw(program_manager, info.display_texture);
             break;
         }
     }
@@ -68,7 +71,7 @@ GLuint Layer::ConfigureDraw(std::array<GLfloat, 3 * 2>& out_matrix,
 
     if (filters.get_scaling_filter() == Settings::ScalingFilter::Fsr) {
         if (!fsr || fsr->NeedsRecreation(layout.screen)) {
-            fsr = std::make_unique<FSR>(layout.screen.GetWidth(), layout.screen.GetHeight());
+            fsr.emplace(layout.screen.GetWidth(), layout.screen.GetHeight());
         }
 
         texture = fsr->Draw(program_manager, texture, info.scaled_width, info.scaled_height, crop);
@@ -199,23 +202,20 @@ void Layer::ConfigureFramebufferTexture(const Tegra::FramebufferConfig& framebuf
     glTextureStorage2D(framebuffer_texture.resource.handle, 1, internal_format,
                        framebuffer_texture.width, framebuffer_texture.height);
 
-    fxaa.reset();
-    smaa.reset();
+    anti_alias.emplace<std::monostate>();
 }
 
 void Layer::CreateFXAA() {
-    smaa.reset();
-    if (!fxaa) {
-        fxaa = std::make_unique<FXAA>(
+    if (!std::holds_alternative<FXAA>(anti_alias)) {
+        anti_alias.emplace<FXAA>(
             Settings::values.resolution_info.ScaleUp(framebuffer_texture.width),
             Settings::values.resolution_info.ScaleUp(framebuffer_texture.height));
     }
 }
 
 void Layer::CreateSMAA() {
-    fxaa.reset();
-    if (!smaa) {
-        smaa = std::make_unique<SMAA>(
+    if (!std::holds_alternative<SMAA>(anti_alias)) {
+        anti_alias.emplace<SMAA>(
             Settings::values.resolution_info.ScaleUp(framebuffer_texture.width),
             Settings::values.resolution_info.ScaleUp(framebuffer_texture.height));
     }
diff --git a/src/video_core/renderer_opengl/present/layer.h b/src/video_core/renderer_opengl/present/layer.h
index 5b15b730fc..e09fd6f696 100644
--- a/src/video_core/renderer_opengl/present/layer.h
+++ b/src/video_core/renderer_opengl/present/layer.h
@@ -1,13 +1,20 @@
+// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #pragma once
 
-#include <memory>
+#include <optional>
+#include <variant>
 #include <vector>
 
 #include "video_core/host1x/gpu_device_memory_manager.h"
 #include "video_core/renderer_opengl/gl_resource_manager.h"
+#include "video_core/renderer_opengl/present/smaa.h"
+#include "video_core/renderer_opengl/present/fxaa.h"
+#include "video_core/renderer_opengl/present/fsr.h"
 
 namespace Layout {
 struct FramebufferLayout;
@@ -26,11 +33,8 @@ struct FramebufferConfig;
 namespace OpenGL {
 
 struct FramebufferTextureInfo;
-class FSR;
-class FXAA;
 class ProgramManager;
 class RasterizerOpenGL;
-class SMAA;
 
 /// Structure used for storing information about the textures for the Switch screen
 struct TextureInfo {
@@ -76,9 +80,8 @@ private:
     /// Display information for Switch screen
     TextureInfo framebuffer_texture;
 
-    std::unique_ptr<FSR> fsr;
-    std::unique_ptr<FXAA> fxaa;
-    std::unique_ptr<SMAA> smaa;
+    std::optional<FSR> fsr;
+    std::variant<std::monostate, FXAA, SMAA> anti_alias;
 };
 
 } // namespace OpenGL
diff --git a/src/video_core/renderer_vulkan/present/anti_alias_pass.h b/src/video_core/renderer_vulkan/present/anti_alias_pass.h
index 1f20fbd7f0..b7d3e2de15 100644
--- a/src/video_core/renderer_vulkan/present/anti_alias_pass.h
+++ b/src/video_core/renderer_vulkan/present/anti_alias_pass.h
@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -16,10 +19,4 @@ public:
                       VkImageView* inout_image_view) = 0;
 };
 
-class NoAA final : public AntiAliasPass {
-public:
-    void Draw(Scheduler& scheduler, size_t image_index, VkImage* inout_image,
-              VkImageView* inout_image_view) override {}
-};
-
 } // namespace Vulkan
diff --git a/src/video_core/renderer_vulkan/present/filters.cpp b/src/video_core/renderer_vulkan/present/filters.cpp
index 0a28ea6349..b6145c4102 100644
--- a/src/video_core/renderer_vulkan/present/filters.cpp
+++ b/src/video_core/renderer_vulkan/present/filters.cpp
@@ -30,32 +30,19 @@
 
 namespace Vulkan {
 
-namespace {
-
-vk::ShaderModule SelectScaleForceShader(const Device& device) {
-    if (device.IsFloat16Supported()) {
-        return BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP16_FRAG_SPV);
-    } else {
-        return BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP32_FRAG_SPV);
-    }
-}
-
-} // Anonymous namespace
-
 std::unique_ptr<WindowAdaptPass> MakeNearestNeighbor(const Device& device, VkFormat frame_format) {
-    return std::make_unique<WindowAdaptPass>(device, frame_format,
-                                             CreateNearestNeighborSampler(device),
-                                             BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
+    return std::make_unique<WindowAdaptPass>(device, frame_format, CreateNearestNeighborSampler(device),
+        BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeBilinear(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
+        BuildShader(device, VULKAN_PRESENT_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeSpline1(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             BuildShader(device, PRESENT_SPLINE1_FRAG_SPV));
+        BuildShader(device, PRESENT_SPLINE1_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeBicubic(const Device& device, VkFormat frame_format, VkCubicFilterWeightsQCOM qcom_weights) {
@@ -84,22 +71,26 @@ std::unique_ptr<WindowAdaptPass> MakeBicubic(const Device& device, VkFormat fram
 
 std::unique_ptr<WindowAdaptPass> MakeGaussian(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             BuildShader(device, PRESENT_GAUSSIAN_FRAG_SPV));
+        BuildShader(device, PRESENT_GAUSSIAN_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeLanczos(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             BuildShader(device, PRESENT_LANCZOS_FRAG_SPV));
+        BuildShader(device, PRESENT_LANCZOS_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeScaleForce(const Device& device, VkFormat frame_format) {
-    return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             SelectScaleForceShader(device));
+    auto const select_fn = [&]() {
+        return device.IsFloat16Supported()
+            ? BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP16_FRAG_SPV)
+            : BuildShader(device, VULKAN_PRESENT_SCALEFORCE_FP32_FRAG_SPV);
+    };
+    return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device), select_fn());
 }
 
 std::unique_ptr<WindowAdaptPass> MakeArea(const Device& device, VkFormat frame_format) {
     return std::make_unique<WindowAdaptPass>(device, frame_format, CreateBilinearSampler(device),
-                                             BuildShader(device, PRESENT_AREA_FRAG_SPV));
+        BuildShader(device, PRESENT_AREA_FRAG_SPV));
 }
 
 std::unique_ptr<WindowAdaptPass> MakeMmpx(const Device& device, VkFormat frame_format) {
diff --git a/src/video_core/renderer_vulkan/present/fsr.cpp b/src/video_core/renderer_vulkan/present/fsr.cpp
index 3f708be704..24fc61e813 100644
--- a/src/video_core/renderer_vulkan/present/fsr.cpp
+++ b/src/video_core/renderer_vulkan/present/fsr.cpp
@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -41,25 +44,18 @@ FSR::FSR(const Device& device, MemoryAllocator& memory_allocator, size_t image_c
 void FSR::CreateImages() {
     m_dynamic_images.resize(m_image_count);
     for (auto& images : m_dynamic_images) {
-        images.images[Easu] =
-            CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
-        images.images[Rcas] =
-            CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
-        images.image_views[Easu] =
-            CreateWrappedImageView(m_device, images.images[Easu], VK_FORMAT_R16G16B16A16_SFLOAT);
-        images.image_views[Rcas] =
-            CreateWrappedImageView(m_device, images.images[Rcas], VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.images[Easu] = CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.images[Rcas] = CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.image_views[Easu] = CreateWrappedImageView(m_device, images.images[Easu], VK_FORMAT_R16G16B16A16_SFLOAT);
+        images.image_views[Rcas] = CreateWrappedImageView(m_device, images.images[Rcas], VK_FORMAT_R16G16B16A16_SFLOAT);
     }
 }
 
 void FSR::CreateRenderPasses() {
     m_renderpass = CreateWrappedRenderPass(m_device, VK_FORMAT_R16G16B16A16_SFLOAT);
-
     for (auto& images : m_dynamic_images) {
-        images.framebuffers[Easu] =
-            CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Easu], m_extent);
-        images.framebuffers[Rcas] =
-            CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Rcas], m_extent);
+        images.framebuffers[Easu] = CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Easu], m_extent);
+        images.framebuffers[Rcas] = CreateWrappedFramebuffer(m_device, m_renderpass, images.image_views[Rcas], m_extent);
     }
 }
 
@@ -87,16 +83,13 @@ void FSR::CreateDescriptorPool() {
 }
 
 void FSR::CreateDescriptorSetLayout() {
-    m_descriptor_set_layout =
-        CreateWrappedDescriptorSetLayout(m_device, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER});
+    m_descriptor_set_layout = CreateWrappedDescriptorSetLayout(m_device, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER});
 }
 
 void FSR::CreateDescriptorSets() {
     std::vector<VkDescriptorSetLayout> layouts(MaxFsrStage, *m_descriptor_set_layout);
-
-    for (auto& images : m_dynamic_images) {
+    for (auto& images : m_dynamic_images)
         images.descriptor_sets = CreateWrappedDescriptorSets(m_descriptor_pool, layouts);
-    }
 }
 
 void FSR::CreatePipelineLayouts() {
@@ -128,31 +121,24 @@ void FSR::CreatePipelines() {
 void FSR::UpdateDescriptorSets(VkImageView image_view, size_t image_index) {
     Images& images = m_dynamic_images[image_index];
     std::vector<VkDescriptorImageInfo> image_infos;
-    std::vector<VkWriteDescriptorSet> updates;
-    image_infos.reserve(2);
-
-    updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, image_view,
-                                               images.descriptor_sets[Easu], 0));
-    updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, *images.image_views[Easu],
-                                               images.descriptor_sets[Rcas], 0));
-
+    std::vector<VkWriteDescriptorSet> updates{
+        CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, images.descriptor_sets[Easu], 0),
+        CreateWriteDescriptorSet(image_infos, *m_sampler, *images.image_views[Easu], images.descriptor_sets[Rcas], 0)
+    };
     m_device.GetLogical().UpdateDescriptorSets(updates, {});
 }
 
 void FSR::UploadImages(Scheduler& scheduler) {
-    if (m_images_ready) {
-        return;
+    if (!m_images_ready) {
+        m_images_ready = true;
+        scheduler.Record([&](vk::CommandBuffer cmdbuf) {
+            for (auto& image : m_dynamic_images) {
+                ClearColorImage(cmdbuf, *image.images[Easu]);
+                ClearColorImage(cmdbuf, *image.images[Rcas]);
+            }
+        });
+        scheduler.Finish();
     }
-
-    scheduler.Record([&](vk::CommandBuffer cmdbuf) {
-        for (auto& image : m_dynamic_images) {
-            ClearColorImage(cmdbuf, *image.images[Easu]);
-            ClearColorImage(cmdbuf, *image.images[Rcas]);
-        }
-    });
-    scheduler.Finish();
-
-    m_images_ready = true;
 }
 
 VkImageView FSR::Draw(Scheduler& scheduler, size_t image_index, VkImage source_image,
diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp
index 5676dfe62a..5bf190bc82 100644
--- a/src/video_core/renderer_vulkan/present/layer.cpp
+++ b/src/video_core/renderer_vulkan/present/layer.cpp
@@ -4,7 +4,12 @@
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include <variant>
 #include "video_core/present.h"
+#include "video_core/renderer_vulkan/present/anti_alias_pass.h"
+/* X11 defines */
+#undef Success
+#undef BadValue
 #include "video_core/renderer_vulkan/vk_rasterizer.h"
 
 #include "common/settings.h"
@@ -58,7 +63,7 @@ Layer::Layer(const Device& device_, MemoryAllocator& memory_allocator_, Schedule
     CreateDescriptorPool();
     CreateDescriptorSets(layout);
     if (filters.get_scaling_filter() == Settings::ScalingFilter::Fsr) {
-        CreateFSR(output_size);
+        fsr.emplace(device, memory_allocator, image_count, output_size);
     }
 }
 
@@ -97,7 +102,11 @@ void Layer::ConfigureDraw(PresentPushConstants* out_push_constants,
     VkImageView source_image_view =
         texture_info ? texture_info->image_view : *raw_image_views[image_index];
 
-    anti_alias->Draw(scheduler, image_index, &source_image, &source_image_view);
+    if (std::holds_alternative<FXAA>(anti_alias)) {
+        std::get<FXAA>(anti_alias).Draw(scheduler, image_index, &source_image, &source_image_view);
+    } else if (std::holds_alternative<SMAA>(anti_alias)) {
+        std::get<SMAA>(anti_alias).Draw(scheduler, image_index, &source_image, &source_image_view);
+    }
 
     auto crop_rect = Tegra::NormalizeCrop(framebuffer, texture_width, texture_height);
     const VkExtent2D render_extent{
@@ -156,10 +165,6 @@ void Layer::CreateRawImages(const Tegra::FramebufferConfig& framebuffer) {
     }
 }
 
-void Layer::CreateFSR(VkExtent2D output_size) {
-    fsr = std::make_unique<FSR>(device, memory_allocator, image_count, output_size);
-}
-
 void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
     if (framebuffer.width == raw_width && framebuffer.height == raw_height &&
         framebuffer.pixel_format == pixel_format && !raw_images.empty()) {
@@ -169,7 +174,7 @@ void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
     raw_width = framebuffer.width;
     raw_height = framebuffer.height;
     pixel_format = framebuffer.pixel_format;
-    anti_alias.reset();
+    anti_alias.emplace<std::monostate>();
 
     ReleaseRawImages();
     CreateStagingBuffer(framebuffer);
@@ -177,9 +182,8 @@ void Layer::RefreshResources(const Tegra::FramebufferConfig& framebuffer) {
 }
 
 void Layer::SetAntiAliasPass() {
-    if (anti_alias && anti_alias_setting == filters.get_anti_aliasing()) {
+    if (!std::holds_alternative<std::monostate>(anti_alias) && anti_alias_setting == filters.get_anti_aliasing())
         return;
-    }
 
     anti_alias_setting = filters.get_anti_aliasing();
 
@@ -190,13 +194,13 @@ void Layer::SetAntiAliasPass() {
 
     switch (anti_alias_setting) {
     case Settings::AntiAliasing::Fxaa:
-        anti_alias = std::make_unique<FXAA>(device, memory_allocator, image_count, render_area);
+        anti_alias.emplace<FXAA>(device, memory_allocator, image_count, render_area);
         break;
     case Settings::AntiAliasing::Smaa:
-        anti_alias = std::make_unique<SMAA>(device, memory_allocator, image_count, render_area);
+        anti_alias.emplace<SMAA>(device, memory_allocator, image_count, render_area);
         break;
     default:
-        anti_alias = std::make_unique<NoAA>();
+        anti_alias.emplace<std::monostate>();
         break;
     }
 }
diff --git a/src/video_core/renderer_vulkan/present/layer.h b/src/video_core/renderer_vulkan/present/layer.h
index f5effdcd7f..062b7792c4 100644
--- a/src/video_core/renderer_vulkan/present/layer.h
+++ b/src/video_core/renderer_vulkan/present/layer.h
@@ -1,11 +1,20 @@
+// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #pragma once
 
+#include <optional>
+#include <variant>
+
 #include "common/math_util.h"
 #include "video_core/host1x/gpu_device_memory_manager.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
+#include "video_core/renderer_vulkan/present/fsr.h"
+#include "video_core/renderer_vulkan/present/fxaa.h"
+#include "video_core/renderer_vulkan/present/smaa.h"
 
 namespace Layout {
 struct FramebufferLayout;
@@ -29,7 +38,6 @@ namespace Vulkan {
 
 class AntiAliasPass;
 class Device;
-class FSR;
 class MemoryAllocator;
 struct PresentPushConstants;
 class RasterizerVulkan;
@@ -54,7 +62,6 @@ private:
     void CreateDescriptorSets(VkDescriptorSetLayout layout);
     void CreateStagingBuffer(const Tegra::FramebufferConfig& framebuffer);
     void CreateRawImages(const Tegra::FramebufferConfig& framebuffer);
-    void CreateFSR(VkExtent2D output_size);
 
     void RefreshResources(const Tegra::FramebufferConfig& framebuffer);
     void SetAntiAliasPass();
@@ -87,9 +94,8 @@ private:
     Service::android::PixelFormat pixel_format{};
 
     Settings::AntiAliasing anti_alias_setting{};
-    std::unique_ptr<AntiAliasPass> anti_alias{};
-
-    std::unique_ptr<FSR> fsr{};
+    std::variant<std::monostate, FXAA, SMAA> anti_alias{};
+    std::optional<FSR> fsr{};
     std::vector<u64> resource_ticks{};
 };