[video_core] Properly disable/avoid building OpenGL when it's disabled (#3692)

- OpenGL symbols would still be included in builds without OpenGL, this pr fixes that
- Same goes for Vulkan, but now with `ENABLE_VULKAN`
- Add support to have OpenGL-only builds (why would you do this?)
- Add support for headless runs (yes you could just select NULL backend, but why not compile it headless? :)

Signed-off-by: lizzie <lizzie@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3692
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: DraVee <chimera@dravee.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

7 files changed, 121 insertions, 85 deletions

diff --git a/src/qt_common/CMakeLists.txt b/src/qt_common/CMakeLists.txt
index f0522c07d2..904b03d288 100644
--- a/src/qt_common/CMakeLists.txt
+++ b/src/qt_common/CMakeLists.txt
@@ -84,7 +84,7 @@ target_link_libraries(qt_common PRIVATE core Qt6::Core Qt6::Concurrent SimpleIni
 target_link_libraries(qt_common PUBLIC frozen::frozen-headers)
 target_link_libraries(qt_common PRIVATE gamemode::headers frontend_common)
 
-if (NOT APPLE AND ENABLE_OPENGL)
+if (ENABLE_OPENGL)
     target_compile_definitions(qt_common PUBLIC HAS_OPENGL)
 endif()
 
diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt
index a58a73cd3c..3324682639 100644
--- a/src/video_core/CMakeLists.txt
+++ b/src/video_core/CMakeLists.txt
@@ -111,64 +111,14 @@ add_library(video_core STATIC
     rasterizer_interface.h
     renderer_base.cpp
     renderer_base.h
+
+    # Null
     renderer_null/null_rasterizer.cpp
     renderer_null/null_rasterizer.h
     renderer_null/renderer_null.cpp
     renderer_null/renderer_null.h
-    renderer_opengl/present/filters.cpp
-    renderer_opengl/present/filters.h
-    renderer_opengl/present/fsr.cpp
-    renderer_opengl/present/fsr.h
-    renderer_opengl/present/fxaa.cpp
-    renderer_opengl/present/fxaa.h
-    renderer_opengl/present/layer.cpp
-    renderer_opengl/present/layer.h
-    renderer_opengl/present/present_uniforms.h
-    renderer_opengl/present/smaa.cpp
-    renderer_opengl/present/smaa.h
-    renderer_opengl/present/util.h
-    renderer_opengl/present/window_adapt_pass.cpp
-    renderer_opengl/present/window_adapt_pass.h
-    renderer_opengl/blit_image.cpp
-    renderer_opengl/blit_image.h
-    renderer_opengl/gl_blit_screen.cpp
-    renderer_opengl/gl_blit_screen.h
-    renderer_opengl/gl_buffer_cache_base.cpp
-    renderer_opengl/gl_buffer_cache.cpp
-    renderer_opengl/gl_buffer_cache.h
-    renderer_opengl/gl_compute_pipeline.cpp
-    renderer_opengl/gl_compute_pipeline.h
-    renderer_opengl/gl_device.cpp
-    renderer_opengl/gl_device.h
-    renderer_opengl/gl_fence_manager.cpp
-    renderer_opengl/gl_fence_manager.h
-    renderer_opengl/gl_graphics_pipeline.cpp
-    renderer_opengl/gl_graphics_pipeline.h
-    renderer_opengl/gl_rasterizer.cpp
-    renderer_opengl/gl_rasterizer.h
-    renderer_opengl/gl_resource_manager.cpp
-    renderer_opengl/gl_resource_manager.h
-    renderer_opengl/gl_shader_cache.cpp
-    renderer_opengl/gl_shader_cache.h
-    renderer_opengl/gl_shader_manager.cpp
-    renderer_opengl/gl_shader_manager.h
-    renderer_opengl/gl_shader_context.h
-    renderer_opengl/gl_shader_util.cpp
-    renderer_opengl/gl_shader_util.h
-    renderer_opengl/gl_state_tracker.cpp
-    renderer_opengl/gl_state_tracker.h
-    renderer_opengl/gl_staging_buffer_pool.cpp
-    renderer_opengl/gl_staging_buffer_pool.h
-    renderer_opengl/gl_texture_cache.cpp
-    renderer_opengl/gl_texture_cache.h
-    renderer_opengl/gl_texture_cache_base.cpp
-    renderer_opengl/gl_query_cache.cpp
-    renderer_opengl/gl_query_cache.h
-    renderer_opengl/maxwell_to_gl.h
-    renderer_opengl/renderer_opengl.cpp
-    renderer_opengl/renderer_opengl.h
-    renderer_opengl/util_shaders.cpp
-    renderer_opengl/util_shaders.h
+
+    # Vulkan
     renderer_vulkan/present/anti_alias_pass.h
     renderer_vulkan/present/filters.cpp
     renderer_vulkan/present/filters.h
@@ -244,6 +194,25 @@ add_library(video_core STATIC
     renderer_vulkan/vk_turbo_mode.h
     renderer_vulkan/vk_update_descriptor.cpp
     renderer_vulkan/vk_update_descriptor.h
+    vulkan_common/vulkan_debug_callback.cpp
+    vulkan_common/vulkan_debug_callback.h
+    vulkan_common/vulkan_device.cpp
+    vulkan_common/vulkan_device.h
+    vulkan_common/vulkan_instance.cpp
+    vulkan_common/vulkan_instance.h
+    vulkan_common/vulkan_library.cpp
+    vulkan_common/vulkan_library.h
+    vulkan_common/vulkan_memory_allocator.cpp
+    vulkan_common/vulkan_memory_allocator.h
+    vulkan_common/vulkan_surface.cpp
+    vulkan_common/vulkan_surface.h
+    vulkan_common/vulkan_wrapper.cpp
+    vulkan_common/vulkan_wrapper.h
+    vulkan_common/nsight_aftermath_tracker.cpp
+    vulkan_common/nsight_aftermath_tracker.h
+    vulkan_common/vma.h
+    vulkan_common/vulkan.h
+
     shader_cache.cpp
     shader_cache.h
     shader_environment.cpp
@@ -293,26 +262,67 @@ add_library(video_core STATIC
     transform_feedback.h
     video_core.cpp
     video_core.h
-    vulkan_common/vulkan_debug_callback.cpp
-    vulkan_common/vulkan_debug_callback.h
-    vulkan_common/vulkan_device.cpp
-    vulkan_common/vulkan_device.h
-    vulkan_common/vulkan_instance.cpp
-    vulkan_common/vulkan_instance.h
-    vulkan_common/vulkan_library.cpp
-    vulkan_common/vulkan_library.h
-    vulkan_common/vulkan_memory_allocator.cpp
-    vulkan_common/vulkan_memory_allocator.h
-    vulkan_common/vulkan_surface.cpp
-    vulkan_common/vulkan_surface.h
-    vulkan_common/vulkan_wrapper.cpp
-    vulkan_common/vulkan_wrapper.h
-    vulkan_common/nsight_aftermath_tracker.cpp
-    vulkan_common/nsight_aftermath_tracker.h
-    vulkan_common/vma.h
-    vulkan_common/vulkan.h
 )
 
+if (ENABLE_OPENGL)
+    target_sources(video_core PRIVATE
+        renderer_opengl/present/filters.cpp
+        renderer_opengl/present/filters.h
+        renderer_opengl/present/fsr.cpp
+        renderer_opengl/present/fsr.h
+        renderer_opengl/present/fxaa.cpp
+        renderer_opengl/present/fxaa.h
+        renderer_opengl/present/layer.cpp
+        renderer_opengl/present/layer.h
+        renderer_opengl/present/present_uniforms.h
+        renderer_opengl/present/smaa.cpp
+        renderer_opengl/present/smaa.h
+        renderer_opengl/present/util.h
+        renderer_opengl/present/window_adapt_pass.cpp
+        renderer_opengl/present/window_adapt_pass.h
+        renderer_opengl/blit_image.cpp
+        renderer_opengl/blit_image.h
+        renderer_opengl/gl_blit_screen.cpp
+        renderer_opengl/gl_blit_screen.h
+        renderer_opengl/gl_buffer_cache_base.cpp
+        renderer_opengl/gl_buffer_cache.cpp
+        renderer_opengl/gl_buffer_cache.h
+        renderer_opengl/gl_compute_pipeline.cpp
+        renderer_opengl/gl_compute_pipeline.h
+        renderer_opengl/gl_device.cpp
+        renderer_opengl/gl_device.h
+        renderer_opengl/gl_fence_manager.cpp
+        renderer_opengl/gl_fence_manager.h
+        renderer_opengl/gl_graphics_pipeline.cpp
+        renderer_opengl/gl_graphics_pipeline.h
+        renderer_opengl/gl_rasterizer.cpp
+        renderer_opengl/gl_rasterizer.h
+        renderer_opengl/gl_resource_manager.cpp
+        renderer_opengl/gl_resource_manager.h
+        renderer_opengl/gl_shader_cache.cpp
+        renderer_opengl/gl_shader_cache.h
+        renderer_opengl/gl_shader_manager.cpp
+        renderer_opengl/gl_shader_manager.h
+        renderer_opengl/gl_shader_context.h
+        renderer_opengl/gl_shader_util.cpp
+        renderer_opengl/gl_shader_util.h
+        renderer_opengl/gl_state_tracker.cpp
+        renderer_opengl/gl_state_tracker.h
+        renderer_opengl/gl_staging_buffer_pool.cpp
+        renderer_opengl/gl_staging_buffer_pool.h
+        renderer_opengl/gl_texture_cache.cpp
+        renderer_opengl/gl_texture_cache.h
+        renderer_opengl/gl_texture_cache_base.cpp
+        renderer_opengl/gl_query_cache.cpp
+        renderer_opengl/gl_query_cache.h
+        renderer_opengl/maxwell_to_gl.h
+        renderer_opengl/renderer_opengl.cpp
+        renderer_opengl/renderer_opengl.h
+        renderer_opengl/util_shaders.cpp
+        renderer_opengl/util_shaders.h
+    )
+endif()
+
 target_link_libraries(video_core PUBLIC common core)
 target_link_libraries(video_core PUBLIC glad shader_recompiler stb bc_decoder gpu_logging)
 
@@ -371,6 +381,10 @@ else()
     endif()
 endif()
 
+if (ENABLE_OPENGL)
+    target_compile_definitions(video_core PUBLIC HAS_OPENGL)
+endif()
+
 if (ARCHITECTURE_x86_64)
     target_link_libraries(video_core PUBLIC xbyak::xbyak)
 endif()
diff --git a/src/video_core/video_core.cpp b/src/video_core/video_core.cpp
index 4a9751e208..fd91ee1667 100644
--- a/src/video_core/video_core.cpp
+++ b/src/video_core/video_core.cpp
@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: 2014 Citra Emulator Project
@@ -9,26 +9,29 @@
 #include "common/logging/log.h"
 #include "common/settings.h"
 #include "core/core.h"
+#include "core/frontend/emu_window.h"
+#include "core/frontend/graphics_context.h"
 #include "video_core/host1x/gpu_device_memory_manager.h"
 #include "video_core/host1x/host1x.h"
 #include "video_core/renderer_base.h"
 #include "video_core/renderer_null/renderer_null.h"
+#ifdef HAS_OPENGL
 #include "video_core/renderer_opengl/renderer_opengl.h"
+#endif
 #include "video_core/renderer_vulkan/renderer_vulkan.h"
 #include "video_core/video_core.h"
 
 namespace {
 
-std::unique_ptr<VideoCore::RendererBase> CreateRenderer(
-    Core::System& system, Core::Frontend::EmuWindow& emu_window, Tegra::GPU& gpu,
-    std::unique_ptr<Core::Frontend::GraphicsContext> context) {
-    auto& device_memory = system.Host1x().MemoryManager();
-
+std::unique_ptr<VideoCore::RendererBase> CreateRenderer(Core::System& system, Core::Frontend::EmuWindow& emu_window, Tegra::GPU& gpu, std::unique_ptr<Core::Frontend::GraphicsContext> context) {
+    [[maybe_unused]] auto& device_memory = system.Host1x().MemoryManager();
     switch (Settings::values.renderer_backend.GetValue()) {
+#ifdef HAS_OPENGL
     case Settings::RendererBackend::OpenGL_GLSL:
     case Settings::RendererBackend::OpenGL_GLASM:
     case Settings::RendererBackend::OpenGL_SPIRV:
         return std::make_unique<OpenGL::RendererOpenGL>(emu_window, device_memory, gpu, std::move(context));
+#endif
     case Settings::RendererBackend::Vulkan:
         return std::make_unique<Vulkan::RendererVulkan>(emu_window, device_memory, gpu, std::move(context));
     case Settings::RendererBackend::Null:
diff --git a/src/yuzu/bootmanager.cpp b/src/yuzu/bootmanager.cpp
index 69b3681d0a..0c0ce6e90c 100644
--- a/src/yuzu/bootmanager.cpp
+++ b/src/yuzu/bootmanager.cpp
@@ -1020,8 +1020,7 @@ bool GRenderWindow::InitializeOpenGL() {
 
     return true;
 #else
-    QMessageBox::warning(this, tr("OpenGL not available!"),
-                         tr("Eden has not been compiled with OpenGL support."));
+    QMessageBox::warning(this, tr("OpenGL not available!"), tr("Eden has not been compiled with OpenGL support."));
     return false;
 #endif
 }
@@ -1031,7 +1030,6 @@ bool GRenderWindow::InitializeVulkan() {
     child_widget = child;
     child_widget->windowHandle()->create();
     main_context = std::make_unique<DummyContext>();
-
     return true;
 }
 
diff --git a/src/yuzu/main_window.cpp b/src/yuzu/main_window.cpp
index a251451bea..029826dfa3 100644
--- a/src/yuzu/main_window.cpp
+++ b/src/yuzu/main_window.cpp
@@ -3784,14 +3784,30 @@ void MainWindow::OnToggleAdaptingFilter() {
 
 void MainWindow::OnToggleGraphicsAPI() {
     auto api = Settings::values.renderer_backend.GetValue();
-    if (api != Settings::RendererBackend::Vulkan) {
-        api = Settings::RendererBackend::Vulkan;
-    } else {
+    switch (api) {
 #ifdef HAS_OPENGL
+    case Settings::RendererBackend::Vulkan:
+        api = Settings::RendererBackend::OpenGL_GLSL;
+        break;
+    case Settings::RendererBackend::OpenGL_GLSL:
         api = Settings::RendererBackend::OpenGL_GLSL;
+        break;
+    case Settings::RendererBackend::OpenGL_SPIRV:
+        api = Settings::RendererBackend::OpenGL_GLASM;
+        break;
+    case Settings::RendererBackend::OpenGL_GLASM:
+        api = Settings::RendererBackend::Null;
+        break;
 #else
+    case Settings::RendererBackend::Vulkan:
         api = Settings::RendererBackend::Null;
+        break;
 #endif
+    case Settings::RendererBackend::Null:
+        api = Settings::RendererBackend::Vulkan;
+        break;
+    default:
+        break;
     }
     Settings::values.renderer_backend.SetValue(api);
     renderer_status_button->setChecked(api == Settings::RendererBackend::Vulkan);
diff --git a/src/yuzu/startup_checks.cpp b/src/yuzu/startup_checks.cpp
index 2e77c7cd06..f54992d19a 100644
--- a/src/yuzu/startup_checks.cpp
+++ b/src/yuzu/startup_checks.cpp
@@ -1,6 +1,10 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+// Don't move this! put it below and MSVC will get angry!
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
 #ifdef _WIN32
diff --git a/src/yuzu/vk_device_info.cpp b/src/yuzu/vk_device_info.cpp
index 2b87ccd3ca..53dacdc090 100644
--- a/src/yuzu/vk_device_info.cpp
+++ b/src/yuzu/vk_device_info.cpp
@@ -77,4 +77,5 @@ void PopulateRecords(std::vector<Record>& records, QWindow* window) try {
 } catch (const Vulkan::vk::Exception& exception) {
     LOG_ERROR(Frontend, "Failed to enumerate devices with error: {}", exception.what());
 }
+
 } // namespace VkDeviceInfo