[VK, MacOS] Fix strict output type mismatch on Metal (MK8D/TOTK fix) (#3414)

Metal validation requires fragment shader output types to strictly match the render target format (e.g., writing float to RGBA32Uint is invalid).

This commit:
1. Adds color_output_types to RuntimeInfo.
2. Detects Integer/SignedInteger render targets in the Vulkan backend (MoltenVK only).
3. Updates the SPIR-V emitter to declare the correct output type (Uint/Sint) and bitcast values accordingly.

This fixes the VK_ERROR_INITIALIZATION_FAILED crash on macOS.

Co-authored-by: crueter <crueter@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3414
Co-authored-by: rayman30 <silentbitdev@gmail.com>
Co-committed-by: rayman30 <silentbitdev@gmail.com>

4 files changed, 42 insertions, 4 deletions

diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp
index 1de5709394..db11def7b2 100644
--- a/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp
+++ b/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp
@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
@@ -492,8 +492,22 @@ void EmitSetPatch(EmitContext& ctx, IR::Patch patch, Id value) {
 
 void EmitSetFragColor(EmitContext& ctx, u32 index, u32 component, Id value) {
     const Id component_id{ctx.Const(component)};
-    const Id pointer{ctx.OpAccessChain(ctx.output_f32, ctx.frag_color.at(index), component_id)};
-    ctx.OpStore(pointer, value);
+    const AttributeType type{ctx.runtime_info.color_output_types[index]};
+    if (type == AttributeType::Float) {
+        const Id pointer{ctx.OpAccessChain(ctx.output_f32, ctx.frag_color.at(index), component_id)};
+        ctx.OpStore(pointer, value);
+    } else if (type == AttributeType::UnsignedInt) {
+        const Id pointer{ctx.OpAccessChain(ctx.output_u32, ctx.frag_color.at(index), component_id)};
+        ctx.OpStore(pointer, ctx.OpBitcast(ctx.U32[1], value));
+    } else if (type == AttributeType::SignedInt) {
+        const Id output_s32{ctx.TypePointer(spv::StorageClass::Output, ctx.S32[1])};
+        const Id pointer{ctx.OpAccessChain(output_s32, ctx.frag_color.at(index), component_id)};
+        ctx.OpStore(pointer, ctx.OpBitcast(ctx.S32[1], value));
+    } else {
+        // Disabled or unknown, treat as float
+        const Id pointer{ctx.OpAccessChain(ctx.output_f32, ctx.frag_color.at(index), component_id)};
+        ctx.OpStore(pointer, value);
+    }
 }
 
 void EmitSetSampleMask(EmitContext& ctx, Id value) {
diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp
index 38d1e075a1..c70f9ae934 100644
--- a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp
+++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp
@@ -1677,7 +1677,8 @@ void EmitContext::DefineOutputs(const IR::Program& program) {
             if (!info.stores_frag_color[index] && !profile.need_declared_frag_colors) {
                 continue;
             }
-            frag_color[index] = DefineOutput(*this, F32[4], std::nullopt);
+            const Id type{GetAttributeType(*this, runtime_info.color_output_types[index])};
+            frag_color[index] = DefineOutput(*this, type, std::nullopt);
             Decorate(frag_color[index], spv::Decoration::Location, index);
             Name(frag_color[index], fmt::format("frag_color{}", index));
         }
diff --git a/src/shader_recompiler/runtime_info.h b/src/shader_recompiler/runtime_info.h
index dc54d932a6..613c598d0c 100644
--- a/src/shader_recompiler/runtime_info.h
+++ b/src/shader_recompiler/runtime_info.h
@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -104,6 +107,9 @@ struct RuntimeInfo {
     /// Transform feedback state for each varying
     std::array<TransformFeedbackVarying, 256> xfb_varyings{};
     u32 xfb_count{0};
+
+    /// Output types for each color attachment
+    std::array<AttributeType, 8> color_output_types{};
 };
 
 } // namespace Shader
diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
index 9f2d16a328..6207e127c1 100644
--- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
@@ -39,6 +39,7 @@
 #include "video_core/shader_cache.h"
 #include "video_core/shader_environment.h"
 #include "video_core/shader_notify.h"
+#include "video_core/surface.h"
 #include "video_core/vulkan_common/vulkan_device.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
@@ -238,6 +239,22 @@ Shader::RuntimeInfo MakeRuntimeInfo(std::span<const Shader::IR::Program> program
         info.alpha_test_func = MaxwellToCompareFunction(
             key.state.UnpackComparisonOp(key.state.alpha_test_func.Value()));
         info.alpha_test_reference = std::bit_cast<float>(key.state.alpha_test_ref);
+
+        if (device.IsMoltenVK()) {
+            for (size_t i = 0; i < 8; ++i) {
+                const auto format = static_cast<Tegra::RenderTargetFormat>(key.state.color_formats[i]);
+                const auto pixel_format = VideoCore::Surface::PixelFormatFromRenderTargetFormat(format);
+                if (VideoCore::Surface::IsPixelFormatInteger(pixel_format)) {
+                    if (VideoCore::Surface::IsPixelFormatSignedInteger(pixel_format)) {
+                        info.color_output_types[i] = Shader::AttributeType::SignedInt;
+                    } else {
+                        info.color_output_types[i] = Shader::AttributeType::UnsignedInt;
+                    }
+                } else {
+                    info.color_output_types[i] = Shader::AttributeType::Float;
+                }
+            }
+        }
         break;
     default:
         break;