[macos, qt] Fix naturalDrawableSizeMVK (#3403)

"The issue is that MoltenVK needs the raw CAMetalLayer, but Qt 6 wraps it. I fixed it by making GetWindowSystemInfo search for the sublayer. Here is the patch file. It tested it on my Mac M1 Pro just now, and Eden plays the games now instead of crashing." - rayman

Authored-by: rayman
Signed-off-by: lizzie <lizzie@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3403
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: DraVee <dravee@eden-emu.dev>
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>

1 files changed, 27 insertions, 2 deletions

diff --git a/src/qt_common/qt_common.cpp b/src/qt_common/qt_common.cpp
index f2091df866..af4e4ffa61 100644
--- a/src/qt_common/qt_common.cpp
+++ b/src/qt_common/qt_common.cpp
@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "qt_common.h"
@@ -22,6 +22,7 @@
 #include <qpa/qplatformnativeinterface.h>
 #elif defined(__APPLE__)
 #include <objc/message.h>
+#include <objc/runtime.h>
 #endif
 
 namespace QtCommon {
@@ -68,8 +69,32 @@ Core::Frontend::EmuWindow::WindowSystemInfo GetWindowSystemInfo(QWindow* window)
     // Our Win32 Qt external doesn't have the private API.
     wsi.render_surface = reinterpret_cast<void*>(window->winId());
 #elif defined(__APPLE__)
-    wsi.render_surface = reinterpret_cast<void* (*) (id, SEL)>(
+    id layer = reinterpret_cast<id (*) (id, SEL)>(
         objc_msgSend)(reinterpret_cast<id>(window->winId()), sel_registerName("layer"));
+
+    // In Qt 6, the layer of the NSView might be a QContainerLayer.
+    // MoltenVK needs a CAMetalLayer. We search for it in sublayers.
+    Class metal_layer_class = objc_getClass("CAMetalLayer");
+    id metal_layer = nullptr;
+
+    if (layer) {
+        if (reinterpret_cast<bool (*) (id, SEL, Class)>(objc_msgSend)(layer, sel_registerName("isKindOfClass:"), metal_layer_class)) {
+            metal_layer = layer;
+        } else {
+            id sublayers = reinterpret_cast<id (*) (id, SEL)>(objc_msgSend)(layer, sel_registerName("sublayers"));
+            if (sublayers) {
+                unsigned long count = reinterpret_cast<unsigned long (*) (id, SEL)>(objc_msgSend)(sublayers, sel_registerName("count"));
+                for (unsigned long i = 0; i < count; ++i) {
+                    id sublayer = reinterpret_cast<id (*) (id, SEL, unsigned long)>(objc_msgSend)(sublayers, sel_registerName("objectAtIndex:"), i);
+                    if (reinterpret_cast<bool (*) (id, SEL, Class)>(objc_msgSend)(sublayer, sel_registerName("isKindOfClass:"), metal_layer_class)) {
+                        metal_layer = sublayer;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    wsi.render_surface = reinterpret_cast<void*>(metal_layer ? metal_layer : layer);
 #else
     QPlatformNativeInterface* pni = QGuiApplication::platformNativeInterface();
     wsi.display_connection = pni->nativeResourceForWindow("display", window);