[desktop] Restore metal layer search (#4033)

This was cut out in #3916 by accident.

Signed-off-by: crueter <crueter@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4033

1 files changed, 30 insertions, 1 deletions

diff --git a/src/qt_common/qt_common.cpp b/src/qt_common/qt_common.cpp
index 7550e931af..98de0a6317 100644
--- a/src/qt_common/qt_common.cpp
+++ b/src/qt_common/qt_common.cpp
@@ -87,8 +87,37 @@ Core::Frontend::EmuWindow::WindowSystemInfo GetWindowSystemInfo(QWindow* window)
     // Our Win32 Qt external doesn't have the private API.
     wsi.render_surface = reinterpret_cast<void*>(window->winId());
 #elif defined(__APPLE__)
-    wsi.render_surface = reinterpret_cast<void* (*)(id, SEL)>(objc_msgSend)(
+    id layer = reinterpret_cast<id (*)(id, SEL)>(objc_msgSend)(
         reinterpret_cast<id>(window->winId()), sel_registerName("layer"));
+
+    // In Qt 6, the layer of the NSView might be a QContainerLayer.
+    // MoltenVK needs a CAMetalLayer. We search for it in sublayers.
+    Class metal_layer_class = objc_getClass("CAMetalLayer");
+    id metal_layer = nullptr;
+
+    if (layer) {
+        if (reinterpret_cast<bool (*)(id, SEL, Class)>(objc_msgSend)(
+                layer, sel_registerName("isKindOfClass:"), metal_layer_class)) {
+            metal_layer = layer;
+        } else {
+            id sublayers = reinterpret_cast<id (*)(id, SEL)>(objc_msgSend)(
+                layer, sel_registerName("sublayers"));
+            if (sublayers) {
+                unsigned long count = reinterpret_cast<unsigned long (*)(id, SEL)>(objc_msgSend)(
+                    sublayers, sel_registerName("count"));
+                for (unsigned long i = 0; i < count; ++i) {
+                    id sublayer = reinterpret_cast<id (*)(id, SEL, unsigned long)>(objc_msgSend)(
+                        sublayers, sel_registerName("objectAtIndex:"), i);
+                    if (reinterpret_cast<bool (*)(id, SEL, Class)>(objc_msgSend)(
+                            sublayer, sel_registerName("isKindOfClass:"), metal_layer_class)) {
+                        metal_layer = sublayer;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    wsi.render_surface = reinterpret_cast<void*>(metal_layer ? metal_layer : layer);
 #else
     QPlatformNativeInterface* pni = QGuiApplication::platformNativeInterface();
     wsi.display_connection = pni->nativeResourceForWindow("display", window);