[cmake] fix & allow FFmpeg cross compilations (#2973)

Should allow to cross compile FFmpeg.

Signed-off-by: lizzie lizzie@eden-emu.dev

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/2973
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>

2 files changed, 120 insertions, 66 deletions

diff --git a/externals/cmake-modules/DetectPlatform.cmake b/externals/cmake-modules/DetectPlatform.cmake
index 6475884f1f..de1a85b793 100644
--- a/externals/cmake-modules/DetectPlatform.cmake
+++ b/externals/cmake-modules/DetectPlatform.cmake
@@ -17,6 +17,10 @@
 
 if (${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
     set(PLATFORM_SUN ON)
+elseif (${CMAKE_SYSTEM_NAME} STREQUAL "OpenOrbis")
+    set(PLATFORM_PS4 ON)
+elseif (${CMAKE_SYSTEM_NAME} STREQUAL "managarm")
+    set(PLATFORM_MANAGARM ON)
 elseif (${CMAKE_SYSTEM_NAME} STREQUAL "FreeBSD")
     set(PLATFORM_FREEBSD ON)
 elseif (${CMAKE_SYSTEM_NAME} STREQUAL "OpenBSD")
diff --git a/externals/ffmpeg/CMakeLists.txt b/externals/ffmpeg/CMakeLists.txt
index fcc48c71c4..2d85876ad2 100644
--- a/externals/ffmpeg/CMakeLists.txt
+++ b/externals/ffmpeg/CMakeLists.txt
@@ -11,63 +11,105 @@ set(FFmpeg_HWACCEL_FLAGS)
 set(FFmpeg_HWACCEL_INCLUDE_DIRS)
 set(FFmpeg_HWACCEL_LDFLAGS)
 
-if (UNIX AND NOT ANDROID)
-    find_package(PkgConfig REQUIRED)
-    if (NOT ANDROID)
-        pkg_check_modules(LIBVA libva)
-        pkg_check_modules(CUDA cuda)
-        pkg_check_modules(FFNVCODEC ffnvcodec)
-        pkg_check_modules(VDPAU vdpau)
+if (NOT YUZU_USE_BUNDLED_FFMPEG)
+    set(FFmpeg_CROSS_COMPILE_FLAGS "")
+    if (ANDROID)
+        # TODO: Maybe use CMAKE_SYSROOT? and probably provide a toolchain file for android
+        # I mean isn't that the "proper" way anyways?
+        string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" FFmpeg_HOST_SYSTEM_NAME)
+        set(TOOLCHAIN "${ANDROID_NDK}/toolchains/llvm/prebuilt/${FFmpeg_HOST_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}")
+        set(SYSROOT "${TOOLCHAIN}/sysroot")
+        set(FFmpeg_CPU "armv8-a")
+        list(APPEND FFmpeg_CROSS_COMPILE_FLAGS
+            --enable-cross-compile
+            --arch=arm64
+            #--cpu=${FFmpeg_CPU}
+            --cross-prefix="${TOOLCHAIN}/bin/aarch64-linux-android-"
+            --sysroot="${SYSROOT}"
+            --target-os=android
+            --extra-ldflags="--ld-path=${TOOLCHAIN}/bin/ld.lld"
+            --extra-ldflags="-nostdlib"
+        )
+        set(FFmpeg_IS_CROSS_COMPILING TRUE)
+    # User attempts to do a FFmpeg cross compilation because...
+    # Here we just quickly test against host/system processors not matching
+    # TODO: Test for versions not matching as well?
+    elseif (NOT (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES CMAKE_SYSTEM_PROCESSOR
+    AND CMAKE_HOST_SYSTEM_NAME MATCHES CMAKE_SYSTEM_NAME))
+        string(TOLOWER "${CMAKE_SYSTEM_NAME}" FFmpeg_SYSTEM_NAME)
+        if (FFmpeg_SYSTEM_NAME STREQUAL "openorbis" OR FFmpeg_SYSTEM_NAME STREQUAL "managarm")
+            set(FFmpeg_SYSTEM_NAME "none")
+        endif()
+        # TODO: Can we really do better? Auto-detection? Something clever?
+        list(APPEND FFmpeg_CROSS_COMPILE_FLAGS
+            --enable-cross-compile
+            --arch="${CMAKE_SYSTEM_PROCESSOR}"
+            --target-os="${FFmpeg_SYSTEM_NAME}"
+            --sysroot="${CMAKE_SYSROOT}"
+        )
+        if (DEFINED FFmpeg_CROSS_PREFIX)
+            list(APPEND FFmpeg_CROSS_COMPILE_FLAGS --cross-prefix="${FFmpeg_CROSS_PREFIX}")
+        else()
+            message(WARNING "Please set FFmpeg_CROSS_PREFIX to your cross toolchain prefix, for example: \${CMAKE_STAGING_PREFIX}/bin/${CMAKE_SYSTEM_PROCESSOR}-${CMAKE_SYSTEM_NAME}-")
+        endif()
+        set(FFmpeg_IS_CROSS_COMPILING TRUE)
     endif()
+endif()
 
-    if (NOT APPLE)
-        # In Solaris needs explicit linking for ffmpeg which links to /lib/amd64/libX11.so
-        if(PLATFORM_SUN)
-            find_library(LIBDRM_LIB libdrm PATHS /usr/lib/64 /usr/lib/amd64 /usr/lib)
-            if(LIBDRM_LIB)
+if (PLATFORM_PS4 OR PLATFORM_MANAGARM)
+    # Doesn't support VA-API, don't go thru the embarrassment of trying to enable it
+    list(APPEND FFmpeg_HWACCEL_FLAGS --disable-vaapi)
+elseif (UNIX AND NOT DEFINED FFmpeg_IS_CROSS_COMPILING)
+    find_package(PkgConfig REQUIRED)
+    pkg_check_modules(LIBVA libva)
+    pkg_check_modules(CUDA cuda)
+    pkg_check_modules(FFNVCODEC ffnvcodec)
+    pkg_check_modules(VDPAU vdpau)
+
+    find_package(X11)
+    if(X11_FOUND)
+        if (NOT APPLE)
+            # In Solaris needs explicit linking for ffmpeg which links to /lib/amd64/libX11.so
+            if(PLATFORM_SUN)
                 list(APPEND FFmpeg_HWACCEL_LIBRARIES
                     X11
-                    "${LIBDRM_LIB}")
-                message(STATUS "Found libdrm at: ${LIBDRM_LIB}")
+                    "${CMAKE_SYSROOT}/usr/lib/xorg/amd64/libdrm.so")
             else()
-                message(WARNING "libdrm not found, disabling libdrm support")
-                list(APPEND FFmpeg_HWACCEL_FLAGS
-                    --disable-libdrm)
+                pkg_check_modules(LIBDRM libdrm REQUIRED)
+                list(APPEND FFmpeg_HWACCEL_LIBRARIES
+                    ${LIBDRM_LIBRARIES})
+                list(APPEND FFmpeg_HWACCEL_INCLUDE_DIRS
+                    ${LIBDRM_INCLUDE_DIRS})
             endif()
-        else()
-            pkg_check_modules(LIBDRM libdrm REQUIRED)
-            list(APPEND FFmpeg_HWACCEL_LIBRARIES
-                ${LIBDRM_LIBRARIES})
-            list(APPEND FFmpeg_HWACCEL_INCLUDE_DIRS
-                ${LIBDRM_INCLUDE_DIRS})
             list(APPEND FFmpeg_HWACCEL_FLAGS
                 --enable-libdrm)
         endif()
-    endif()
-
-    if(LIBVA_FOUND)
-        find_package(X11 REQUIRED)
-        pkg_check_modules(LIBVA-DRM libva-drm REQUIRED)
-        pkg_check_modules(LIBVA-X11 libva-x11 REQUIRED)
-        list(APPEND FFmpeg_HWACCEL_LIBRARIES
-            ${X11_LIBRARIES}
-            ${LIBVA-DRM_LIBRARIES}
-            ${LIBVA-X11_LIBRARIES}
-            ${LIBVA_LIBRARIES})
-        list(APPEND FFmpeg_HWACCEL_FLAGS
-            --enable-hwaccel=h264_vaapi
-            --enable-hwaccel=vp8_vaapi
-            --enable-hwaccel=vp9_vaapi)
-        list(APPEND FFmpeg_HWACCEL_INCLUDE_DIRS
-            ${X11_INCLUDE_DIRS}
-            ${LIBVA-DRM_INCLUDE_DIRS}
-            ${LIBVA-X11_INCLUDE_DIRS}
-            ${LIBVA_INCLUDE_DIRS}
-        )
-        message(STATUS "ffmpeg: va-api libraries version ${LIBVA_VERSION} found")
+        if(LIBVA_FOUND)
+            pkg_check_modules(LIBVA-DRM libva-drm REQUIRED)
+            pkg_check_modules(LIBVA-X11 libva-x11 REQUIRED)
+            list(APPEND FFmpeg_HWACCEL_LIBRARIES
+                ${X11_LIBRARIES}
+                ${LIBVA-DRM_LIBRARIES}
+                ${LIBVA-X11_LIBRARIES}
+                ${LIBVA_LIBRARIES})
+            list(APPEND FFmpeg_HWACCEL_FLAGS
+                --enable-hwaccel=h264_vaapi
+                --enable-hwaccel=vp8_vaapi
+                --enable-hwaccel=vp9_vaapi)
+            list(APPEND FFmpeg_HWACCEL_INCLUDE_DIRS
+                ${X11_INCLUDE_DIRS}
+                ${LIBVA-DRM_INCLUDE_DIRS}
+                ${LIBVA-X11_INCLUDE_DIRS}
+                ${LIBVA_INCLUDE_DIRS}
+            )
+            message(STATUS "ffmpeg: va-api libraries version ${LIBVA_VERSION} found")
+        else()
+            list(APPEND FFmpeg_HWACCEL_FLAGS --disable-vaapi)
+            message(WARNING "ffmpeg: libva-dev not found, disabling Video Acceleration API (VA-API)...")
+        endif()
     else()
         list(APPEND FFmpeg_HWACCEL_FLAGS --disable-vaapi)
-        message(WARNING "ffmpeg: libva-dev not found, disabling Video Acceleration API (VA-API)...")
+        message(WARNING "ffmpeg: X11 libraries not found, disabling VA-API...")
     endif()
 
     if (FFNVCODEC_FOUND)
@@ -111,6 +153,28 @@ if (UNIX AND NOT ANDROID)
     endif()
 endif()
 
+if (PLATFORM_PS4)
+    list(APPEND FFmpeg_CROSS_COMPILE_LIBS
+        -lkernel
+        -lSceUserService
+        -lSceSysmodule
+        -lSceNet
+        -lSceLibcInternal
+    )
+    list(APPEND FFmpeg_CROSS_COMPILE_FLAGS
+        --disable-pthreads
+        --extra-cflags=${CMAKE_SYSROOT}/usr/include
+        --extra-cxxflags=${CMAKE_SYSROOT}/usr/include
+        --extra-libs="${FFmpeg_CROSS_COMPILE_LIBS}"
+    )
+elseif (PLATFORM_MANAGARM)
+    # Required for proper stuff
+    list(APPEND FFmpeg_CROSS_COMPILE_FLAGS 
+        --disable-pthreads
+        --extra-libs="${FFmpeg_CROSS_COMPILE_LIBS}"
+    )
+endif()
+
 if (YUZU_USE_BUNDLED_FFMPEG)
     AddJsonPackage(ffmpeg-ci)
 
@@ -181,24 +245,6 @@ else()
 
     find_program(BASH_PROGRAM bash REQUIRED)
 
-    set(FFmpeg_CROSS_COMPILE_FLAGS "")
-    if (ANDROID)
-        string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" FFmpeg_HOST_SYSTEM_NAME)
-        set(TOOLCHAIN "${ANDROID_NDK}/toolchains/llvm/prebuilt/${FFmpeg_HOST_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}")
-        set(SYSROOT "${TOOLCHAIN}/sysroot")
-        set(FFmpeg_CPU "armv8-a")
-        list(APPEND FFmpeg_CROSS_COMPILE_FLAGS
-            --arch=arm64
-            #--cpu=${FFmpeg_CPU}
-            --enable-cross-compile
-            --cross-prefix=${TOOLCHAIN}/bin/aarch64-linux-android-
-            --sysroot=${SYSROOT}
-            --target-os=android
-            --extra-ldflags="--ld-path=${TOOLCHAIN}/bin/ld.lld"
-            --extra-ldflags="-nostdlib"
-        )
-    endif()
-
     # `configure` parameters builds only exactly what yuzu needs from FFmpeg
     # `--disable-vdpau` is needed to avoid linking issues
     set(FFmpeg_CC ${CMAKE_C_COMPILER_LAUNCHER}  ${CMAKE_C_COMPILER})
@@ -221,8 +267,12 @@ else()
                 --enable-decoder=vp9
                 --enable-filter=yadif,scale
                 --enable-pic
-                --cc="${FFmpeg_CC}"
-                --cxx="${FFmpeg_CXX}"
+                --cc=${FFmpeg_CC}
+                --cxx=${FFmpeg_CXX}
+                --ld=${CMAKE_LINKER}
+                --extra-cflags=${CMAKE_C_FLAGS}
+                --extra-cxxflags=${CMAKE_CXX_FLAGS}
+                --extra-ldflags=${CMAKE_C_LINK_FLAGS}
                 ${FFmpeg_HWACCEL_FLAGS}
                 ${FFmpeg_CROSS_COMPILE_FLAGS}
         WORKING_DIRECTORY