[host1x] Improve FFmpeg error handling (#2643)

This improves the FFmpeg error handling.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/2643
Co-authored-by: MaranBr <maranbr@outlook.com>
Co-committed-by: MaranBr <maranbr@outlook.com>

4 files changed, 21 insertions, 12 deletions

diff --git a/src/video_core/host1x/codecs/decoder.cpp b/src/video_core/host1x/codecs/decoder.cpp
index cb17784b19..887eb28c8c 100755
--- a/src/video_core/host1x/codecs/decoder.cpp
+++ b/src/video_core/host1x/codecs/decoder.cpp
@@ -38,6 +38,10 @@ void Decoder::Decode() {
     // Receive output frames from decoder.
     auto frame = decode_api.ReceiveFrame();
 
+    if (!frame) {
+        return;
+    }
+
     if (IsInterlaced()) {
         auto [luma_top, luma_bottom, chroma_top, chroma_bottom] = GetInterlacedOffsets();
         auto frame_copy = frame;
diff --git a/src/video_core/host1x/ffmpeg/ffmpeg.cpp b/src/video_core/host1x/ffmpeg/ffmpeg.cpp
index 536a01fcc8..bbbbe615ce 100644
--- a/src/video_core/host1x/ffmpeg/ffmpeg.cpp
+++ b/src/video_core/host1x/ffmpeg/ffmpeg.cpp
@@ -233,7 +233,7 @@ bool DecoderContext::OpenContext(const Decoder& decoder) {
 }
 
 bool DecoderContext::SendPacket(const Packet& packet) {
-	if (const int ret = avcodec_send_packet(m_codec_context, packet.GetPacket()); ret < 0 && ret != AVERROR_EOF) {
+	if (const int ret = avcodec_send_packet(m_codec_context, packet.GetPacket()); ret < 0 && ret != AVERROR_EOF && ret != AVERROR(EAGAIN)) {
 		LOG_ERROR(HW_GPU, "avcodec_send_packet error: {}", AVError(ret));
 		return false;
 	}
@@ -242,31 +242,31 @@ bool DecoderContext::SendPacket(const Packet& packet) {
 }
 
 std::shared_ptr<Frame> DecoderContext::ReceiveFrame() {
-	auto ReceiveImpl = [&](AVFrame* frame) -> bool {
-		if (const int ret = avcodec_receive_frame(m_codec_context, frame); ret < 0 && ret != AVERROR_EOF) {
+	auto ReceiveImpl = [&](AVFrame* frame) -> int {
+        const int ret = avcodec_receive_frame(m_codec_context, frame);
+		if (ret < 0 && ret != AVERROR_EOF && ret != AVERROR(EAGAIN)) {
 			LOG_ERROR(HW_GPU, "avcodec_receive_frame error: {}", AVError(ret));
-			return false;
 		}
-		return true;
+		return ret;
 	};
 
 	std::shared_ptr<Frame> intermediate_frame = std::make_shared<Frame>();
-	if (!ReceiveImpl(intermediate_frame->GetFrame())) {
+	if (ReceiveImpl(intermediate_frame->GetFrame()) < 0) {
 		return {};
 	}
 
-	m_temp_frame = std::make_shared<Frame>();
+	m_final_frame = std::make_shared<Frame>();
 	if (m_codec_context->hw_device_ctx) {
-		m_temp_frame->SetFormat(PreferredGpuFormat);
-		if (int ret = av_hwframe_transfer_data(m_temp_frame->GetFrame(), intermediate_frame->GetFrame(), 0); ret < 0) {
+		m_final_frame->SetFormat(PreferredGpuFormat);
+		if (const int ret = av_hwframe_transfer_data(m_final_frame->GetFrame(), intermediate_frame->GetFrame(), 0); ret < 0) {
 			LOG_ERROR(HW_GPU, "av_hwframe_transfer_data error: {}", AVError(ret));
 			return {};
 		}
 	} else {
-		m_temp_frame = std::move(intermediate_frame);
+		m_final_frame = std::move(intermediate_frame);
 	}
 
-	return std::move(m_temp_frame);
+	return std::move(m_final_frame);
 }
 
 void DecodeApi::Reset() {
diff --git a/src/video_core/host1x/ffmpeg/ffmpeg.h b/src/video_core/host1x/ffmpeg/ffmpeg.h
index 0dd7c7cb04..d60a8ac4a7 100644
--- a/src/video_core/host1x/ffmpeg/ffmpeg.h
+++ b/src/video_core/host1x/ffmpeg/ffmpeg.h
@@ -194,7 +194,7 @@ public:
 private:
     const Decoder& m_decoder;
     AVCodecContext* m_codec_context{};
-    std::shared_ptr<Frame> m_temp_frame{};
+    std::shared_ptr<Frame> m_final_frame{};
     bool m_decode_order{};
 };
 
diff --git a/src/video_core/host1x/vic.cpp b/src/video_core/host1x/vic.cpp
index 9c33370337..3dbbfa5552 100644
--- a/src/video_core/host1x/vic.cpp
+++ b/src/video_core/host1x/vic.cpp
@@ -146,6 +146,11 @@ void Vic::Execute() {
             }
 
             auto frame = frame_queue.GetFrame(nvdec_id, luma_offset);
+
+            if (!frame) {
+                continue;
+            }
+
             if (!frame.get()) {
                 LOG_ERROR(HW_GPU, "Vic {} failed to get frame with offset {:#X}", id, luma_offset);
                 continue;