Merge pull request #356 from Djihads80/fix/nx2-video-decode

nx2: video decoding fixes (green movies, end of stream, H.264 detection)

5 files changed, 210 insertions, 27 deletions

diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h
index 259681a2a2..290d82746d 100644
--- a/src/core/hle/service/nx2_gsv.h
+++ b/src/core/hle/service/nx2_gsv.h
@@ -2347,7 +2347,7 @@ public:
             {8, &INx2VideoSession::Flush, "Flush"},
             {101, &INx2VideoSession::Configure, "Configure"},
             {109, &INx2VideoSession::SubmitInput, "SubmitInputBuffer"},
-            {115, &INx2VideoSession::Succeed, "Finalize"},
+            {115, &INx2VideoSession::Finalize, "Finalize"},
             {200, &INx2VideoSession::Succeed, "Initialize"},
             {201, &INx2VideoSession::Succeed, "InitializeWithFlag"},
             {202, &INx2VideoSession::Succeed, "InitializeWithFlagAndValue"},
@@ -2419,6 +2419,10 @@ private:
         if (Nx2FirstVideoCommand(GetServiceName(), ctx.GetCommand())) {
             LOG_INFO(Service, "{} TIPC {} {} (first use): words {:08x}", GetServiceName(),
                      ctx.GetCommand(), what, fmt::join(std::span<const u32>{w, 12}, " "));
+        } else if (static const bool trace = std::getenv("EDEN_NX2_VDEC_TRACE") != nullptr; trace) {
+            // EDEN_NX2_VDEC_TRACE=1: every video decoder call at Info level.
+            LOG_INFO(Service, "{} TIPC {} {}: [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x}",
+                     GetServiceName(), ctx.GetCommand(), what, w[2], w[3], w[4], w[5]);
         } else {
             LOG_DEBUG(Service, "{} TIPC {} {}: [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x}",
                       GetServiceName(), ctx.GetCommand(), what, w[2], w[3], w[4], w[5]);
@@ -2447,10 +2451,31 @@ private:
     void Configure(HLERequestContext& ctx) {
         const auto* w = ctx.CommandBuffer();
         std::string codec;
+        std::span<const u8> name_buf;
         if (!ctx.BufferDescriptorA().empty()) {
-            const auto name = ctx.ReadBufferA(0);
-            codec.assign(reinterpret_cast<const char*>(name.data()),
-                         strnlen(reinterpret_cast<const char*>(name.data()), name.size()));
+            name_buf = ctx.ReadBufferA(0);
+            codec.assign(reinterpret_cast<const char*>(name_buf.data()),
+                         strnlen(reinterpret_cast<const char*>(name_buf.data()), name_buf.size()));
+        }
+        if (name_buf.empty() && !ctx.BufferDescriptorX().empty()) {
+            name_buf = ctx.ReadBufferX(0);
+            codec.assign(reinterpret_cast<const char*>(name_buf.data()),
+                         strnlen(reinterpret_cast<const char*>(name_buf.data()), name_buf.size()));
+        }
+        if (!name_buf.empty()) {
+            if (codec.empty()) {
+                const std::string_view sv{reinterpret_cast<const char*>(name_buf.data()), name_buf.size()};
+                for (const auto& candidate : {"video/avc", "video/vp9", "video/vp8", "video/hevc"}) {
+                    if (sv.find(candidate) != std::string_view::npos) {
+                        codec = candidate;
+                        break;
+                    }
+                }
+            }
+            if (codec.empty()) {
+                LOG_INFO(Service, "{} configure: no codec name in {:02x}", GetServiceName(),
+                         fmt::join(name_buf, " "));
+            }
         }
         Log(ctx, "configure");
         LOG_INFO(Service, "{} configure codec \"{}\" {}x{} rate {}", GetServiceName(), codec, w[5],
@@ -2513,7 +2538,24 @@ private:
             if (release_fence) {
                 m_release_fences[image] = *release_fence;
             }
-            value = m_sync_base + ++m_handed;
+            // After the end of a stream nothing fills an image again, so a release completes at
+            // once: its fence is the syncpoint's current value. Mario Kart World releases its
+            // images after a movie ends and waits for the returned fences; a fence for a frame
+            // that never comes hung the game on a black screen after its title movie.
+            static const bool release_now = [] {
+                const char* value = std::getenv("EDEN_NX2_VDEC_EOS_RELEASE_NOW");
+                return value != nullptr && value[0] != '0';
+            }();
+            if (release_now && release_fence && m_stream_ended && m_inputs.empty() &&
+                m_decoded.empty()) {
+                const u32 reached = static_cast<u32>(
+                    system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(m_syncpoint));
+                m_sync_base = reached;
+                m_handed = 0;
+                value = reached;
+            } else {
+                value = m_sync_base + ++m_handed;
+            }
         }
         m_cv.notify_one();
         return {~m_syncpoint, 0, value, 0};
@@ -2559,7 +2601,8 @@ private:
         {
             std::scoped_lock lk{m_mutex};
             m_inputs.push_back(std::move(input));
-            m_ready->room.store(m_inputs.size() < MaxQueuedInputs, std::memory_order_relaxed);
+            m_stream_ended = false;
+            m_ready->room.store(!end && m_inputs.size() < MaxQueuedInputs, std::memory_order_relaxed);
         }
         m_cv.notify_one();
         const u64 id = ++m_submitted;
@@ -2574,12 +2617,41 @@ private:
             std::scoped_lock lk{m_mutex};
             m_inputs.clear();
             m_flush = true;
+            m_stream_ended = false;
             m_ready->room.store(true, std::memory_order_relaxed);
+            // Only Stop: Bananza flushes (8) after its movies end and treats every completed
+            // fence as a frame to acquire.
+            if (ctx.GetCommand() == 207) {
+                CompleteHandedFences();
+            }
         }
         m_cv.notify_one();
         TipcReplyWords(ctx, {});
     }
 
+    void Finalize(HLERequestContext& ctx) {
+        Log(ctx, "finalize");
+        {
+            std::scoped_lock lk{m_mutex};
+            CompleteHandedFences();
+        }
+        TipcReplyWords(ctx, {});
+    }
+
+    // Advances the decoder's syncpoint to the last value handed out (m_mutex held).
+    void CompleteHandedFences() {
+        if (m_syncpoint == NoSyncpoint) {
+            return;
+        }
+        auto& syncpoints = system.Host1x().GetSyncpointManager();
+        const u32 target = m_sync_base + m_handed;
+        u32 current = static_cast<u32>(syncpoints.GetGuestSyncpointValue(m_syncpoint));
+        while (static_cast<s32>(target - current) > 0) {
+            syncpoints.IncrementGuest(m_syncpoint);
+            ++current;
+        }
+    }
+
     void GetOutputInfo(HLERequestContext& ctx) {
         Log(ctx, "output info");
         std::optional<Output> output;
@@ -2647,6 +2719,10 @@ private:
                 lk.lock();
                 // Even an image written during a flush counts: the client waits for its value.
                 m_outputs.push_back({image, width, height, frame.pts, frame.end});
+                if (frame.end) {
+                    m_stream_ended = true;
+                    m_ready->room.store(false, std::memory_order_relaxed);
+                }
                 system.Host1x().GetSyncpointManager().IncrementGuest(m_syncpoint);
                 ++m_frames_out;
                 if (m_frames_out == 1 || (m_frames_out % 600) == 0 || frame.end) {
@@ -2658,8 +2734,16 @@ private:
             }
             Input input = std::move(m_inputs.front());
             m_inputs.pop_front();
-            m_ready->room.store(m_inputs.size() < MaxQueuedInputs, std::memory_order_relaxed);
-            const std::string codec = m_codec;
+            if (!m_stream_ended) {
+                m_ready->room.store(m_inputs.size() < MaxQueuedInputs, std::memory_order_relaxed);
+            }
+            std::string codec = m_codec;
+            if (codec.empty() && Tegra::Host1x::IsH264Stream(input.data)) {
+                codec = "video/avc";
+                m_codec = codec;
+                LOG_INFO(Service, "{} empty codec, detected H.264 stream: using \"{}\"",
+                         GetServiceName(), codec);
+            }
             lk.unlock();
             if (!decoder || decoder->Name() != codec) {
                 decoder = std::make_unique<Tegra::Host1x::Nx2VideoDecoder>(codec);
@@ -2754,6 +2838,10 @@ private:
             memory.WriteBlock(address + layout.chroma_offset, bytes.data() + layout.chroma_offset,
                               layout.pitch * ((height + 1) / 2));
         }
+        // The game samples the image on the GPU: drop the texture cache's copy of the old frame.
+        if (const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address)) {
+            system.GPU().InvalidateRegion(*device, layout.size);
+        }
         return {width, height};
     }
 
@@ -2809,6 +2897,7 @@ private:
     std::deque<u32> m_free;            ///< Output images, in the order the client acquires them
     std::map<u32, Fence> m_release_fences;
     std::deque<Output> m_outputs;      ///< Finished, for 213
+    bool m_stream_ended{};             ///< End-of-stream frame handed out, nothing submitted since
     bool m_flush{};
     bool m_quit{};
     std::thread m_worker;
diff --git a/src/video_core/host1x/codec_types.h b/src/video_core/host1x/codec_types.h
index b2f80c822a..da9b713b26 100644
--- a/src/video_core/host1x/codec_types.h
+++ b/src/video_core/host1x/codec_types.h
@@ -312,6 +312,10 @@ struct Vp9PictureInfo {
     bool allow_high_precision_mv;
     bool segment_enabled;
     bool mode_ref_delta_enabled;
+
+    Segmentation segmentation;
+    std::array<u8, 7> seg_tree_probs;
+    std::array<u8, 3> seg_pred_probs;
 };
 
 struct Vp9FrameContainer {
@@ -377,6 +381,10 @@ struct PictureInfo {
             .allow_high_precision_mv = allow_high_precision_mv != 0,
             .segment_enabled = segmentation.enabled != 0,
             .mode_ref_delta_enabled = loop_filter.mode_ref_delta_enabled != 0,
+
+            .segmentation = segmentation,
+            .seg_tree_probs = {},
+            .seg_pred_probs = {},
         };
     }
 };
diff --git a/src/video_core/host1x/codecs/vp9.cpp b/src/video_core/host1x/codecs/vp9.cpp
index dcc6261e5c..dd91f7f1b7 100644
--- a/src/video_core/host1x/codecs/vp9.cpp
+++ b/src/video_core/host1x/codecs/vp9.cpp
@@ -331,19 +331,16 @@ void VP9::WriteMvProbabilityUpdate(VpxRangeEncoder& writer, u8 new_prob, u8 old_
 }
 
 void VP9::WriteSegmentation(VpxBitStreamWriter& writer) {
-    bool enabled = current_picture_info.segmentation.enabled != 0;
+    bool enabled = current_frame_info.segmentation.enabled != 0;
     writer.WriteBit(enabled);
     if (!enabled) {
         return;
     }
 
-    auto update_map = current_picture_info.segmentation.update_map != 0;
+    auto update_map = current_frame_info.segmentation.update_map != 0;
     writer.WriteBit(update_map);
 
     if (update_map) {
-        EntropyProbs entropy_probs{};
-        host1x.gmmu_manager.ReadBlock(regs.vp9_prob_tab_buffer_offset.Address(), &entropy_probs, sizeof(entropy_probs));
-
         auto WriteProb = [&](u8 prob) {
             bool coded = prob != 255;
             writer.WriteBit(coded);
@@ -352,55 +349,55 @@ void VP9::WriteSegmentation(VpxBitStreamWriter& writer) {
             }
         };
 
-        for (size_t i = 0; i < entropy_probs.mb_segment_tree_probs.size(); i++) {
-            WriteProb(entropy_probs.mb_segment_tree_probs[i]);
+        for (size_t i = 0; i < current_frame_info.seg_tree_probs.size(); i++) {
+            WriteProb(current_frame_info.seg_tree_probs[i]);
         }
 
-        auto temporal_update = current_picture_info.segmentation.temporal_update != 0;
+        auto temporal_update = current_frame_info.segmentation.temporal_update != 0;
         writer.WriteBit(temporal_update);
 
         if (temporal_update) {
             for (s32 i = 0; i < 3; i++) {
-                WriteProb(entropy_probs.segment_pred_probs[i]);
+                WriteProb(current_frame_info.seg_pred_probs[i]);
             }
         }
     }
 
-    if (last_segmentation == current_picture_info.segmentation) {
+    if (last_segmentation == current_frame_info.segmentation) {
         writer.WriteBit(false);
         return;
     }
 
-    last_segmentation = current_picture_info.segmentation;
+    last_segmentation = current_frame_info.segmentation;
     writer.WriteBit(true);
-    writer.WriteBit(current_picture_info.segmentation.abs_delta != 0);
+    writer.WriteBit(current_frame_info.segmentation.abs_delta != 0);
 
     constexpr s32 MAX_SEGMENTS = 8;
     constexpr std::array SegmentationFeatureBits = {8, 6, 2, 0};
 
     for (s32 i = 0; i < MAX_SEGMENTS; i++) {
-        auto q_enabled = current_picture_info.segmentation.feature_enabled[i][0] != 0;
+        auto q_enabled = current_frame_info.segmentation.feature_enabled[i][0] != 0;
         writer.WriteBit(q_enabled);
         if (q_enabled) {
-            writer.WriteS(current_picture_info.segmentation.feature_data[i][0],
+            writer.WriteS(current_frame_info.segmentation.feature_data[i][0],
                           SegmentationFeatureBits[0]);
         }
 
-        auto lf_enabled = current_picture_info.segmentation.feature_enabled[i][1] != 0;
+        auto lf_enabled = current_frame_info.segmentation.feature_enabled[i][1] != 0;
         writer.WriteBit(lf_enabled);
         if (lf_enabled) {
-            writer.WriteS(current_picture_info.segmentation.feature_data[i][1],
+            writer.WriteS(current_frame_info.segmentation.feature_data[i][1],
                           SegmentationFeatureBits[1]);
         }
 
-        auto ref_enabled = current_picture_info.segmentation.feature_enabled[i][2] != 0;
+        auto ref_enabled = current_frame_info.segmentation.feature_enabled[i][2] != 0;
         writer.WriteBit(ref_enabled);
         if (ref_enabled) {
-            writer.WriteU(current_picture_info.segmentation.feature_data[i][2],
+            writer.WriteU(current_frame_info.segmentation.feature_data[i][2],
                           SegmentationFeatureBits[2]);
         }
 
-        auto skip_enabled = current_picture_info.segmentation.feature_enabled[i][3] != 0;
+        auto skip_enabled = current_frame_info.segmentation.feature_enabled[i][3] != 0;
         writer.WriteBit(skip_enabled);
     }
 }
@@ -411,6 +408,14 @@ Vp9PictureInfo VP9::GetVp9PictureInfo() {
 
     InsertEntropy(regs.vp9_prob_tab_buffer_offset.Address(), vp9_info.entropy);
 
+    EntropyProbs entropy_probs{};
+    host1x.gmmu_manager.ReadBlock(regs.vp9_prob_tab_buffer_offset.Address(),
+                                  &entropy_probs, sizeof(entropy_probs));
+
+    vp9_info.seg_tree_probs = entropy_probs.mb_segment_tree_probs;
+    vp9_info.seg_pred_probs = entropy_probs.segment_pred_probs;
+
+
     // surface_luma_offset[0:3] contains the address of the reference frame offsets in the following
     // order: last, golden, altref, current.
     for (size_t i = 0; i < 4; i++) {
diff --git a/src/video_core/host1x/nx2_video_decoder.cpp b/src/video_core/host1x/nx2_video_decoder.cpp
index 9b34bd84af..86869ad5e2 100644
--- a/src/video_core/host1x/nx2_video_decoder.cpp
+++ b/src/video_core/host1x/nx2_video_decoder.cpp
@@ -47,6 +47,45 @@ std::optional<NvdecCommon::VideoCodec> CodecFromMime(std::string_view mime) {
 
 } // namespace
 
+bool IsH264Stream(std::span<const u8> data) {
+    if (data.size() >= 4) {
+        // Annex-B 3-byte start code: 00 00 01 <nal_header>
+        if (data[0] == 0 && data[1] == 0 && data[2] == 1) {
+            const u8 nal_type = data[3] & 0x1F;
+            if ((data[3] & 0x80) == 0 && nal_type >= 1 && nal_type <= 23) {
+                return true;
+            }
+        }
+    }
+    if (data.size() >= 5) {
+        // Annex-B 4-byte start code: 00 00 00 01 <nal_header>
+        if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1) {
+            const u8 nal_type = data[4] & 0x1F;
+            if ((data[4] & 0x80) == 0 && nal_type >= 1 && nal_type <= 23) {
+                return true;
+            }
+        }
+        // avcC / MP4 length-prefixed NAL unit: [u32 be length] <nal_header>
+        const u32 nalu_len = (static_cast<u32>(data[0]) << 24) |
+                             (static_cast<u32>(data[1]) << 16) |
+                             (static_cast<u32>(data[2]) << 8) |
+                             static_cast<u32>(data[3]);
+        if (nalu_len > 0 && nalu_len <= data.size() - 4) {
+            const u8 nal_type = data[4] & 0x1F;
+            if ((data[4] & 0x80) == 0 && nal_type >= 1 && nal_type <= 23) {
+                return true;
+            }
+        }
+    }
+    if (data.size() >= 7) {
+        // avcC configuration record: version=1, profile, compat, level, lengthSizeMinusOne | 0xFC, numSPS | 0xE0
+        if (data[0] == 1 && (data[4] & 0xFC) == 0xFC && (data[5] & 0xE0) == 0xE0 && (data[5] & 0x1F) > 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
 struct Nx2VideoDecoder::Impl {
     std::string name;
     std::optional<FFmpeg::Decoder> decoder;
@@ -123,6 +162,31 @@ void Nx2VideoDecoder::Decode(std::span<const u8> data, u64 pts, std::vector<Nx2P
     }
     impl->packet.assign(data.size() + AV_INPUT_BUFFER_PADDING_SIZE, 0);
     std::memcpy(impl->packet.data(), data.data(), data.size());
+
+    // Convert MP4/avcC length-prefixed NALUs to Annex-B start codes for FFmpeg if needed.
+    if (impl->name.find("avc") != std::string::npos || impl->name.find("264") != std::string::npos) {
+        const bool is_annex_b = data.size() >= 4 &&
+            ((data[0] == 0 && data[1] == 0 && data[2] == 1) ||
+             (data.size() >= 5 && data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1));
+        if (!is_annex_b && data.size() >= 5) {
+            size_t offset = 0;
+            while (offset + 4 < data.size()) {
+                const u32 nalu_len = (static_cast<u32>(impl->packet[offset]) << 24) |
+                                     (static_cast<u32>(impl->packet[offset + 1]) << 16) |
+                                     (static_cast<u32>(impl->packet[offset + 2]) << 8) |
+                                     static_cast<u32>(impl->packet[offset + 3]);
+                if (nalu_len == 0 || offset + 4 + nalu_len > data.size()) {
+                    break;
+                }
+                impl->packet[offset] = 0;
+                impl->packet[offset + 1] = 0;
+                impl->packet[offset + 2] = 0;
+                impl->packet[offset + 3] = 1;
+                offset += 4 + nalu_len;
+            }
+        }
+    }
+
     FFmpeg::Packet packet{std::span<const u8>{impl->packet.data(), data.size()}};
     packet.GetPacket()->pts = static_cast<s64>(pts);
     // Hidden VP9 frames never come out: keep the set from growing over a long movie.
@@ -162,6 +226,20 @@ void Nx2VideoDecoder::Render(const Nx2Picture& picture, Nx2PictureFormat format,
     const AVPixelFormat target = format == Nx2PictureFormat::Nv12    ? AV_PIX_FMT_NV12
                                  : format == Nx2PictureFormat::Rgba8 ? AV_PIX_FMT_RGBA
                                                                      : AV_PIX_FMT_BGRA;
+    // Frames copied back from a host GPU decoder are already NV12. Copy them as they are:
+    // FFmpeg 6.1's swscale leaves the chroma plane of an NV12 to NV12 copy at zero, which shows
+    // the movies green.
+    if (target == AV_PIX_FMT_NV12 && f->format == AV_PIX_FMT_NV12 &&
+        static_cast<u32>(f->width) == width && static_cast<u32>(f->height) == height) {
+        for (u32 y = 0; y < height; ++y) {
+            std::memcpy(out.data() + y * pitch, f->data[0] + y * f->linesize[0], width);
+        }
+        for (u32 y = 0; y < (height + 1) / 2; ++y) {
+            std::memcpy(out.data() + chroma_offset + y * pitch, f->data[1] + y * f->linesize[1],
+                        width);
+        }
+        return;
+    }
     impl->scaler = sws_getCachedContext(impl->scaler, f->width, f->height,
                                         static_cast<AVPixelFormat>(f->format),
                                         static_cast<int>(width), static_cast<int>(height), target,
diff --git a/src/video_core/host1x/nx2_video_decoder.h b/src/video_core/host1x/nx2_video_decoder.h
index c979a2c801..347b0e6525 100644
--- a/src/video_core/host1x/nx2_video_decoder.h
+++ b/src/video_core/host1x/nx2_video_decoder.h
@@ -32,6 +32,9 @@ enum class Nx2PictureFormat : u32 {
     Bgra8,
 };
 
+/// Checks if stream data begins with an H.264 Annex-B start code or avcC/MP4 length-prefixed NALUs.
+bool IsH264Stream(std::span<const u8> data);
+
 /// The decoder behind the Switch 2 video decoder service (nn::codec::video): FFmpeg decodes the
 /// elementary stream the game demuxed, in software or on the host GPU (NVDEC emulation setting).
 /// Used from one thread at a time.