nx2: vdecsvc picks H.264 when the configure carries no codec name and the stream is H.264 (Yoshi demo)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

3 files changed, 98 insertions, 4 deletions

diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h
index e488be098c..3e29076b8e 100644
--- a/src/core/hle/service/nx2_gsv.h
+++ b/src/core/hle/service/nx2_gsv.h
@@ -2447,10 +2447,31 @@ private:
     void Configure(HLERequestContext& ctx) {
         const auto* w = ctx.CommandBuffer();
         std::string codec;
+        std::span<const u8> name_buf;
         if (!ctx.BufferDescriptorA().empty()) {
-            const auto name = ctx.ReadBufferA(0);
-            codec.assign(reinterpret_cast<const char*>(name.data()),
-                         strnlen(reinterpret_cast<const char*>(name.data()), name.size()));
+            name_buf = ctx.ReadBufferA(0);
+            codec.assign(reinterpret_cast<const char*>(name_buf.data()),
+                         strnlen(reinterpret_cast<const char*>(name_buf.data()), name_buf.size()));
+        }
+        if (name_buf.empty() && !ctx.BufferDescriptorX().empty()) {
+            name_buf = ctx.ReadBufferX(0);
+            codec.assign(reinterpret_cast<const char*>(name_buf.data()),
+                         strnlen(reinterpret_cast<const char*>(name_buf.data()), name_buf.size()));
+        }
+        if (!name_buf.empty()) {
+            if (codec.empty()) {
+                const std::string_view sv{reinterpret_cast<const char*>(name_buf.data()), name_buf.size()};
+                for (const auto& candidate : {"video/avc", "video/vp9", "video/vp8", "video/hevc"}) {
+                    if (sv.find(candidate) != std::string_view::npos) {
+                        codec = candidate;
+                        break;
+                    }
+                }
+            }
+            if (codec.empty()) {
+                LOG_INFO(Service, "{} configure: no codec name in {:02x}", GetServiceName(),
+                         fmt::join(name_buf, " "));
+            }
         }
         Log(ctx, "configure");
         LOG_INFO(Service, "{} configure codec \"{}\" {}x{} rate {}", GetServiceName(), codec, w[5],
@@ -2679,7 +2700,13 @@ private:
             if (!m_stream_ended) {
                 m_ready->room.store(m_inputs.size() < MaxQueuedInputs, std::memory_order_relaxed);
             }
-            const std::string codec = m_codec;
+            std::string codec = m_codec;
+            if (codec.empty() && Tegra::Host1x::IsH264Stream(input.data)) {
+                codec = "video/avc";
+                m_codec = codec;
+                LOG_INFO(Service, "{} empty codec, detected H.264 stream: using \"{}\"",
+                         GetServiceName(), codec);
+            }
             lk.unlock();
             if (!decoder || decoder->Name() != codec) {
                 decoder = std::make_unique<Tegra::Host1x::Nx2VideoDecoder>(codec);
diff --git a/src/video_core/host1x/nx2_video_decoder.cpp b/src/video_core/host1x/nx2_video_decoder.cpp
index 9b34bd84af..87265ee6f6 100644
--- a/src/video_core/host1x/nx2_video_decoder.cpp
+++ b/src/video_core/host1x/nx2_video_decoder.cpp
@@ -47,6 +47,45 @@ std::optional<NvdecCommon::VideoCodec> CodecFromMime(std::string_view mime) {
 
 } // namespace
 
+bool IsH264Stream(std::span<const u8> data) {
+    if (data.size() >= 4) {
+        // Annex-B 3-byte start code: 00 00 01 <nal_header>
+        if (data[0] == 0 && data[1] == 0 && data[2] == 1) {
+            const u8 nal_type = data[3] & 0x1F;
+            if ((data[3] & 0x80) == 0 && nal_type >= 1 && nal_type <= 23) {
+                return true;
+            }
+        }
+    }
+    if (data.size() >= 5) {
+        // Annex-B 4-byte start code: 00 00 00 01 <nal_header>
+        if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1) {
+            const u8 nal_type = data[4] & 0x1F;
+            if ((data[4] & 0x80) == 0 && nal_type >= 1 && nal_type <= 23) {
+                return true;
+            }
+        }
+        // avcC / MP4 length-prefixed NAL unit: [u32 be length] <nal_header>
+        const u32 nalu_len = (static_cast<u32>(data[0]) << 24) |
+                             (static_cast<u32>(data[1]) << 16) |
+                             (static_cast<u32>(data[2]) << 8) |
+                             static_cast<u32>(data[3]);
+        if (nalu_len > 0 && nalu_len <= data.size() - 4) {
+            const u8 nal_type = data[4] & 0x1F;
+            if ((data[4] & 0x80) == 0 && nal_type >= 1 && nal_type <= 23) {
+                return true;
+            }
+        }
+    }
+    if (data.size() >= 7) {
+        // avcC configuration record: version=1, profile, compat, level, lengthSizeMinusOne | 0xFC, numSPS | 0xE0
+        if (data[0] == 1 && (data[4] & 0xFC) == 0xFC && (data[5] & 0xE0) == 0xE0 && (data[5] & 0x1F) > 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
 struct Nx2VideoDecoder::Impl {
     std::string name;
     std::optional<FFmpeg::Decoder> decoder;
@@ -123,6 +162,31 @@ void Nx2VideoDecoder::Decode(std::span<const u8> data, u64 pts, std::vector<Nx2P
     }
     impl->packet.assign(data.size() + AV_INPUT_BUFFER_PADDING_SIZE, 0);
     std::memcpy(impl->packet.data(), data.data(), data.size());
+
+    // Convert MP4/avcC length-prefixed NALUs to Annex-B start codes for FFmpeg if needed.
+    if (impl->name.find("avc") != std::string::npos || impl->name.find("264") != std::string::npos) {
+        const bool is_annex_b = data.size() >= 4 &&
+            ((data[0] == 0 && data[1] == 0 && data[2] == 1) ||
+             (data.size() >= 5 && data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1));
+        if (!is_annex_b && data.size() >= 5) {
+            size_t offset = 0;
+            while (offset + 4 < data.size()) {
+                const u32 nalu_len = (static_cast<u32>(impl->packet[offset]) << 24) |
+                                     (static_cast<u32>(impl->packet[offset + 1]) << 16) |
+                                     (static_cast<u32>(impl->packet[offset + 2]) << 8) |
+                                     static_cast<u32>(impl->packet[offset + 3]);
+                if (nalu_len == 0 || offset + 4 + nalu_len > data.size()) {
+                    break;
+                }
+                impl->packet[offset] = 0;
+                impl->packet[offset + 1] = 0;
+                impl->packet[offset + 2] = 0;
+                impl->packet[offset + 3] = 1;
+                offset += 4 + nalu_len;
+            }
+        }
+    }
+
     FFmpeg::Packet packet{std::span<const u8>{impl->packet.data(), data.size()}};
     packet.GetPacket()->pts = static_cast<s64>(pts);
     // Hidden VP9 frames never come out: keep the set from growing over a long movie.
diff --git a/src/video_core/host1x/nx2_video_decoder.h b/src/video_core/host1x/nx2_video_decoder.h
index c979a2c801..347b0e6525 100644
--- a/src/video_core/host1x/nx2_video_decoder.h
+++ b/src/video_core/host1x/nx2_video_decoder.h
@@ -32,6 +32,9 @@ enum class Nx2PictureFormat : u32 {
     Bgra8,
 };
 
+/// Checks if stream data begins with an H.264 Annex-B start code or avcC/MP4 length-prefixed NALUs.
+bool IsH264Stream(std::span<const u8> data);
+
 /// The decoder behind the Switch 2 video decoder service (nn::codec::video): FFmpeg decodes the
 /// elementary stream the game demuxed, in software or on the host GPU (NVDEC emulation setting).
 /// Used from one thread at a time.