From c30df71ee0752005e0be3bbf2f3233b65f38cabc Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:55:22 -0400 Subject: nx2: decode Switch 2 video (nn::codec::video) with FFmpeg The decoder service used to complete every submitted frame at once without decoding it. Each decoder service (gsv 902/906/1002 -> TIPC 1) now decodes on a thread of its own with FFmpeg (VP9, VP8, H.264; video_core/host1x/nx2_video_decoder), in software or on the host GPU per the NVDEC emulation setting. - 101 takes the codec name ("video/vp9") from its A buffer. - 109 queues the A buffer's frame; the ready event fires while the queue has room. - 311/314 reply a NativeSyncHandle {~syncpoint, 0, value}: the service allocates a syncpoint and increments it per finished frame, frames filling output images in register/release order like the client's ring. 314 waits for the client's release fence before reuse. - 213 returns the frame's size, pts (TimeSpan, ns) and end of stream. - gsv 911 creates a decoder image pool (count images in one memory) and gsv 961 returns the 128-byte CpuUserImageDescriptor that nn::codec::ReadDecoderOutputImage* copy frames out with; images are pitch-linear NV12, RGBA or BGRA. - Each new video command is logged once in full; EDEN_NX2_VDEC_DUMP= saves every 120th decoded frame's luma. Title 1's Mushroom Cup opening (2560x1440 VP9, 1423 frames) now decodes in real time and the game goes on to the race. The picture on screen still stays on the first frame: the game memcpys each frame into GPU memory and the texture cache keeps its first upload (notes 148). --- src/video_core/CMakeLists.txt | 2 + src/video_core/host1x/nx2_video_decoder.cpp | 193 ++++++++++++++++++++++++++++ src/video_core/host1x/nx2_video_decoder.h | 67 ++++++++++ 3 files changed, 262 insertions(+) create mode 100644 src/video_core/host1x/nx2_video_decoder.cpp create mode 100644 src/video_core/host1x/nx2_video_decoder.h (limited to 'src/video_core') diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index e619c4f2de..7f83e0d3c2 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -84,6 +84,8 @@ add_library(video_core STATIC host1x/nvdec.cpp host1x/nvdec.h host1x/nvdec_common.h + host1x/nx2_video_decoder.cpp + host1x/nx2_video_decoder.h host1x/syncpoint_manager.cpp host1x/syncpoint_manager.h host1x/vic.cpp diff --git a/src/video_core/host1x/nx2_video_decoder.cpp b/src/video_core/host1x/nx2_video_decoder.cpp new file mode 100644 index 0000000000..7d185ca573 --- /dev/null +++ b/src/video_core/host1x/nx2_video_decoder.cpp @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include +#include + +#include "common/logging.h" +#include "common/settings.h" +#include "video_core/host1x/ffmpeg.h" +#include "video_core/host1x/nx2_video_decoder.h" + +extern "C" { +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + +#include + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif +} + +namespace Tegra::Host1x { + +namespace { + +std::optional CodecFromMime(std::string_view mime) { + if (mime.find("vp9") != std::string_view::npos) { + return NvdecCommon::VideoCodec::VP9; + } + if (mime.find("vp8") != std::string_view::npos) { + return NvdecCommon::VideoCodec::VP8; + } + if (mime.find("avc") != std::string_view::npos || mime.find("264") != std::string_view::npos) { + return NvdecCommon::VideoCodec::H264; + } + return std::nullopt; +} + +} // namespace + +struct Nx2VideoDecoder::Impl { + std::string name; + std::optional decoder; + std::optional hardware; + std::optional context; + SwsContext* scaler{}; + bool open{}; + /// Timestamps of submitted frames not yet output. Frames copied back from a host GPU decoder + /// lose theirs and take the earliest one instead. + std::multiset pending_pts; + std::vector packet; ///< FFmpeg reads up to AV_INPUT_BUFFER_PADDING_SIZE past the data + + ~Impl() { + sws_freeContext(scaler); + context.reset(); + hardware.reset(); + } + + void Receive(std::vector& pictures) { + while (auto frame = context->ReceiveFrame()) { + const AVFrame* f = frame->GetFrame(); + const s64 stamp = f->best_effort_timestamp != AV_NOPTS_VALUE ? f->best_effort_timestamp + : f->pts; + u64 pts{}; + if (stamp != AV_NOPTS_VALUE) { + pts = static_cast(stamp); + if (const auto it = pending_pts.find(pts); it != pending_pts.end()) { + pending_pts.erase(it); + } + } else if (!pending_pts.empty()) { + pts = *pending_pts.begin(); + pending_pts.erase(pending_pts.begin()); + } + pictures.push_back({static_cast(f->width), static_cast(f->height), pts, + std::move(frame)}); + } + } +}; + +Nx2VideoDecoder::Nx2VideoDecoder(std::string_view mime) : impl{std::make_unique()} { + impl->name = std::string{mime}; + const auto codec = CodecFromMime(mime); + if (!codec) { + LOG_ERROR(HW_GPU, "nx2 video: no decoder for codec \"{}\"", mime); + return; + } + impl->decoder.emplace(*codec); + if (impl->decoder->GetCodec() == nullptr) { + LOG_ERROR(HW_GPU, "nx2 video: FFmpeg has no decoder for \"{}\"", mime); + return; + } + impl->context.emplace(*impl->decoder); + if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) { + impl->hardware.emplace(); + impl->hardware->InitializeForDecoder(*impl->context, *impl->decoder); + } + impl->open = impl->context->OpenContext(*impl->decoder); + LOG_INFO(HW_GPU, "nx2 video: decoder for \"{}\" {}", mime, impl->open ? "open" : "failed to open"); +} + +Nx2VideoDecoder::~Nx2VideoDecoder() = default; + +bool Nx2VideoDecoder::IsOpen() const { + return impl->open; +} + +const std::string& Nx2VideoDecoder::Name() const { + return impl->name; +} + +void Nx2VideoDecoder::Decode(std::span data, u64 pts, std::vector& pictures) { + if (!impl->open || data.empty()) { + return; + } + impl->packet.assign(data.size() + AV_INPUT_BUFFER_PADDING_SIZE, 0); + std::memcpy(impl->packet.data(), data.data(), data.size()); + FFmpeg::Packet packet{std::span{impl->packet.data(), data.size()}}; + packet.GetPacket()->pts = static_cast(pts); + // Hidden VP9 frames never come out: keep the set from growing over a long movie. + if (impl->pending_pts.size() >= 64) { + impl->pending_pts.erase(impl->pending_pts.begin()); + } + impl->pending_pts.insert(pts); + impl->context->SendPacket(packet); + impl->Receive(pictures); +} + +void Nx2VideoDecoder::Drain(std::vector& pictures) { + if (!impl->open) { + return; + } + avcodec_send_packet(impl->context->GetCodecContext(), nullptr); + impl->Receive(pictures); + Reset(); +} + +void Nx2VideoDecoder::Reset() { + if (impl->open) { + avcodec_flush_buffers(impl->context->GetCodecContext()); + } + impl->pending_pts.clear(); +} + +void Nx2VideoDecoder::Render(const Nx2Picture& picture, Nx2PictureFormat format, u32 width, + u32 height, u64 pitch, u64 chroma_offset, std::span out) { + const AVFrame* f = picture.frame->GetFrame(); + const u64 needed = format == Nx2PictureFormat::Nv12 + ? chroma_offset + pitch * ((u64{height} + 1) / 2) + : pitch * height; + if (width == 0 || height == 0 || out.size() < needed) { + return; + } + const AVPixelFormat target = format == Nx2PictureFormat::Nv12 ? AV_PIX_FMT_NV12 + : format == Nx2PictureFormat::Rgba8 ? AV_PIX_FMT_RGBA + : AV_PIX_FMT_BGRA; + impl->scaler = sws_getCachedContext(impl->scaler, f->width, f->height, + static_cast(f->format), + static_cast(width), static_cast(height), target, + SWS_BILINEAR, nullptr, nullptr, nullptr); + if (impl->scaler == nullptr) { + LOG_ERROR(HW_GPU, "nx2 video: cannot convert {} to {}", + av_get_pix_fmt_name(static_cast(f->format)), + av_get_pix_fmt_name(target)); + return; + } + if (format != Nx2PictureFormat::Nv12) { + // The stream's matrix and range; untagged HD video is BT.709. + int space = SWS_CS_ITU601; + if (f->colorspace == AVCOL_SPC_BT709 || + (f->colorspace == AVCOL_SPC_UNSPECIFIED && f->height >= 720)) { + space = SWS_CS_ITU709; + } else if (f->colorspace == AVCOL_SPC_BT2020_NCL || f->colorspace == AVCOL_SPC_BT2020_CL) { + space = SWS_CS_BT2020; + } + const int* coefficients = sws_getCoefficients(space); + sws_setColorspaceDetails(impl->scaler, coefficients, f->color_range == AVCOL_RANGE_JPEG, + coefficients, 1, 0, 1 << 16, 1 << 16); + } + std::array planes{out.data(), nullptr, nullptr, nullptr}; + std::array strides{static_cast(pitch), 0, 0, 0}; + if (format == Nx2PictureFormat::Nv12) { + planes[1] = out.data() + chroma_offset; + strides[1] = static_cast(pitch); + } + sws_scale(impl->scaler, f->data, f->linesize, 0, f->height, planes.data(), strides.data()); +} + +} // namespace Tegra::Host1x diff --git a/src/video_core/host1x/nx2_video_decoder.h b/src/video_core/host1x/nx2_video_decoder.h new file mode 100644 index 0000000000..c979a2c801 --- /dev/null +++ b/src/video_core/host1x/nx2_video_decoder.h @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/common_types.h" + +namespace FFmpeg { +class Frame; +} + +namespace Tegra::Host1x { + +/// A picture decoded by Nx2VideoDecoder, in display order. +struct Nx2Picture { + u32 width{}; + u32 height{}; + u64 pts{}; ///< As submitted with the compressed frame + std::shared_ptr frame; +}; + +/// Pixel layouts of a Switch 2 decoder output image. +enum class Nx2PictureFormat : u32 { + Nv12, ///< Luma rows, then interleaved Cb/Cr rows at chroma_offset + Rgba8, + Bgra8, +}; + +/// The decoder behind the Switch 2 video decoder service (nn::codec::video): FFmpeg decodes the +/// elementary stream the game demuxed, in software or on the host GPU (NVDEC emulation setting). +/// Used from one thread at a time. +class Nx2VideoDecoder { +public: + /// mime: the codec name nn::codec sends with Configure ("video/vp9", "video/vp8", ...). + explicit Nx2VideoDecoder(std::string_view mime); + ~Nx2VideoDecoder(); + + bool IsOpen() const; + + const std::string& Name() const; + + /// Decodes one compressed frame and appends the pictures it completes. + void Decode(std::span data, u64 pts, std::vector& pictures); + + /// End of stream: appends the pictures the decoder still holds, then starts over. + void Drain(std::vector& pictures); + + /// Drops everything queued in the decoder (the game flushed it to seek). + void Reset(); + + /// Converts a picture to `format` in `out`: rows `pitch` bytes apart, NV12's chroma plane at + /// `chroma_offset`, scaled to width x height. + void Render(const Nx2Picture& picture, Nx2PictureFormat format, u32 width, u32 height, + u64 pitch, u64 chroma_offset, std::span out); + +private: + struct Impl; + std::unique_ptr impl; +}; + +} // namespace Tegra::Host1x -- cgit v1.2.3