From c30df71ee0752005e0be3bbf2f3233b65f38cabc Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:55:22 -0400 Subject: nx2: decode Switch 2 video (nn::codec::video) with FFmpeg The decoder service used to complete every submitted frame at once without decoding it. Each decoder service (gsv 902/906/1002 -> TIPC 1) now decodes on a thread of its own with FFmpeg (VP9, VP8, H.264; video_core/host1x/nx2_video_decoder), in software or on the host GPU per the NVDEC emulation setting. - 101 takes the codec name ("video/vp9") from its A buffer. - 109 queues the A buffer's frame; the ready event fires while the queue has room. - 311/314 reply a NativeSyncHandle {~syncpoint, 0, value}: the service allocates a syncpoint and increments it per finished frame, frames filling output images in register/release order like the client's ring. 314 waits for the client's release fence before reuse. - 213 returns the frame's size, pts (TimeSpan, ns) and end of stream. - gsv 911 creates a decoder image pool (count images in one memory) and gsv 961 returns the 128-byte CpuUserImageDescriptor that nn::codec::ReadDecoderOutputImage* copy frames out with; images are pitch-linear NV12, RGBA or BGRA. - Each new video command is logged once in full; EDEN_NX2_VDEC_DUMP= saves every 120th decoded frame's luma. Title 1's Mushroom Cup opening (2560x1440 VP9, 1423 frames) now decodes in real time and the game goes on to the race. The picture on screen still stays on the first frame: the game memcpys each frame into GPU memory and the texture cache keeps its first upload (notes 148). --- src/video_core/host1x/nx2_video_decoder.cpp | 193 ++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 src/video_core/host1x/nx2_video_decoder.cpp (limited to 'src/video_core/host1x/nx2_video_decoder.cpp') diff --git a/src/video_core/host1x/nx2_video_decoder.cpp b/src/video_core/host1x/nx2_video_decoder.cpp new file mode 100644 index 0000000000..7d185ca573 --- /dev/null +++ b/src/video_core/host1x/nx2_video_decoder.cpp @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include +#include + +#include "common/logging.h" +#include "common/settings.h" +#include "video_core/host1x/ffmpeg.h" +#include "video_core/host1x/nx2_video_decoder.h" + +extern "C" { +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + +#include + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif +} + +namespace Tegra::Host1x { + +namespace { + +std::optional CodecFromMime(std::string_view mime) { + if (mime.find("vp9") != std::string_view::npos) { + return NvdecCommon::VideoCodec::VP9; + } + if (mime.find("vp8") != std::string_view::npos) { + return NvdecCommon::VideoCodec::VP8; + } + if (mime.find("avc") != std::string_view::npos || mime.find("264") != std::string_view::npos) { + return NvdecCommon::VideoCodec::H264; + } + return std::nullopt; +} + +} // namespace + +struct Nx2VideoDecoder::Impl { + std::string name; + std::optional decoder; + std::optional hardware; + std::optional context; + SwsContext* scaler{}; + bool open{}; + /// Timestamps of submitted frames not yet output. Frames copied back from a host GPU decoder + /// lose theirs and take the earliest one instead. + std::multiset pending_pts; + std::vector packet; ///< FFmpeg reads up to AV_INPUT_BUFFER_PADDING_SIZE past the data + + ~Impl() { + sws_freeContext(scaler); + context.reset(); + hardware.reset(); + } + + void Receive(std::vector& pictures) { + while (auto frame = context->ReceiveFrame()) { + const AVFrame* f = frame->GetFrame(); + const s64 stamp = f->best_effort_timestamp != AV_NOPTS_VALUE ? f->best_effort_timestamp + : f->pts; + u64 pts{}; + if (stamp != AV_NOPTS_VALUE) { + pts = static_cast(stamp); + if (const auto it = pending_pts.find(pts); it != pending_pts.end()) { + pending_pts.erase(it); + } + } else if (!pending_pts.empty()) { + pts = *pending_pts.begin(); + pending_pts.erase(pending_pts.begin()); + } + pictures.push_back({static_cast(f->width), static_cast(f->height), pts, + std::move(frame)}); + } + } +}; + +Nx2VideoDecoder::Nx2VideoDecoder(std::string_view mime) : impl{std::make_unique()} { + impl->name = std::string{mime}; + const auto codec = CodecFromMime(mime); + if (!codec) { + LOG_ERROR(HW_GPU, "nx2 video: no decoder for codec \"{}\"", mime); + return; + } + impl->decoder.emplace(*codec); + if (impl->decoder->GetCodec() == nullptr) { + LOG_ERROR(HW_GPU, "nx2 video: FFmpeg has no decoder for \"{}\"", mime); + return; + } + impl->context.emplace(*impl->decoder); + if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) { + impl->hardware.emplace(); + impl->hardware->InitializeForDecoder(*impl->context, *impl->decoder); + } + impl->open = impl->context->OpenContext(*impl->decoder); + LOG_INFO(HW_GPU, "nx2 video: decoder for \"{}\" {}", mime, impl->open ? "open" : "failed to open"); +} + +Nx2VideoDecoder::~Nx2VideoDecoder() = default; + +bool Nx2VideoDecoder::IsOpen() const { + return impl->open; +} + +const std::string& Nx2VideoDecoder::Name() const { + return impl->name; +} + +void Nx2VideoDecoder::Decode(std::span data, u64 pts, std::vector& pictures) { + if (!impl->open || data.empty()) { + return; + } + impl->packet.assign(data.size() + AV_INPUT_BUFFER_PADDING_SIZE, 0); + std::memcpy(impl->packet.data(), data.data(), data.size()); + FFmpeg::Packet packet{std::span{impl->packet.data(), data.size()}}; + packet.GetPacket()->pts = static_cast(pts); + // Hidden VP9 frames never come out: keep the set from growing over a long movie. + if (impl->pending_pts.size() >= 64) { + impl->pending_pts.erase(impl->pending_pts.begin()); + } + impl->pending_pts.insert(pts); + impl->context->SendPacket(packet); + impl->Receive(pictures); +} + +void Nx2VideoDecoder::Drain(std::vector& pictures) { + if (!impl->open) { + return; + } + avcodec_send_packet(impl->context->GetCodecContext(), nullptr); + impl->Receive(pictures); + Reset(); +} + +void Nx2VideoDecoder::Reset() { + if (impl->open) { + avcodec_flush_buffers(impl->context->GetCodecContext()); + } + impl->pending_pts.clear(); +} + +void Nx2VideoDecoder::Render(const Nx2Picture& picture, Nx2PictureFormat format, u32 width, + u32 height, u64 pitch, u64 chroma_offset, std::span out) { + const AVFrame* f = picture.frame->GetFrame(); + const u64 needed = format == Nx2PictureFormat::Nv12 + ? chroma_offset + pitch * ((u64{height} + 1) / 2) + : pitch * height; + if (width == 0 || height == 0 || out.size() < needed) { + return; + } + const AVPixelFormat target = format == Nx2PictureFormat::Nv12 ? AV_PIX_FMT_NV12 + : format == Nx2PictureFormat::Rgba8 ? AV_PIX_FMT_RGBA + : AV_PIX_FMT_BGRA; + impl->scaler = sws_getCachedContext(impl->scaler, f->width, f->height, + static_cast(f->format), + static_cast(width), static_cast(height), target, + SWS_BILINEAR, nullptr, nullptr, nullptr); + if (impl->scaler == nullptr) { + LOG_ERROR(HW_GPU, "nx2 video: cannot convert {} to {}", + av_get_pix_fmt_name(static_cast(f->format)), + av_get_pix_fmt_name(target)); + return; + } + if (format != Nx2PictureFormat::Nv12) { + // The stream's matrix and range; untagged HD video is BT.709. + int space = SWS_CS_ITU601; + if (f->colorspace == AVCOL_SPC_BT709 || + (f->colorspace == AVCOL_SPC_UNSPECIFIED && f->height >= 720)) { + space = SWS_CS_ITU709; + } else if (f->colorspace == AVCOL_SPC_BT2020_NCL || f->colorspace == AVCOL_SPC_BT2020_CL) { + space = SWS_CS_BT2020; + } + const int* coefficients = sws_getCoefficients(space); + sws_setColorspaceDetails(impl->scaler, coefficients, f->color_range == AVCOL_RANGE_JPEG, + coefficients, 1, 0, 1 << 16, 1 << 16); + } + std::array planes{out.data(), nullptr, nullptr, nullptr}; + std::array strides{static_cast(pitch), 0, 0, 0}; + if (format == Nx2PictureFormat::Nv12) { + planes[1] = out.data() + chroma_offset; + strides[1] = static_cast(pitch); + } + sws_scale(impl->scaler, f->data, f->linesize, 0, f->height, planes.data(), strides.data()); +} + +} // namespace Tegra::Host1x -- cgit v1.2.3