diff options
| author | Takoroka <137927095+Takoroka@users.noreply.github.com> | 2026-10-29 15:55:22 -0400 |
|---|---|---|
| committer | Takoroka <137927095+Takoroka@users.noreply.github.com> | 2026-10-29 20:49:48 -0400 |
| commit | c30df71ee0752005e0be3bbf2f3233b65f38cabc (patch) | |
| tree | ea10dd41f86ce4e44c65574a621fdc0dfe17fde7 /src | |
| parent | 158e3733f25d61cfaeebbf312cafb389fb8e84f3 (diff) | |
nx2: decode Switch 2 video (nn::codec::video) with FFmpeg
The decoder service used to complete every submitted frame at once
without decoding it. Each decoder service (gsv 902/906/1002 -> TIPC 1)
now decodes on a thread of its own with FFmpeg (VP9, VP8, H.264;
video_core/host1x/nx2_video_decoder), in software or on the host GPU per
the NVDEC emulation setting.
- 101 takes the codec name ("video/vp9") from its A buffer.
- 109 queues the A buffer's frame; the ready event fires while the queue
has room.
- 311/314 reply a NativeSyncHandle {~syncpoint, 0, value}: the service
allocates a syncpoint and increments it per finished frame, frames
filling output images in register/release order like the client's
ring. 314 waits for the client's release fence before reuse.
- 213 returns the frame's size, pts (TimeSpan, ns) and end of stream.
- gsv 911 creates a decoder image pool (count images in one memory) and
gsv 961 returns the 128-byte CpuUserImageDescriptor that
nn::codec::ReadDecoderOutputImage* copy frames out with; images are
pitch-linear NV12, RGBA or BGRA.
- Each new video command is logged once in full; EDEN_NX2_VDEC_DUMP=<dir>
saves every 120th decoded frame's luma.
Title 1's Mushroom Cup opening (2560x1440 VP9, 1423 frames) now decodes
in real time and the game goes on to the race. The picture on screen
still stays on the first frame: the game memcpys each frame into GPU
memory and the texture cache keeps its first upload (notes 148).
Diffstat (limited to 'src')
| -rw-r--r-- | src/core/hle/service/nx2_gsv.h | 534 | ||||
| -rw-r--r-- | src/video_core/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | src/video_core/host1x/nx2_video_decoder.cpp | 193 | ||||
| -rw-r--r-- | src/video_core/host1x/nx2_video_decoder.h | 67 |
4 files changed, 723 insertions, 73 deletions
diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h index 9a90a27f79..cfef3f1229 100644 --- a/src/core/hle/service/nx2_gsv.h +++ b/src/core/hle/service/nx2_gsv.h @@ -7,6 +7,10 @@ #include <array> #include <thread> #include <chrono> +#include <condition_variable> +#include <cstring> +#include <set> +#include <span> #include <string> #include <cstdlib> #include <cstdio> @@ -21,6 +25,7 @@ #include "common/alignment.h" #include "common/settings.h" +#include "common/thread.h" #include "core/core.h" #include "video_core/rasterizer_interface.h" #include "video_core/renderer_base.h" @@ -43,10 +48,13 @@ #include "core/hle/kernel/k_transfer_memory.h" #include "core/memory.h" #include "video_core/host1x/host1x.h" +#include "video_core/host1x/nx2_video_decoder.h" #include "video_core/nx2.h" #include "core/hle/service/hle_ipc.h" #include "core/hle/service/ipc_helpers.h" #include "core/hle/service/kernel_helpers.h" +#include "core/hle/service/nvdrv/core/container.h" +#include "core/hle/service/nvdrv/core/syncpoint_manager.h" #include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/hle/service/nvdrv/nvdrv.h" #include "core/hle/service/nvdrv/nvdrv_interface.h" @@ -80,7 +88,7 @@ inline Kernel::KClientSession* CreateRawSession(Core::System& system, HLERequest // Replies to a TIPC request with success and raw u32 data words. ResponseBuilder::Push(Result) // writes the CMIF two-word result, which shifts TIPC data one word past where the client reads // it (and past the copied size); TIPC has a one-word result followed directly by the data. -inline void TipcReplyWords(HLERequestContext& ctx, std::initializer_list<u32> words) { +inline void TipcReplyWords(HLERequestContext& ctx, std::span<const u32> words) { IPC::ResponseBuilder rb{ctx, static_cast<u32>(2 + words.size())}; rb.PushRaw<u32>(ResultSuccess.raw); for (const u32 w : words) { @@ -88,6 +96,10 @@ inline void TipcReplyWords(HLERequestContext& ctx, std::initializer_list<u32> wo } } +inline void TipcReplyWords(HLERequestContext& ctx, std::initializer_list<u32> words) { + TipcReplyWords(ctx, std::span<const u32>{words.begin(), words.size()}); +} + // Replies with success and a copied handle to a new session served by `iface`. template <class T> inline void ReturnRawSession(Core::System& system, HLERequestContext& ctx, std::shared_ptr<T> iface) { @@ -640,7 +652,9 @@ struct Nx2NativeObjects { std::array<u32, 16> preference{}; u64 size{}; u32 count{1}; // images in the pool; handle id + i is image i + u64 offset{}; // into its memory (images of a decoder pool share one) bool window_pool{}; + bool decoder{}; // a video decoder output image (gsv 911) }; std::mutex mutex; std::map<u32, Memory> memories; @@ -724,6 +738,50 @@ private: std::map<u32, Kernel::KEvent*> m_events; }; +// Video decoder output images (nn::codec image pools, gsv 991/911/961). The preference +// (MakeDecodeImagePreference) holds an output layout code at word 1 and (width - 1) | +// (height - 1) << 16 at word 2. Codes come in pairs, one per CodecMode: 1/4 NV12 (format 3, what +// the movie players use), 2/5 BGRA, 3/6 RGBA; the HDR codes (10-13) are written as RGBA. The +// server picks the layout and describes it to the client with gsv 961: here pitch-linear rows, +// NV12's half-height chroma plane below the luma plane, each plane padded to 128 rows. +struct Nx2DecoderImageLayout { + u32 code; + u32 width; + u32 height; + Tegra::Host1x::Nx2PictureFormat format; + u64 pitch; + u64 chroma_offset; + u64 size; +}; + +inline Nx2DecoderImageLayout Nx2CalculateDecoderImageLayout(const u32* preference) { + using Tegra::Host1x::Nx2PictureFormat; + Nx2DecoderImageLayout layout{}; + layout.code = preference[1]; + layout.width = (preference[2] & 0xFFFF) + 1; + layout.height = (preference[2] >> 16) + 1; + switch (layout.code) { + case 1: + case 4: + layout.format = Nx2PictureFormat::Nv12; + break; + case 2: + case 5: + layout.format = Nx2PictureFormat::Bgra8; + break; + default: + layout.format = Nx2PictureFormat::Rgba8; + break; + } + const bool nv12 = layout.format == Nx2PictureFormat::Nv12; + constexpr u64 BlockRows = 128; + layout.pitch = Common::AlignUp(u64{layout.width} * (nv12 ? 1 : 4), 64); + layout.chroma_offset = layout.pitch * Common::AlignUp(u64{layout.height}, BlockRows); + layout.size = layout.chroma_offset + + (nv12 ? layout.pitch * Common::AlignUp((u64{layout.height} + 1) / 2, BlockRows) : 0); + return layout; +} + // Placeholder for the Switch 2 display session (nn::vi2::sdk), handed out by gsv 305. Reply // shapes read from the SDK: TIPC 1 -> u32 < 0x400, TIPC 3 -> u32 < 8, TIPC 11/31(u32) and // 32(handle, u32) -> one handle at word 3 then the result. Handles returned here are events (a @@ -1143,24 +1201,35 @@ private: std::shared_ptr<std::atomic<bool>> m_capture_pending = std::make_shared<std::atomic<bool>>(false); }; +// True the first time a Switch 2 video command is seen, so each one is logged once in full and new +// or unexpected commands stand out. +inline bool Nx2FirstVideoCommand(std::string_view service, u32 command) { + static std::mutex mutex; + static std::set<std::pair<std::string, u32>> seen; + std::scoped_lock lk{mutex}; + return seen.emplace(std::string{service}, command).second; +} + // The Switch 2 video decoder (nn::codec::video::detail::ipc). gsv 902/906/1002 hand out a session // whose TIPC 1 (CreateVideoDecoderService: handle at word 3, result at word 4) creates a decoder // service, another session of this class. Layouts read from the SDK's nn::codec VP9 decoder: // - 200(copy handle: work buffer transfer memory, u64 size, u32, u32, u32): initialize. -// - 101(X: codec configuration, u32 0, u32 width, u32 height, u32 rate, u32 max input): configure. -// - 3(): an event (copy handle at word 3, result at word 4) the client attaches as its -// submission-ready event. +// - 101(A: codec name, "video/vp9\0"; u32 width, u32 height, u32 rate, u32): configure. +// - 3(): the submission-ready event (copy handle at word 3, result at word 4). // - 6 / 207 / 8: start / stop / flush; 115: finalize. -// - 311(u32 image id): register an output image (gsv 911 creates it); reply a 16-byte -// NativeSyncHandle, all zero reading as already signalled. The client acquires output images -// in registration/release order from a ring of its own (no IPC). -// - 314(u32 image id, u32, NativeSyncHandle): give an output image back; reply a new handle. +// - 311(u32 image id): register an output image (gsv 911 creates it); reply its NativeSyncHandle, +// signalled once a frame has been decoded into the image. The client acquires output images in +// registration/release order from a ring of its own (no IPC), so frames fill them in that order. +// - 314(u32 image id, u32, NativeSyncHandle): give an output image back; the image may be +// rewritten once that handle (the client's reads) has passed. Reply the image's next handle. // - 212(u32 image id): unregister it. // - 109(A: one compressed frame, u64 pts in us, u8 end of stream): submit input; reply a u64. // - 213(): the next finished frame (16 words): u32 image id, u32 width, u32 height, u32 width, -// u32 height, u32 colour space, u64 pts, u8 end of stream. -// Frames are not decoded yet: each submitted frame completes at once into the next free output -// image, so movies run to their end (with no picture) instead of stalling. +// u32 height, u32 colour space, TimeSpan pts (ns), u8 end of stream. +// A NativeSyncHandle is {u32 SyncCounterId (~syncpoint), u32 mode, u64 value}; clients wait on it +// with nx2:syncpt 103/201. Each decoder service owns a syncpoint that counts its finished frames. +// FFmpeg decodes on a thread of the session's own (a 1440p VP9 frame takes milliseconds, and the +// service thread also serves presents and syncpoint waits) into the layout gsv 961 describes. class INx2VideoSession final : public ServiceFramework<INx2VideoSession> { public: explicit INx2VideoSession(Core::System& system_, std::shared_ptr<Nx2NativeObjects> objects_, @@ -1171,12 +1240,12 @@ public: {1, &INx2VideoSession::CreateDecoderService, "CreateVideoDecoderService"}, {3, &INx2VideoSession::GetEvent, "GetEvent"}, {6, &INx2VideoSession::Succeed, "Start"}, - {8, &INx2VideoSession::Succeed, "Flush"}, + {8, &INx2VideoSession::Flush, "Flush"}, {101, &INx2VideoSession::Configure, "Configure"}, {109, &INx2VideoSession::SubmitInput, "SubmitInputBuffer"}, {115, &INx2VideoSession::Succeed, "Finalize"}, {200, &INx2VideoSession::Succeed, "Initialize"}, - {207, &INx2VideoSession::Succeed, "Stop"}, + {207, &INx2VideoSession::Flush, "Stop"}, {212, &INx2VideoSession::UnregisterBuffer, "UnregisterBuffer"}, {213, &INx2VideoSession::GetOutputInfo, "GetOutputInfo"}, {311, &INx2VideoSession::RegisterBuffer, "RegisterBuffer"}, @@ -1185,6 +1254,14 @@ public: } ~INx2VideoSession() override { + if (m_worker.joinable()) { + { + std::scoped_lock lk{m_mutex}; + m_quit = true; + } + m_cv.notify_all(); + m_worker.join(); + } // The ready timer runs on the HostTiming thread: clear its event under its lock first. { std::scoped_lock lk{m_ready->mutex}; @@ -1196,23 +1273,58 @@ public: if (m_event != nullptr) { m_context.CloseEvent(m_event); } + if (m_syncpoint != NoSyncpoint) { + NvdrvSyncpoints().FreeSyncpoint(m_syncpoint); + } } private: + struct Input { + std::vector<u8> data; + u64 pts; // us + bool end; + }; + struct Decoded { + std::optional<Tegra::Host1x::Nx2Picture> picture; // none: an empty end-of-stream frame + u64 pts; + bool end; + }; struct Output { u32 image; + u32 width; + u32 height; u64 pts; bool end; }; + struct Fence { + u32 syncpoint; + u32 value; + }; + + static constexpr u32 NoSyncpoint = ~0U; + static constexpr size_t MaxQueuedInputs = 8; + static constexpr size_t MaxDecodedAhead = 2; // decoded frames held while no image is free + static constexpr u32 HostSyncpoints = 192; // Tegra::Host1x::SyncpointManager void Log(HLERequestContext& ctx, std::string_view what) { - if (m_logged++ < 64) { - const auto* w = ctx.CommandBuffer(); - LOG_INFO(Service, "{} TIPC {} {}: [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x} [6]={:#x} [7]={:#x}", - GetServiceName(), ctx.GetCommand(), what, w[2], w[3], w[4], w[5], w[6], w[7]); + const auto* w = ctx.CommandBuffer(); + if (Nx2FirstVideoCommand(GetServiceName(), ctx.GetCommand())) { + LOG_INFO(Service, "{} TIPC {} {} (first use): words {:08x}", GetServiceName(), + ctx.GetCommand(), what, fmt::join(std::span<const u32>{w, 12}, " ")); + } else { + LOG_DEBUG(Service, "{} TIPC {} {}: [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x}", + GetServiceName(), ctx.GetCommand(), what, w[2], w[3], w[4], w[5]); } } + Nvidia::NvCore::SyncpointManager& NvdrvSyncpoints() { + return system.ServiceManager() + .GetService<Nvidia::NVDRV>("nvdrv", true) + ->GetModule() + ->GetContainer() + .GetSyncpointManager(); + } + void CreateDecoderService(HLERequestContext& ctx) { Log(ctx, "create decoder service"); ReturnRawSession(system, ctx, @@ -1226,10 +1338,24 @@ private: void Configure(HLERequestContext& ctx) { const auto* w = ctx.CommandBuffer(); - // Words 2-3 are the X descriptor of the configuration data. - m_width = w[5]; - m_height = w[6]; + std::string codec; + if (!ctx.BufferDescriptorA().empty()) { + const auto name = ctx.ReadBufferA(0); + codec.assign(reinterpret_cast<const char*>(name.data()), + strnlen(reinterpret_cast<const char*>(name.data()), name.size())); + } Log(ctx, "configure"); + LOG_INFO(Service, "{} configure codec \"{}\" {}x{} rate {}", GetServiceName(), codec, w[5], + w[6], w[7]); + { + std::scoped_lock lk{m_mutex}; + m_codec = std::move(codec); + m_width = w[5]; + m_height = w[6]; + } + if (!m_worker.joinable()) { + m_worker = std::thread{[this] { DecodeLoop(); }}; + } TipcReplyWords(ctx, {}); } @@ -1237,9 +1363,9 @@ private: Log(ctx, "event"); if (m_event == nullptr) { m_event = m_context.CreateEvent(fmt::format("{}:Ready", GetServiceName())); - // Ready for input at all times (frames complete as they are submitted). The client - // attaches it auto-clearing, so one signal is consumed by the first wait: re-signal - // every 4 ms; a spurious wake only makes it check again. + // The client attaches it auto-clearing, so one signal is consumed by the first wait: + // while the input queue has room, re-signal every 4 ms; a spurious wake only makes it + // check again. { std::scoped_lock lk{m_ready->mutex}; m_ready->event = m_event; @@ -1249,7 +1375,7 @@ private: [ready = m_ready, &kernel = system.Kernel()]( s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> { std::scoped_lock lk{ready->mutex}; - if (ready->event != nullptr) { + if (ready->event != nullptr && ready->room.load(std::memory_order_relaxed)) { ready->event->Signal(kernel); } return std::nullopt; @@ -1263,32 +1389,50 @@ private: rb.PushCopyObjects(ctx, m_event->GetReadableEvent()); } + // Queues an output image for the next decoded frame; returns the NativeSyncHandle words that + // signal once that frame is in it. + std::array<u32, 4> HandOut(u32 image, std::optional<Fence> release_fence) { + if (m_syncpoint == NoSyncpoint) { + m_syncpoint = NvdrvSyncpoints().AllocateSyncpoint(false); + m_sync_base = system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(m_syncpoint); + LOG_INFO(Service, "{} counts decoded frames on syncpoint {} from {}", + GetServiceName(), m_syncpoint, m_sync_base); + } + u32 value; + { + std::scoped_lock lk{m_mutex}; + m_free.push_back(image); + if (release_fence) { + m_release_fences[image] = *release_fence; + } + value = m_sync_base + ++m_handed; + } + m_cv.notify_one(); + return {~m_syncpoint, 0, value, 0}; + } + void RegisterBuffer(HLERequestContext& ctx) { Log(ctx, "register output"); - GiveBack(ctx.CommandBuffer()[2]); - TipcReplyWords(ctx, {0, 0, 0, 0}); + const auto handle = HandOut(ctx.CommandBuffer()[2], std::nullopt); + TipcReplyWords(ctx, {handle[0], handle[1], handle[2], handle[3]}); } void ReleaseBuffer(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); Log(ctx, "release output"); - GiveBack(ctx.CommandBuffer()[2]); - TipcReplyWords(ctx, {0, 0, 0, 0}); + const auto handle = HandOut(w[2], Fence{~w[4], w[6]}); + TipcReplyWords(ctx, {handle[0], handle[1], handle[2], handle[3]}); } void UnregisterBuffer(HLERequestContext& ctx) { Log(ctx, "unregister output"); - std::erase(m_free, ctx.CommandBuffer()[2]); - TipcReplyWords(ctx, {}); - } - - // An output image is free again: a frame waiting for one completes into it. - void GiveBack(u32 image) { - if (!m_waiting.empty()) { - m_outputs.push_back({image, m_waiting.front().pts, m_waiting.front().end}); - m_waiting.pop_front(); - return; + const u32 image = ctx.CommandBuffer()[2]; + { + std::scoped_lock lk{m_mutex}; + std::erase(m_free, image); + m_release_fences.erase(image); } - m_free.push_back(image); + TipcReplyWords(ctx, {}); } void SubmitInput(HLERequestContext& ctx) { @@ -1297,37 +1441,221 @@ private: const u64 pts = u64{w[5]} | (u64{w[6]} << 32); const bool end = (w[7] & 0xFF) != 0; Log(ctx, "submit"); - if (!m_free.empty()) { - m_outputs.push_back({m_free.front(), pts, end}); - m_free.pop_front(); - } else { - m_waiting.push_back({0, pts, end}); + Input input{{}, pts, end}; + if (!ctx.BufferDescriptorA().empty()) { + const auto data = ctx.ReadBufferA(0); + input.data.assign(data.begin(), data.end()); } - const u64 id = ++m_submitted; - if (m_event != nullptr) { - m_event->Signal(system.Kernel()); + { + std::scoped_lock lk{m_mutex}; + m_inputs.push_back(std::move(input)); + m_ready->room.store(m_inputs.size() < MaxQueuedInputs, std::memory_order_relaxed); } + m_cv.notify_one(); + const u64 id = ++m_submitted; TipcReplyWords(ctx, {static_cast<u32>(id), static_cast<u32>(id >> 32)}); } + // Flush (8) and Stop (207): drop queued input and frames not yet in an image. Images already + // handed out keep their place, so the sync values stay in step with the client's ring. + void Flush(HLERequestContext& ctx) { + Log(ctx, ctx.GetCommand() == 8 ? "flush" : "stop"); + { + std::scoped_lock lk{m_mutex}; + m_inputs.clear(); + m_flush = true; + m_ready->room.store(true, std::memory_order_relaxed); + } + m_cv.notify_one(); + TipcReplyWords(ctx, {}); + } + void GetOutputInfo(HLERequestContext& ctx) { Log(ctx, "output info"); - if (m_outputs.empty()) { + std::optional<Output> output; + { + std::scoped_lock lk{m_mutex}; + if (!m_outputs.empty()) { + output = m_outputs.front(); + m_outputs.pop_front(); + } + } + if (!output) { // Nothing finished: the code the client's own Acquire uses for an empty ring. IPC::ResponseBuilder rb{ctx, 2}; rb.PushRaw<u32>(0x1c286f); return; } - const Output output = m_outputs.front(); - m_outputs.pop_front(); - TipcReplyWords(ctx, {output.image, m_width, m_height, m_width, m_height, 0, - static_cast<u32>(output.pts), static_cast<u32>(output.pts >> 32), - output.end ? 1U : 0U, 0, 0, 0, 0, 0, 0, 0}); + const u64 pts_ns = output->pts * 1000; + TipcReplyWords(ctx, {output->image, output->width, output->height, output->width, + output->height, 0, static_cast<u32>(pts_ns), + static_cast<u32>(pts_ns >> 32), output->end ? 1U : 0U, 0, 0, 0, 0, + 0, 0, 0}); + } + + // The decode thread: turns queued input into decoded frames and decoded frames into output + // images, in order. + void DecodeLoop() { + Common::SetCurrentThreadName("nx2:vdec"); + std::unique_ptr<Tegra::Host1x::Nx2VideoDecoder> decoder; + std::vector<Tegra::Host1x::Nx2Picture> pictures; + std::vector<u8> bytes; + std::unique_lock lk{m_mutex}; + while (true) { + m_cv.wait(lk, [&] { + return m_quit || m_flush || (!m_decoded.empty() && !m_free.empty()) || + (!m_inputs.empty() && m_decoded.size() < MaxDecodedAhead); + }); + if (m_quit) { + return; + } + if (m_flush) { + m_flush = false; + m_decoded.clear(); + if (decoder) { + lk.unlock(); + decoder->Reset(); + lk.lock(); + } + continue; + } + if (!m_decoded.empty() && !m_free.empty()) { + Decoded frame = std::move(m_decoded.front()); + m_decoded.pop_front(); + const u32 image = m_free.front(); + m_free.pop_front(); + std::optional<Fence> fence; + if (const auto it = m_release_fences.find(image); it != m_release_fences.end()) { + fence = it->second; + m_release_fences.erase(it); + } + lk.unlock(); + if (fence) { + WaitForRelease(*fence); + } + const auto [width, height] = WriteImage(decoder.get(), image, frame, bytes); + lk.lock(); + // Even an image written during a flush counts: the client waits for its value. + m_outputs.push_back({image, width, height, frame.pts, frame.end}); + system.Host1x().GetSyncpointManager().IncrementGuest(m_syncpoint); + ++m_frames_out; + if (m_frames_out == 1 || (m_frames_out % 600) == 0 || frame.end) { + LOG_INFO(Service, "{} frame {} ({}x{}, pts {} us{}) in image {}", + GetServiceName(), m_frames_out, width, height, frame.pts, + frame.end ? ", end of stream" : "", image); + } + continue; + } + Input input = std::move(m_inputs.front()); + m_inputs.pop_front(); + m_ready->room.store(m_inputs.size() < MaxQueuedInputs, std::memory_order_relaxed); + const std::string codec = m_codec; + lk.unlock(); + if (!decoder || decoder->Name() != codec) { + decoder = std::make_unique<Tegra::Host1x::Nx2VideoDecoder>(codec); + } + pictures.clear(); + decoder->Decode(input.data, input.pts, pictures); + if (input.end) { + decoder->Drain(pictures); + } + lk.lock(); + if (m_flush) { + continue; + } + for (auto& picture : pictures) { + const u64 pts = picture.pts; + m_decoded.push_back({std::move(picture), pts, false}); + } + if (input.end) { + // The client stops at the frame marked end of stream; give it one even when the + // last input held no picture. + if (m_decoded.empty()) { + m_decoded.push_back({std::nullopt, input.pts, true}); + } else { + m_decoded.back().end = true; + } + } + } + } + + // Waits (bounded) until the client's reads of an image it gave back have passed. + void WaitForRelease(const Fence& fence) { + if (fence.syncpoint >= HostSyncpoints) { + return; + } + auto& syncpoints = system.Host1x().GetSyncpointManager(); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds{100}; + while (!syncpoints.IsReadyGuest(fence.syncpoint, fence.value) && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + } + + // Writes a decoded frame into an output image; returns the size the frame reports. + std::pair<u32, u32> WriteImage(Tegra::Host1x::Nx2VideoDecoder* decoder, u32 image_id, + const Decoded& frame, std::vector<u8>& bytes) { + u64 address{}; + Nx2DecoderImageLayout layout{}; + { + std::scoped_lock lk{m_objects->mutex}; + const auto image = m_objects->images.find(image_id); + if (image == m_objects->images.end()) { + return {m_width, m_height}; + } + layout = Nx2CalculateDecoderImageLayout(image->second.preference.data()); + const auto memory = m_objects->memories.find(image->second.memory); + if (memory != m_objects->memories.end() && memory->second.address != 0) { + address = memory->second.address + image->second.offset; + } + } + if (!frame.picture || decoder == nullptr || address == 0 || + system.ApplicationProcess() == nullptr) { + return {std::min(m_width, layout.width), std::min(m_height, layout.height)}; + } + // A frame larger than the image is scaled down to it. + const u32 width = std::min(frame.picture->width, layout.width); + const u32 height = std::min(frame.picture->height, layout.height); + bytes.resize(layout.size); + decoder->Render(*frame.picture, layout.format, width, height, layout.pitch, + layout.chroma_offset, bytes); + DumpFrame(bytes, layout, width, height); + auto& memory = system.ApplicationMemory(); + memory.WriteBlock(address, bytes.data(), layout.pitch * height); + if (layout.format == Tegra::Host1x::Nx2PictureFormat::Nv12) { + memory.WriteBlock(address + layout.chroma_offset, bytes.data() + layout.chroma_offset, + layout.pitch * ((height + 1) / 2)); + } + return {width, height}; + } + + // EDEN_NX2_VDEC_DUMP=<dir>: save the luma of every 120th decoded frame, a quarter of its size, + // as nx2-vdec-<frame>.pgm (to check what the decoder wrote, apart from how the game shows it). + void DumpFrame(const std::vector<u8>& bytes, const Nx2DecoderImageLayout& layout, u32 width, + u32 height) { + static const char* const directory = std::getenv("EDEN_NX2_VDEC_DUMP"); + if (directory == nullptr || (m_frames_out % 120) != 0) { + return; + } + const u32 bpp = layout.format == Tegra::Host1x::Nx2PictureFormat::Nv12 ? 1 : 4; + const u32 w = width / 4; + const u32 h = height / 4; + const std::string path = fmt::format("{}/nx2-vdec-{:05}.pgm", directory, m_frames_out + 1); + if (std::FILE* file = std::fopen(path.c_str(), "wb")) { + std::fprintf(file, "P5 %u %u 255\n", w, h); + for (u32 y = 0; y < h; ++y) { + for (u32 x = 0; x < w; ++x) { + std::fputc(bytes[y * 4 * layout.pitch + x * 4 * bpp], file); + } + } + std::fclose(file); + } } struct ReadyState { std::mutex mutex; Kernel::KEvent* event{}; + std::atomic<bool> room{true}; ///< The input queue takes more frames }; KernelHelpers::ServiceContext m_context; @@ -1335,13 +1663,26 @@ private: Kernel::KEvent* m_event{}; std::shared_ptr<ReadyState> m_ready = std::make_shared<ReadyState>(); std::shared_ptr<Core::Timing::EventType> m_ready_timer; + u64 m_submitted{}; + u32 m_syncpoint{NoSyncpoint}; ///< Counts finished frames (allocated at the first register) + u32 m_sync_base{}; ///< Its value before the first frame + + // Shared with the decode thread. + std::mutex m_mutex; + std::condition_variable m_cv; + std::string m_codec; u32 m_width{}; u32 m_height{}; - u64 m_submitted{}; - u32 m_logged{}; - std::deque<u32> m_free; - std::deque<Output> m_waiting; - std::deque<Output> m_outputs; + u32 m_handed{}; ///< Output images handed out (registered or released) so far + u64 m_frames_out{}; + std::deque<Input> m_inputs; + std::deque<Decoded> m_decoded; ///< Waiting for a free output image + std::deque<u32> m_free; ///< Output images, in the order the client acquires them + std::map<u32, Fence> m_release_fences; + std::deque<Output> m_outputs; ///< Finished, for 213 + bool m_flush{}; + bool m_quit{}; + std::thread m_worker; }; // Placeholder for the Switch 2 graphics-system server that nn::gsv::BootWithRemoteMemory talks @@ -1387,8 +1728,8 @@ public: {1211, &INx2GraphicsSystemSession::CreateNativeImage, "CreateChatImagePool (guessed)"}, {1291, &INx2GraphicsSystemSession::CalculateChatImagePoolSize, "CalculateChatImagePoolSize (guessed)"}, - // Same request as 429: the image backing a decoder output (nn::codec image pools). - {911, &INx2GraphicsSystemSession::CreateNativeImage, "CreateDecoderImage (guessed)"}, + {911, &INx2GraphicsSystemSession::CreateDecoderImage, "CreateDecoderImage"}, + {961, &INx2GraphicsSystemSession::GetCpuImageDescriptor, "GetCpuUserImageDescriptor"}, }); } @@ -1596,21 +1937,12 @@ private: // bytes per pixel for the other codes; nothing is decoded into them yet. void CalculateDecoderImagePoolBuffer(HLERequestContext& ctx) { const auto* w = ctx.CommandBuffer(); - const u32 code = w[3]; - const u64 width = (w[4] & 0xFFFF) + 1; - const u64 height = (w[4] >> 16) + 1; + const auto layout = Nx2CalculateDecoderImageLayout(w + 2); const u32 count = std::max<u32>(w[18], 1); - constexpr u64 BlockRows = 128; - const bool nv12 = code == 1 || code == 4; - const u64 pitch = Common::AlignUp(width * (nv12 ? 1 : 4), 64); - u64 size = pitch * Common::AlignUp(height, BlockRows); - if (nv12) { - size += pitch * Common::AlignUp((height + 1) / 2, BlockRows); - } - size *= count; - const u64 alignment = 64 * BlockRows; + const u64 size = layout.size * count; + const u64 alignment = 64 * 128; LOG_INFO(Service, "nx2:gsv 991 decoder image pool code={} {}x{} count={} -> size={:#x}", - code, width, height, count, size); + layout.code, layout.width, layout.height, count, size); TipcReplyWords(ctx, {static_cast<u32>(size), static_cast<u32>(size >> 32), static_cast<u32>(alignment), static_cast<u32>(alignment >> 32)}); } @@ -1761,6 +2093,62 @@ private: } } + // 911(preference[16], u32 count, u32 memory id) (nn::codec::InitializeDecoderImagePool): + // result, the id of the first of `count` decoder output images; the client names image i of + // the pool id + i. They share the pool's memory, one image size apart. + void CreateDecoderImage(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u32 count = std::max<u32>(w[18], 1); + const auto layout = Nx2CalculateDecoderImageLayout(w + 2); + std::scoped_lock lk{m_objects->mutex}; + const u32 first = m_objects->next_image; + m_objects->next_image += count; + for (u32 i = 0; i < count; ++i) { + auto& image = m_objects->images[first + i]; + image.memory = w[19]; + std::copy_n(w + 2, image.preference.size(), image.preference.begin()); + image.size = layout.size; + image.offset = layout.size * i; + image.decoder = true; + } + LOG_INFO(Service, "nx2:gsv 911 decoder images {}..{} code={} {}x{} in memory {}", first, + first + count - 1, layout.code, layout.width, layout.height, w[19]); + TipcReplyWords(ctx, {first}); + } + + // 961(u32 image id): result, the 128-byte nn::gsv::CpuUserImageDescriptor that + // nn::gs::sdk::ReadUserImage* (behind nn::codec::ReadDecoderOutputImage*) copy an image out + // with: u8 kind at +0; for the pitch-linear kinds (8 RGBA, 9 BGRA, 10 NV12) the plane's + // offset into the pool memory (u32 +0x8) and row pitch (u64 +0x10), NV12's chroma plane at + // +0x18/+0x20. 11-13 are the block-linear kinds, not used here. + void GetCpuImageDescriptor(HLERequestContext& ctx) { + using Tegra::Host1x::Nx2PictureFormat; + const u32 id = ctx.CommandBuffer()[2]; + std::array<u32, 32> words{}; + { + std::scoped_lock lk{m_objects->mutex}; + const auto image = m_objects->images.find(id); + if (image != m_objects->images.end() && image->second.decoder) { + const auto layout = Nx2CalculateDecoderImageLayout(image->second.preference.data()); + const u64 offset = image->second.offset; + words[0] = layout.format == Nx2PictureFormat::Nv12 ? 0xa + : layout.format == Nx2PictureFormat::Bgra8 ? 0x9 + : 0x8; + words[2] = static_cast<u32>(offset); + words[4] = static_cast<u32>(layout.pitch); + if (layout.format == Nx2PictureFormat::Nv12) { + words[6] = static_cast<u32>(offset + layout.chroma_offset); + words[8] = static_cast<u32>(layout.pitch); + } + } + } + if (Nx2FirstVideoCommand("nx2:gsv", 961) || words[0] == 0) { + LOG_INFO(Service, "nx2:gsv 961 image {} -> kind {:#x} offset {:#x} pitch {:#x}", id, + words[0], words[2], words[4]); + } + TipcReplyWords(ctx, words); + } + void DestroyNativeImage(HLERequestContext& ctx) { const u32 id = ctx.CommandBuffer()[2]; std::scoped_lock lk{m_objects->mutex}; diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index e619c4f2de..7f83e0d3c2 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -84,6 +84,8 @@ add_library(video_core STATIC host1x/nvdec.cpp host1x/nvdec.h host1x/nvdec_common.h + host1x/nx2_video_decoder.cpp + host1x/nx2_video_decoder.h host1x/syncpoint_manager.cpp host1x/syncpoint_manager.h host1x/vic.cpp diff --git a/src/video_core/host1x/nx2_video_decoder.cpp b/src/video_core/host1x/nx2_video_decoder.cpp new file mode 100644 index 0000000000..7d185ca573 --- /dev/null +++ b/src/video_core/host1x/nx2_video_decoder.cpp @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <array> +#include <cstring> +#include <optional> +#include <set> + +#include "common/logging.h" +#include "common/settings.h" +#include "video_core/host1x/ffmpeg.h" +#include "video_core/host1x/nx2_video_decoder.h" + +extern "C" { +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + +#include <libswscale/swscale.h> + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif +} + +namespace Tegra::Host1x { + +namespace { + +std::optional<NvdecCommon::VideoCodec> CodecFromMime(std::string_view mime) { + if (mime.find("vp9") != std::string_view::npos) { + return NvdecCommon::VideoCodec::VP9; + } + if (mime.find("vp8") != std::string_view::npos) { + return NvdecCommon::VideoCodec::VP8; + } + if (mime.find("avc") != std::string_view::npos || mime.find("264") != std::string_view::npos) { + return NvdecCommon::VideoCodec::H264; + } + return std::nullopt; +} + +} // namespace + +struct Nx2VideoDecoder::Impl { + std::string name; + std::optional<FFmpeg::Decoder> decoder; + std::optional<FFmpeg::HardwareContext> hardware; + std::optional<FFmpeg::DecoderContext> context; + SwsContext* scaler{}; + bool open{}; + /// Timestamps of submitted frames not yet output. Frames copied back from a host GPU decoder + /// lose theirs and take the earliest one instead. + std::multiset<u64> pending_pts; + std::vector<u8> packet; ///< FFmpeg reads up to AV_INPUT_BUFFER_PADDING_SIZE past the data + + ~Impl() { + sws_freeContext(scaler); + context.reset(); + hardware.reset(); + } + + void Receive(std::vector<Nx2Picture>& pictures) { + while (auto frame = context->ReceiveFrame()) { + const AVFrame* f = frame->GetFrame(); + const s64 stamp = f->best_effort_timestamp != AV_NOPTS_VALUE ? f->best_effort_timestamp + : f->pts; + u64 pts{}; + if (stamp != AV_NOPTS_VALUE) { + pts = static_cast<u64>(stamp); + if (const auto it = pending_pts.find(pts); it != pending_pts.end()) { + pending_pts.erase(it); + } + } else if (!pending_pts.empty()) { + pts = *pending_pts.begin(); + pending_pts.erase(pending_pts.begin()); + } + pictures.push_back({static_cast<u32>(f->width), static_cast<u32>(f->height), pts, + std::move(frame)}); + } + } +}; + +Nx2VideoDecoder::Nx2VideoDecoder(std::string_view mime) : impl{std::make_unique<Impl>()} { + impl->name = std::string{mime}; + const auto codec = CodecFromMime(mime); + if (!codec) { + LOG_ERROR(HW_GPU, "nx2 video: no decoder for codec \"{}\"", mime); + return; + } + impl->decoder.emplace(*codec); + if (impl->decoder->GetCodec() == nullptr) { + LOG_ERROR(HW_GPU, "nx2 video: FFmpeg has no decoder for \"{}\"", mime); + return; + } + impl->context.emplace(*impl->decoder); + if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) { + impl->hardware.emplace(); + impl->hardware->InitializeForDecoder(*impl->context, *impl->decoder); + } + impl->open = impl->context->OpenContext(*impl->decoder); + LOG_INFO(HW_GPU, "nx2 video: decoder for \"{}\" {}", mime, impl->open ? "open" : "failed to open"); +} + +Nx2VideoDecoder::~Nx2VideoDecoder() = default; + +bool Nx2VideoDecoder::IsOpen() const { + return impl->open; +} + +const std::string& Nx2VideoDecoder::Name() const { + return impl->name; +} + +void Nx2VideoDecoder::Decode(std::span<const u8> data, u64 pts, std::vector<Nx2Picture>& pictures) { + if (!impl->open || data.empty()) { + return; + } + impl->packet.assign(data.size() + AV_INPUT_BUFFER_PADDING_SIZE, 0); + std::memcpy(impl->packet.data(), data.data(), data.size()); + FFmpeg::Packet packet{std::span<const u8>{impl->packet.data(), data.size()}}; + packet.GetPacket()->pts = static_cast<s64>(pts); + // Hidden VP9 frames never come out: keep the set from growing over a long movie. + if (impl->pending_pts.size() >= 64) { + impl->pending_pts.erase(impl->pending_pts.begin()); + } + impl->pending_pts.insert(pts); + impl->context->SendPacket(packet); + impl->Receive(pictures); +} + +void Nx2VideoDecoder::Drain(std::vector<Nx2Picture>& pictures) { + if (!impl->open) { + return; + } + avcodec_send_packet(impl->context->GetCodecContext(), nullptr); + impl->Receive(pictures); + Reset(); +} + +void Nx2VideoDecoder::Reset() { + if (impl->open) { + avcodec_flush_buffers(impl->context->GetCodecContext()); + } + impl->pending_pts.clear(); +} + +void Nx2VideoDecoder::Render(const Nx2Picture& picture, Nx2PictureFormat format, u32 width, + u32 height, u64 pitch, u64 chroma_offset, std::span<u8> out) { + const AVFrame* f = picture.frame->GetFrame(); + const u64 needed = format == Nx2PictureFormat::Nv12 + ? chroma_offset + pitch * ((u64{height} + 1) / 2) + : pitch * height; + if (width == 0 || height == 0 || out.size() < needed) { + return; + } + const AVPixelFormat target = format == Nx2PictureFormat::Nv12 ? AV_PIX_FMT_NV12 + : format == Nx2PictureFormat::Rgba8 ? AV_PIX_FMT_RGBA + : AV_PIX_FMT_BGRA; + impl->scaler = sws_getCachedContext(impl->scaler, f->width, f->height, + static_cast<AVPixelFormat>(f->format), + static_cast<int>(width), static_cast<int>(height), target, + SWS_BILINEAR, nullptr, nullptr, nullptr); + if (impl->scaler == nullptr) { + LOG_ERROR(HW_GPU, "nx2 video: cannot convert {} to {}", + av_get_pix_fmt_name(static_cast<AVPixelFormat>(f->format)), + av_get_pix_fmt_name(target)); + return; + } + if (format != Nx2PictureFormat::Nv12) { + // The stream's matrix and range; untagged HD video is BT.709. + int space = SWS_CS_ITU601; + if (f->colorspace == AVCOL_SPC_BT709 || + (f->colorspace == AVCOL_SPC_UNSPECIFIED && f->height >= 720)) { + space = SWS_CS_ITU709; + } else if (f->colorspace == AVCOL_SPC_BT2020_NCL || f->colorspace == AVCOL_SPC_BT2020_CL) { + space = SWS_CS_BT2020; + } + const int* coefficients = sws_getCoefficients(space); + sws_setColorspaceDetails(impl->scaler, coefficients, f->color_range == AVCOL_RANGE_JPEG, + coefficients, 1, 0, 1 << 16, 1 << 16); + } + std::array<u8*, 4> planes{out.data(), nullptr, nullptr, nullptr}; + std::array<int, 4> strides{static_cast<int>(pitch), 0, 0, 0}; + if (format == Nx2PictureFormat::Nv12) { + planes[1] = out.data() + chroma_offset; + strides[1] = static_cast<int>(pitch); + } + sws_scale(impl->scaler, f->data, f->linesize, 0, f->height, planes.data(), strides.data()); +} + +} // namespace Tegra::Host1x diff --git a/src/video_core/host1x/nx2_video_decoder.h b/src/video_core/host1x/nx2_video_decoder.h new file mode 100644 index 0000000000..c979a2c801 --- /dev/null +++ b/src/video_core/host1x/nx2_video_decoder.h @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <memory> +#include <span> +#include <string> +#include <string_view> +#include <vector> + +#include "common/common_types.h" + +namespace FFmpeg { +class Frame; +} + +namespace Tegra::Host1x { + +/// A picture decoded by Nx2VideoDecoder, in display order. +struct Nx2Picture { + u32 width{}; + u32 height{}; + u64 pts{}; ///< As submitted with the compressed frame + std::shared_ptr<FFmpeg::Frame> frame; +}; + +/// Pixel layouts of a Switch 2 decoder output image. +enum class Nx2PictureFormat : u32 { + Nv12, ///< Luma rows, then interleaved Cb/Cr rows at chroma_offset + Rgba8, + Bgra8, +}; + +/// The decoder behind the Switch 2 video decoder service (nn::codec::video): FFmpeg decodes the +/// elementary stream the game demuxed, in software or on the host GPU (NVDEC emulation setting). +/// Used from one thread at a time. +class Nx2VideoDecoder { +public: + /// mime: the codec name nn::codec sends with Configure ("video/vp9", "video/vp8", ...). + explicit Nx2VideoDecoder(std::string_view mime); + ~Nx2VideoDecoder(); + + bool IsOpen() const; + + const std::string& Name() const; + + /// Decodes one compressed frame and appends the pictures it completes. + void Decode(std::span<const u8> data, u64 pts, std::vector<Nx2Picture>& pictures); + + /// End of stream: appends the pictures the decoder still holds, then starts over. + void Drain(std::vector<Nx2Picture>& pictures); + + /// Drops everything queued in the decoder (the game flushed it to seek). + void Reset(); + + /// Converts a picture to `format` in `out`: rows `pitch` bytes apart, NV12's chroma plane at + /// `chroma_offset`, scaled to width x height. + void Render(const Nx2Picture& picture, Nx2PictureFormat format, u32 width, u32 height, + u64 pitch, u64 chroma_offset, std::span<u8> out); + +private: + struct Impl; + std::unique_ptr<Impl> impl; +}; + +} // namespace Tegra::Host1x |
