// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later #pragma once #include #include #include #include #include #include #include #include #include "common/bit_field.h" #include "common/common_types.h" #include "common/scratch_buffer.h" #include "video_core/engines/engine_interface.h" #include "video_core/engines/puller.h" namespace Core { class System; } namespace Tegra { namespace Control { struct ChannelState; } class GPU; class MemoryManager; enum class SubmissionMode : u32 { IncreasingOld = 0, Increasing = 1, NonIncreasingOld = 2, NonIncreasing = 3, Inline = 4, IncreaseOnce = 5 }; // Note that, traditionally, methods are treated as 4-byte addressable locations, and hence // their numbers are written down multiplied by 4 in Docs. Here we are not multiply by 4. // So the values you see in docs might be multiplied by 4. // Register documentation: // https://github.com/NVIDIA/open-gpu-doc/blob/ab27fc22db5de0d02a4cabe08e555663b62db4d4/classes/host/cla26f.h // // Register Description (approx): // https://github.com/NVIDIA/open-gpu-doc/blob/ab27fc22db5de0d02a4cabe08e555663b62db4d4/manuals/volta/gv100/dev_pbdma.ref.txt enum class BufferMethods : u32 { BindObject = 0x0, Illegal = 0x1, Nop = 0x2, SemaphoreAddressHigh = 0x4, SemaphoreAddressLow = 0x5, SemaphoreSequencePayload = 0x6, SemaphoreOperation = 0x7, NonStallInterrupt = 0x8, WrcacheFlush = 0x9, MemOpA = 0xA, MemOpB = 0xB, MemOpC = 0xC, MemOpD = 0xD, RefCnt = 0x14, SemaphoreAcquire = 0x1A, SemaphoreRelease = 0x1B, SyncpointPayload = 0x1C, SyncpointOperation = 0x1D, WaitForIdle = 0x1E, CRCCheck = 0x1F, Yield = 0x20, NonPullerMethods = 0x40, }; struct CommandListHeader { union { u64 raw; BitField<0, 40, GPUVAddr> addr; BitField<40, 1, u64> allow_flush; BitField<41, 1, u64> is_push_buffer; BitField<42, 21, u64> size; BitField<63, 1, u64> sync; }; }; static_assert(sizeof(CommandListHeader) == sizeof(u64), "CommandListHeader is incorrect size"); union CommandHeader { u32 argument; BitField<0, 13, u32> method; BitField<0, 24, u32> method_count_; BitField<13, 3, u32> subchannel; BitField<16, 13, u32> arg_count; BitField<16, 13, u32> method_count; BitField<29, 3, SubmissionMode> mode; }; static_assert(std::is_standard_layout_v, "CommandHeader is not standard layout"); static_assert(sizeof(CommandHeader) == sizeof(u32), "CommandHeader has incorrect size!"); inline CommandHeader BuildCommandHeader(BufferMethods method, u32 arg_count, SubmissionMode mode) { CommandHeader result{}; result.method.Assign(static_cast(method)); result.arg_count.Assign(arg_count); result.mode.Assign(mode); return result; } struct CommandList final { CommandList() = default; explicit CommandList(std::size_t size) : command_lists(size) {} explicit CommandList(boost::container::small_vector&& prefetch_command_list_) : prefetch_command_list{std::move(prefetch_command_list_)} {} boost::container::small_vector command_lists; boost::container::small_vector prefetch_command_list; /// EDEN_NX2_PB_CHECK: hash of each entry's words when it was submitted. std::vector submit_hashes; /// The entries' words when they were submitted (NX2), each entry's at submitted_offsets[i]: /// executed instead of the memory, which the game may already be refilling for a later frame. std::vector submitted_words; std::vector submitted_offsets; /// DmaPusher::suspensions when the words were copied. A list copied during a suspension, or /// that waited behind one, reads memory instead: games write the commands it runs before they /// release the acquire. u64 snapshot_suspensions = 0; }; /// FNV-1a over pushbuffer words (EDEN_NX2_PB_CHECK). inline u64 HashPushbufferWords(std::span words) { u64 hash = 0xcbf29ce484222325ULL; for (const u32 word : words) { hash = (hash ^ word) * 0x100000001b3ULL; } return hash; } /// @brief The DmaPusher class implements DMA submission to FIFOs, providing an area of memory that the /// emulated app fills with commands and tells PFIFO to process. The pushbuffers are then assembled /// into a "command stream" consisting of 32-bit words that make up "commands". /// See https://envytools.readthedocs.io/en/latest/hw/fifo/dma-pusher.html#fifo-dma-pusher for /// details on this implementation. class DmaPusher final { public: /// Counts suspensions on a semaphore acquire and the resumes after them, so it is odd while /// the channel is suspended (see CommandList::snapshot_suspensions). std::atomic suspensions{0}; explicit DmaPusher(Core::System& system_, MemoryManager& memory_manager_, Control::ChannelState& channel_state_); ~DmaPusher(); void Push(CommandList&& entries) { dma_pushbuffer.push(std::move(entries)); } void DispatchCalls(); /// Switch 2: true while the channel is suspended on a semaphore acquire. [[nodiscard]] bool HasPendingAcquire() const { return puller.HasPendingAcquire(); } [[nodiscard]] bool CanResumeAcquire() { return puller.CanResumeAcquire(*this); } void BindSubchannel(Engines::EngineInterface* engine, u32 subchannel_id, Engines::EngineTypes engine_type) { subchannels[subchannel_id] = engine; subchannel_type[subchannel_id] = engine_type; } void BindRasterizer(VideoCore::RasterizerInterface* rasterizer); private: static constexpr u32 non_puller_methods = 0x40; static constexpr u32 max_subchannels = 8; bool Step(); /// Returns how many words were processed: fewer than given when a semaphore acquire /// suspended the channel. size_t ProcessCommands(std::span commands, size_t word_offset = 0); size_t command_resume_offset{}; ///< Words of the current entry done before a suspension void SetState(const CommandHeader& command_header); void CallMethod(u32 argument); void CallMultiMethod(const u32* base_start, u32 num_methods); [[nodiscard]] bool WantsSegment() const; void ReportUnboundSubchannel(); void WatchArgument(u32 method, u32 argument) const; public: /// Logs the GPFIFO entry being processed (and the one before it), read through the GPU MMU /// and straight from guest memory, to find pushbuffer desyncs. Debug aid for Switch 2. void DumpCurrentEntry(const char* reason); void CheckSubmitHash(const CommandListHeader& header, u64 submit_hash); void RecordParsedEntry(const CommandListHeader& header, std::span commands); /// Whether the entry lies on device pages the GPU has written (NX2). bool IsGpuWrittenCommands(GPUVAddr address, size_t size); /// Reads a command list on pages the GPU wrote into command_headers, from the copies the /// buffer cache made after the work that wrote them or from guest memory for pages flushed /// since. False when not all are current (or the list isn't GPU-written): the caller drains, /// flushes and reads guest memory. bool ReadGpuWrittenCommands(GPUVAddr address, u32 count); /// Brings command list words the GPU wrote (still only in host buffers) to guest memory. void FlushGpuWrittenCommands(GPUVAddr address, size_t size); private: public: Common::ScratchBuffer command_headers; ///< Buffer for list of commands fetched at once std::queue dma_pushbuffer; ///< Queue of command lists to be processed std::size_t dma_pushbuffer_subindex{}; ///< Index within a command list within the pushbuffer struct DmaState { u32 method; ///< Current method u32 subchannel; ///< Current subchannel u32 method_count; ///< Current method count u32 length_pending; ///< Large NI command length pending GPUVAddr dma_get; ///< Currently read segment u64 dma_word_offset; ///< Current word offset from address bool non_incrementing; ///< Current command's NI flag bool is_last_call; }; Core::System& system; MemoryManager& memory_manager; Control::ChannelState& channel_state; DmaState dma_state{}; // Switch 2: methods whose data continues in a later GPFIFO entry, set aside while other // command buffers run (NVN interleaves several, so more than one can be pending). struct ParkedMethod { u32 method; u32 subchannel; u32 method_count; bool non_incrementing; bool increment_once; GPUVAddr resume_address; ///< Where the entry that continues the method starts }; void ResolveSplitMethod(GPUVAddr entry_address); static constexpr size_t MaxParkedMethods = 16; // Debug aid: hashes of the words of the last entries as they were parsed. struct ParsedEntry { GPUVAddr address; u32 size; u64 hash; }; std::array parsed_entries{}; size_t parsed_entry_count{}; std::vector parked_methods; GPUVAddr split_method_end{}; ///< End of the last entry that ended inside a method std::array subchannels{}; std::array subchannel_type; Engines::Puller puller; std::mutex sync_mutex; std::condition_variable sync_cv; VideoCore::RasterizerInterface* rasterizer = nullptr; const bool ib_enable : 1 = true; ///< IB mode enabled bool dma_increment_once : 1 = false; bool signal_sync : 1 = false; bool synced : 1 = false; }; } // namespace Tegra