From 8f1435f8db104351fa3388551eee9d12d2043b40 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:24:19 -0500 Subject: nx2: read GPU-written command lists from copies instead of draining The trees fix (#52) drains the split and flushes for every command list on a GPU-written page: in a Mario Kart World race ~13,500 lists and ~400 GPU round trips a second, during which the back has nothing to do. The replay ran at ~40 fps instead of ~45. The front now marks the pages those lists are on. The buffer cache notes writes to them; after a compute dispatch or DMA copy that writes one it copies the page into host memory and submits. The front waits for the back only up to the last compute or copy work it sent (not everything queued), then reads the copy once the GPU is done with it. Pages the GPU wrote once, long before, and the CPU now writes lists to have no copy: their first list drains and flushes the whole page, which is then read from guest memory until GPU work writes it again. CPU writes and inline uploads reach guest memory, so they keep such a page synced but make a copy stale. A page a draw writes always drains. Mario Kart World time-trial replay, Radeon 780M: ~40 -> ~45 fps, trees drawn, no list drains left. --- src/video_core/buffer_cache/buffer_cache.h | 191 ++++++++++++++++++++++++ src/video_core/buffer_cache/buffer_cache_base.h | 51 +++++++ 2 files changed, 242 insertions(+) (limited to 'src/video_core/buffer_cache') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 53bbae7610..cd176dec27 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -174,6 +174,7 @@ void BufferCache

::UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size template void BufferCache

::WriteMemory(DAddr device_addr, u64 size) { + Nx2NoteListHostWrite(device_addr, size); if (IsRegionGpuModified(device_addr, size)) { ClearDownload(device_addr, size); gpu_modified_ranges.Subtract(device_addr, size); @@ -194,6 +195,7 @@ void BufferCache

::CachedWriteMemory(DAddr device_addr, u64 size) { return; } + Nx2NoteListHostWrite(device_addr, size); tmp_buffer.resize_destructive(size); device_memory.ReadBlockUnsafe(device_addr, tmp_buffer.data(), size); @@ -301,6 +303,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am gpu_modified_ranges.Add(pair.first, pair.second); if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { Tegra::Nx2MarkGpuWritten(pair.first, pair.second); + Nx2NoteListWrite(buffer_b, pair.first, pair.second); } } const auto& copy = copies[0]; @@ -1660,6 +1663,194 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 } if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { Tegra::Nx2MarkGpuWritten(device_addr, size); + Nx2NoteListWrite(buffer_id, device_addr, size); + } +} + +template +void BufferCache

::Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 size) { + if constexpr (!IS_OPENGL) { + if (size == 0 || !Tegra::g_nx2_has_command_list_pages.load(std::memory_order_relaxed)) { + return; + } + constexpr u64 page_bits = Tegra::Nx2WrittenPageBits; + const u64 last = (std::min)((device_addr + size - 1) >> page_bits, + Tegra::g_nx2_command_list_pages.size() * 64 - 1); + for (u64 page = device_addr >> page_bits; page <= last;) { + // Storage buffers can be large: skip 64 pages at a time where none is a list page. + const u64 word = + Tegra::g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> + (page % 64); + if (word == 0) { + page = (page / 64 + 1) * 64; + continue; + } + if (word & 1) { + const DAddr page_addr = page << page_bits; + { + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page_addr]; + ++entry.writes; + if (!nx2_collect_list_copies && !entry.draw_written) { + entry.draw_written = true; + LOG_INFO(HW_GPU, "Command list page {:#x} is written by draws too: its " + "lists drain the split", + page_addr); + } + } + if (nx2_collect_list_copies) { + nx2_list_copy_requests.emplace_back(buffer_id, page_addr); + } + } + ++page; + } + } +} + +template +bool BufferCache

::Nx2CopyListPages(u64 tick) { + if constexpr (IS_OPENGL) { + return false; + } else { + if (nx2_list_copy_requests.empty()) { + return false; + } + std::ranges::sort(nx2_list_copy_requests, {}, &std::pair::second); + constexpr DAddr page_size = DAddr{1} << Tegra::Nx2WrittenPageBits; + bool recorded = false; + for (size_t i = 0; i < nx2_list_copy_requests.size(); ++i) { + const auto [buffer_id, page] = nx2_list_copy_requests[i]; + if (i + 1 < nx2_list_copy_requests.size() && + nx2_list_copy_requests[i + 1].second == page) { + continue; + } + Buffer& buffer = slot_buffers[buffer_id]; + const DAddr begin = (std::max)(page, buffer.CpuAddr()); + const DAddr end = + (std::min)(page + page_size, buffer.CpuAddr() + buffer.SizeBytes()); + if (begin >= end) { + continue; + } + Nx2Staging staging = runtime.DownloadStagingBuffer(page_size, true); + const BufferCopy copy{ + .src_offset = buffer.Offset(begin), + .dst_offset = staging.offset + (begin - page), + .size = end - begin, + }; + buffer.MarkUsage(copy.src_offset, copy.size); + runtime.CopyBuffer(staging.buffer, buffer, std::span{©, 1}, true); + recorded = true; + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page]; + if (entry.copy) { + nx2_retired_list_copies.emplace_back(entry.copy_tick, std::move(*entry.copy)); + } + entry.copy = staging; + entry.copy_writes = entry.writes; + entry.copy_tick = tick; + entry.copy_begin = static_cast(begin - page); + entry.copy_end = static_cast(end - page); + } + nx2_list_copy_requests.clear(); + std::erase_if(nx2_retired_list_copies, [this](auto& retired) { + if (!runtime.IsFree(retired.first)) { + return false; + } + runtime.FreeDeferredStagingBuffer(retired.second); + return true; + }); + return recorded; + } +} + +template +auto BufferCache

::Nx2ReadListCopy(DAddr device_addr, std::span out) -> Nx2ListRead { + if constexpr (IS_OPENGL) { + return Nx2ListRead::Unknown; + } else { + constexpr DAddr page_size = DAddr{1} << Tegra::Nx2WrittenPageBits; + const DAddr page = device_addr & ~(page_size - 1); + const u64 offset = device_addr - page; + if (offset + out.size() > page_size) { + return Nx2ListRead::Unknown; + } + const auto current = [&](const Nx2ListPage& entry) { + return entry.copy && !entry.draw_written && entry.copy_writes == entry.writes && + offset >= entry.copy_begin && offset + out.size() <= entry.copy_end; + }; + u64 tick{}; + u64 writes{}; + { + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page); + if (it == nx2_list_pages.end()) { + return Nx2ListRead::Unknown; + } + const Nx2ListPage& entry = it->second; + // Pages the CPU writes the lists of (GPU-written once, long before) stay synced. + if (!entry.draw_written && entry.synced_writes == entry.writes) { + return Nx2ListRead::Synced; + } + if (!current(entry)) { + return Nx2ListRead::Unknown; + } + // The lists after the first of a batch find the copy done: read it at once. + if (runtime.IsFree(entry.copy_tick)) { + std::memcpy(out.data(), entry.copy->mapped_span.data() + offset, out.size()); + return Nx2ListRead::Copied; + } + tick = entry.copy_tick; + writes = entry.copy_writes; + } + // Submitted with the work that wrote the page (Nx2CopyListPages's caller flushes). + runtime.WaitSubmittedTick(tick); + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page); + if (it == nx2_list_pages.end() || !current(it->second) || it->second.copy_tick != tick || + it->second.copy_writes != writes) { + return Nx2ListRead::Unknown; + } + std::memcpy(out.data(), it->second.copy->mapped_span.data() + offset, out.size()); + return Nx2ListRead::Copied; + } +} + +template +void BufferCache

::Nx2NoteListSynced(DAddr page) { + if constexpr (!IS_OPENGL) { + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page]; + entry.synced_writes = entry.writes; + } +} + +template +void BufferCache

::Nx2NoteListHostWrite(DAddr device_addr, u64 size) { + if constexpr (!IS_OPENGL) { + if (size == 0 || !Tegra::g_nx2_has_command_list_pages.load(std::memory_order_relaxed)) { + return; + } + constexpr u64 page_bits = Tegra::Nx2WrittenPageBits; + const u64 last = (std::min)((device_addr + size - 1) >> page_bits, + Tegra::g_nx2_command_list_pages.size() * 64 - 1); + for (u64 page = device_addr >> page_bits; page <= last; ++page) { + if (!((Tegra::g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> + (page % 64)) & + 1)) { + continue; + } + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page << page_bits); + if (it == nx2_list_pages.end()) { + continue; + } + Nx2ListPage& entry = it->second; + const bool synced = entry.synced_writes == entry.writes; + ++entry.writes; + if (synced) { + entry.synced_writes = entry.writes; + } + } } } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 47f342932d..1f0c6cf10f 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -14,7 +14,9 @@ #include #include #include +#include #include +#include #include #include "common/container/unordered_map.h" @@ -321,6 +323,28 @@ public: bool DMAClear(GPUVAddr src_address, u64 amount, u32 value); + /// Switch 2 GPU-written command lists (nx2.h, g_nx2_command_list_pages). While collecting (a + /// compute dispatch or a DMA copy being recorded), writes to command list pages are noted; + /// Nx2CopyListPages then records copies of those pages into host memory after the work. + void Nx2CollectListCopies(bool collect) { + nx2_collect_list_copies = collect; + if (!collect) { + nx2_list_copy_requests.clear(); + } + } + /// True when it recorded copies, which the GPU completes at `tick` (the caller submits). + bool Nx2CopyListPages(u64 tick); + enum class Nx2ListRead { Unknown, Copied, Synced }; + /// Front thread: Synced when guest memory holds the page `device_addr` lies in (flushed and + /// written by no GPU work since), else Copied with `out` filled from the copy of its last + /// write once the GPU is done with it. Unknown otherwise: the caller drains and flushes. + Nx2ListRead Nx2ReadListCopy(DAddr device_addr, std::span out); + /// Front thread, the split drained: the command list page was flushed to guest memory. + void Nx2NoteListSynced(DAddr page); + /// CPU and inline writes: they reach guest memory, so a synced page stays synced, but its copy + /// no longer holds the page. + void Nx2NoteListHostWrite(DAddr device_addr, u64 size); + /// Return true when a CPU region is modified from the GPU [[nodiscard]] bool IsRegionGpuModified(DAddr addr, size_t size); @@ -440,6 +464,10 @@ private: void MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size, bool needs_sync = false); + /// Counts a GPU write to the command list pages in the range (invalidating their copies) and, + /// while collecting, asks for new copies. + void Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 size); + [[nodiscard]] BufferId FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible); void SynchronizeBufferWrites(Buffer& buffer); @@ -604,6 +632,29 @@ private: // Covers the device address space (MaxwellDeviceTraits::device_virtual_bits). std::array> CACHING_PAGEBITS)> page_table; Common::ScratchBuffer tmp_buffer; + + /// A command list page (Nx2CopyListPages): the GPU writes noted to it and the last copy. + using Nx2Staging = decltype(std::declval().DownloadStagingBuffer(size_t{}, true)); + struct Nx2ListPage { + u64 writes{}; ///< GPU writes noted to the page + u64 copy_writes{}; ///< `writes` when the copy was made: the copy is current while equal + u64 copy_tick{}; ///< GPU tick that completes the copy + /// `writes` when the front last flushed the page: guest memory holds it while equal + u64 synced_writes{~u64{0}}; + u32 copy_begin{}; ///< Bytes of the page the copy holds (a buffer can end inside it) + u32 copy_end{}; + /// Written outside a compute dispatch or a DMA copy (a draw's storage writes) at least + /// once: the front can't tell from what it sent whether such a write is still queued, so + /// the page's lists always drain the split instead of reading a copy. + bool draw_written{}; + std::optional copy; + }; + std::mutex nx2_list_mutex; ///< nx2_list_pages: the split's front reads the copies + std::unordered_map nx2_list_pages; + /// Copies replaced while the GPU may still write them: freed once it is past their tick. + std::vector> nx2_retired_list_copies; + boost::container::small_vector, 8> nx2_list_copy_requests; + bool nx2_collect_list_copies{}; }; } // namespace VideoCommon -- cgit v1.2.3