From bef001c1515a13a9b8e361a5a7d372fc2e01094c Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:39:08 -0400 Subject: nx2: read command lists the GPU wrote from the GPU's copy Mario Kart World runs small command lists (20-byte entries) that its culling dispatches fill in each frame. Accurate GPU mode reads every command list through the caches; Fast mode reads guest memory directly (on the split's front thread too), where those entries were still the words of an older frame, so the draws behind them were left out: the trees around the courses vanished in Fast mode only. The buffer cache now also marks the device pages the GPU writes (shader stores, copies of GPU-written data) in a bitmap the split's front can read. A command list on such a page waits for the back to run what was queued before it and is flushed when its words are still only in host buffers. In a race that is ~5% of command lists (13,500 a second of 262,000) and ~400 flushes a second. --- src/video_core/buffer_cache/buffer_cache.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/video_core/buffer_cache') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 42d7ba066d..a8f1edc68d 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -17,6 +17,7 @@ #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/guest_memory.h" #include "video_core/host1x/gpu_device_memory_manager.h" +#include "video_core/nx2.h" #include "video_core/nx2_profile.h" #include "video_core/texture_cache/util.h" @@ -289,6 +290,9 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am const bool has_new_downloads = tmp_intervals.size() != 0; for (const auto& pair : tmp_intervals) { gpu_modified_ranges.Add(pair.first, pair.second); + if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { + Tegra::Nx2MarkGpuWritten(pair.first, pair.second); + } } const auto& copy = copies[0]; src_buffer.MarkUsage(copy.src_offset, copy.size); @@ -1606,6 +1610,9 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 if (AsyncBufferDownloads()) { uncommitted_gpu_modified_ranges.Add(device_addr, size); } + if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { + Tegra::Nx2MarkGpuWritten(device_addr, size); + } } template -- cgit v1.2.3 From 594e07ee5302098e417858608dd03c46336274f5 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:11:09 -0400 Subject: buffer_cache: collect Switch 2 buffers from 512 MiB The collector ran on Switch 2 titles only once their buffers reached 1 GiB (aggressively at 2 GiB), so geometry of scenes long gone stayed in VRAM: Mario Kart World's title screen grew from 0.2 to 0.9 GB of buffers in ten minutes, most of a ~1.4 GB climb in VRAM in use. It now starts at 512 MiB (aggressive at 1 GiB); it only evicts buffers unused for 120 frames or more. (cherry picked from commit fb5d797636f2269591b68fda7c8d65787a9e3a9f) --- src/video_core/buffer_cache/buffer_cache.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/video_core/buffer_cache') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e90af3214d..53874e75cc 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -141,9 +141,10 @@ void BufferCache

::TickFrame() { // title 1's buffers were ~200 MiB of an 8 GB GPU's ~6.8 GB in use, yet this collector // ran aggressively every frame on the device-wide figure, evicting up to 64 buffers (and // downloading the GPU-written ones, ~5% of the GPU thread) that were recreated later. - // Collect when the buffers themselves are large. - if (buffer_bytes >= 1_GiB) { - RunGarbageCollector(buffer_bytes >= 2_GiB); + // Collect when the buffers themselves are large. At 1 GiB it held geometry of scenes + // long gone: title 1's title screen grew to ~0.9 GB of buffers in ten minutes. + if (buffer_bytes >= 512_MiB) { + RunGarbageCollector(buffer_bytes >= 1_GiB); } } else if (total_used_memory >= minimum_memory) { RunGarbageCollector(total_used_memory >= critical_memory); -- cgit v1.2.3 From 86968cb5ce3195cb131eeb0ef68d2d8a4860f7df Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:42:26 -0500 Subject: buffer_cache: mark guest memory large buffer uploads came from cold Like the texture cache's uploads: guest ranges of 256 KiB or more uploaded to the host GPU are mostly geometry the guest rarely reads again, so every 600 frames they are marked cold and the OS pages them out first when it needs RAM. (cherry picked from commit 5c942a5158a0f9769553eeba21b6121f277d0dc5) --- src/video_core/buffer_cache/buffer_cache.h | 11 +++++++++++ src/video_core/buffer_cache/buffer_cache_base.h | 2 ++ 2 files changed, 13 insertions(+) (limited to 'src/video_core/buffer_cache') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 42d7ba066d..633f8760e4 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -100,6 +100,12 @@ void BufferCache

::TickFrame() { return; } runtime.TickFrame(slot_buffers); + if (frame_tick % 600 == 0 && !uploaded_buffer_ranges.empty()) { + for (const auto& [address, size] : uploaded_buffer_ranges) { + device_memory.HintCold(address, size); + } + uploaded_buffer_ranges.clear(); + } if (static u64 frames{}; ++frames % 600 == 0) { u64 total{}; size_t count{}; @@ -1923,6 +1929,11 @@ void BufferCache

::MappedUploadMemory([[maybe_unused]] Buffer& buffer, u8* const src_pointer = staging_pointer.data() + copy.src_offset; const DAddr device_addr = buffer.CpuAddr() + copy.dst_offset; device_memory.ReadBlockUnsafe(device_addr, src_pointer, copy.size); + // Large uploads are mostly geometry the host GPU now has its own copy of: TickFrame + // marks their guest memory cold (see TextureCache::RefreshContents). + if (copy.size >= 256_KiB) { + uploaded_buffer_ranges.emplace_back(device_addr, copy.size); + } // Apply the staging offset copy.src_offset += upload_staging.offset; } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 45a1836341..0db8bdb1eb 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -549,6 +549,8 @@ private: size_t immediate_buffer_capacity = 0; Common::ScratchBuffer immediate_buffer_alloc; + /// Guest memory large uploads came from since the last cold hint (device address, size). + std::vector> uploaded_buffer_ranges; struct LRUItemParams { using ObjectType = BufferId; -- cgit v1.2.3 From 8f1435f8db104351fa3388551eee9d12d2043b40 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:24:19 -0500 Subject: nx2: read GPU-written command lists from copies instead of draining The trees fix (#52) drains the split and flushes for every command list on a GPU-written page: in a Mario Kart World race ~13,500 lists and ~400 GPU round trips a second, during which the back has nothing to do. The replay ran at ~40 fps instead of ~45. The front now marks the pages those lists are on. The buffer cache notes writes to them; after a compute dispatch or DMA copy that writes one it copies the page into host memory and submits. The front waits for the back only up to the last compute or copy work it sent (not everything queued), then reads the copy once the GPU is done with it. Pages the GPU wrote once, long before, and the CPU now writes lists to have no copy: their first list drains and flushes the whole page, which is then read from guest memory until GPU work writes it again. CPU writes and inline uploads reach guest memory, so they keep such a page synced but make a copy stale. A page a draw writes always drains. Mario Kart World time-trial replay, Radeon 780M: ~40 -> ~45 fps, trees drawn, no list drains left. --- src/video_core/buffer_cache/buffer_cache.h | 191 +++++++++++++++++++++ src/video_core/buffer_cache/buffer_cache_base.h | 51 ++++++ src/video_core/dma_pusher.cpp | 98 ++++++++++- src/video_core/dma_pusher.h | 5 + src/video_core/gpu_split.cpp | 30 +++- src/video_core/gpu_split.h | 10 ++ src/video_core/nx2.h | 32 ++++ src/video_core/rasterizer_interface.h | 14 ++ src/video_core/renderer_vulkan/vk_buffer_cache.cpp | 4 + src/video_core/renderer_vulkan/vk_buffer_cache.h | 3 + src/video_core/renderer_vulkan/vk_rasterizer.cpp | 40 ++++- src/video_core/renderer_vulkan/vk_rasterizer.h | 6 + 12 files changed, 478 insertions(+), 6 deletions(-) (limited to 'src/video_core/buffer_cache') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 53bbae7610..cd176dec27 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -174,6 +174,7 @@ void BufferCache

::UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size template void BufferCache

::WriteMemory(DAddr device_addr, u64 size) { + Nx2NoteListHostWrite(device_addr, size); if (IsRegionGpuModified(device_addr, size)) { ClearDownload(device_addr, size); gpu_modified_ranges.Subtract(device_addr, size); @@ -194,6 +195,7 @@ void BufferCache

::CachedWriteMemory(DAddr device_addr, u64 size) { return; } + Nx2NoteListHostWrite(device_addr, size); tmp_buffer.resize_destructive(size); device_memory.ReadBlockUnsafe(device_addr, tmp_buffer.data(), size); @@ -301,6 +303,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am gpu_modified_ranges.Add(pair.first, pair.second); if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { Tegra::Nx2MarkGpuWritten(pair.first, pair.second); + Nx2NoteListWrite(buffer_b, pair.first, pair.second); } } const auto& copy = copies[0]; @@ -1660,6 +1663,194 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 } if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { Tegra::Nx2MarkGpuWritten(device_addr, size); + Nx2NoteListWrite(buffer_id, device_addr, size); + } +} + +template +void BufferCache

::Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 size) { + if constexpr (!IS_OPENGL) { + if (size == 0 || !Tegra::g_nx2_has_command_list_pages.load(std::memory_order_relaxed)) { + return; + } + constexpr u64 page_bits = Tegra::Nx2WrittenPageBits; + const u64 last = (std::min)((device_addr + size - 1) >> page_bits, + Tegra::g_nx2_command_list_pages.size() * 64 - 1); + for (u64 page = device_addr >> page_bits; page <= last;) { + // Storage buffers can be large: skip 64 pages at a time where none is a list page. + const u64 word = + Tegra::g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> + (page % 64); + if (word == 0) { + page = (page / 64 + 1) * 64; + continue; + } + if (word & 1) { + const DAddr page_addr = page << page_bits; + { + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page_addr]; + ++entry.writes; + if (!nx2_collect_list_copies && !entry.draw_written) { + entry.draw_written = true; + LOG_INFO(HW_GPU, "Command list page {:#x} is written by draws too: its " + "lists drain the split", + page_addr); + } + } + if (nx2_collect_list_copies) { + nx2_list_copy_requests.emplace_back(buffer_id, page_addr); + } + } + ++page; + } + } +} + +template +bool BufferCache

::Nx2CopyListPages(u64 tick) { + if constexpr (IS_OPENGL) { + return false; + } else { + if (nx2_list_copy_requests.empty()) { + return false; + } + std::ranges::sort(nx2_list_copy_requests, {}, &std::pair::second); + constexpr DAddr page_size = DAddr{1} << Tegra::Nx2WrittenPageBits; + bool recorded = false; + for (size_t i = 0; i < nx2_list_copy_requests.size(); ++i) { + const auto [buffer_id, page] = nx2_list_copy_requests[i]; + if (i + 1 < nx2_list_copy_requests.size() && + nx2_list_copy_requests[i + 1].second == page) { + continue; + } + Buffer& buffer = slot_buffers[buffer_id]; + const DAddr begin = (std::max)(page, buffer.CpuAddr()); + const DAddr end = + (std::min)(page + page_size, buffer.CpuAddr() + buffer.SizeBytes()); + if (begin >= end) { + continue; + } + Nx2Staging staging = runtime.DownloadStagingBuffer(page_size, true); + const BufferCopy copy{ + .src_offset = buffer.Offset(begin), + .dst_offset = staging.offset + (begin - page), + .size = end - begin, + }; + buffer.MarkUsage(copy.src_offset, copy.size); + runtime.CopyBuffer(staging.buffer, buffer, std::span{©, 1}, true); + recorded = true; + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page]; + if (entry.copy) { + nx2_retired_list_copies.emplace_back(entry.copy_tick, std::move(*entry.copy)); + } + entry.copy = staging; + entry.copy_writes = entry.writes; + entry.copy_tick = tick; + entry.copy_begin = static_cast(begin - page); + entry.copy_end = static_cast(end - page); + } + nx2_list_copy_requests.clear(); + std::erase_if(nx2_retired_list_copies, [this](auto& retired) { + if (!runtime.IsFree(retired.first)) { + return false; + } + runtime.FreeDeferredStagingBuffer(retired.second); + return true; + }); + return recorded; + } +} + +template +auto BufferCache

::Nx2ReadListCopy(DAddr device_addr, std::span out) -> Nx2ListRead { + if constexpr (IS_OPENGL) { + return Nx2ListRead::Unknown; + } else { + constexpr DAddr page_size = DAddr{1} << Tegra::Nx2WrittenPageBits; + const DAddr page = device_addr & ~(page_size - 1); + const u64 offset = device_addr - page; + if (offset + out.size() > page_size) { + return Nx2ListRead::Unknown; + } + const auto current = [&](const Nx2ListPage& entry) { + return entry.copy && !entry.draw_written && entry.copy_writes == entry.writes && + offset >= entry.copy_begin && offset + out.size() <= entry.copy_end; + }; + u64 tick{}; + u64 writes{}; + { + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page); + if (it == nx2_list_pages.end()) { + return Nx2ListRead::Unknown; + } + const Nx2ListPage& entry = it->second; + // Pages the CPU writes the lists of (GPU-written once, long before) stay synced. + if (!entry.draw_written && entry.synced_writes == entry.writes) { + return Nx2ListRead::Synced; + } + if (!current(entry)) { + return Nx2ListRead::Unknown; + } + // The lists after the first of a batch find the copy done: read it at once. + if (runtime.IsFree(entry.copy_tick)) { + std::memcpy(out.data(), entry.copy->mapped_span.data() + offset, out.size()); + return Nx2ListRead::Copied; + } + tick = entry.copy_tick; + writes = entry.copy_writes; + } + // Submitted with the work that wrote the page (Nx2CopyListPages's caller flushes). + runtime.WaitSubmittedTick(tick); + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page); + if (it == nx2_list_pages.end() || !current(it->second) || it->second.copy_tick != tick || + it->second.copy_writes != writes) { + return Nx2ListRead::Unknown; + } + std::memcpy(out.data(), it->second.copy->mapped_span.data() + offset, out.size()); + return Nx2ListRead::Copied; + } +} + +template +void BufferCache

::Nx2NoteListSynced(DAddr page) { + if constexpr (!IS_OPENGL) { + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page]; + entry.synced_writes = entry.writes; + } +} + +template +void BufferCache

::Nx2NoteListHostWrite(DAddr device_addr, u64 size) { + if constexpr (!IS_OPENGL) { + if (size == 0 || !Tegra::g_nx2_has_command_list_pages.load(std::memory_order_relaxed)) { + return; + } + constexpr u64 page_bits = Tegra::Nx2WrittenPageBits; + const u64 last = (std::min)((device_addr + size - 1) >> page_bits, + Tegra::g_nx2_command_list_pages.size() * 64 - 1); + for (u64 page = device_addr >> page_bits; page <= last; ++page) { + if (!((Tegra::g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> + (page % 64)) & + 1)) { + continue; + } + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page << page_bits); + if (it == nx2_list_pages.end()) { + continue; + } + Nx2ListPage& entry = it->second; + const bool synced = entry.synced_writes == entry.writes; + ++entry.writes; + if (synced) { + entry.synced_writes = entry.writes; + } + } } } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 47f342932d..1f0c6cf10f 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -14,7 +14,9 @@ #include #include #include +#include #include +#include #include #include "common/container/unordered_map.h" @@ -321,6 +323,28 @@ public: bool DMAClear(GPUVAddr src_address, u64 amount, u32 value); + /// Switch 2 GPU-written command lists (nx2.h, g_nx2_command_list_pages). While collecting (a + /// compute dispatch or a DMA copy being recorded), writes to command list pages are noted; + /// Nx2CopyListPages then records copies of those pages into host memory after the work. + void Nx2CollectListCopies(bool collect) { + nx2_collect_list_copies = collect; + if (!collect) { + nx2_list_copy_requests.clear(); + } + } + /// True when it recorded copies, which the GPU completes at `tick` (the caller submits). + bool Nx2CopyListPages(u64 tick); + enum class Nx2ListRead { Unknown, Copied, Synced }; + /// Front thread: Synced when guest memory holds the page `device_addr` lies in (flushed and + /// written by no GPU work since), else Copied with `out` filled from the copy of its last + /// write once the GPU is done with it. Unknown otherwise: the caller drains and flushes. + Nx2ListRead Nx2ReadListCopy(DAddr device_addr, std::span out); + /// Front thread, the split drained: the command list page was flushed to guest memory. + void Nx2NoteListSynced(DAddr page); + /// CPU and inline writes: they reach guest memory, so a synced page stays synced, but its copy + /// no longer holds the page. + void Nx2NoteListHostWrite(DAddr device_addr, u64 size); + /// Return true when a CPU region is modified from the GPU [[nodiscard]] bool IsRegionGpuModified(DAddr addr, size_t size); @@ -440,6 +464,10 @@ private: void MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size, bool needs_sync = false); + /// Counts a GPU write to the command list pages in the range (invalidating their copies) and, + /// while collecting, asks for new copies. + void Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 size); + [[nodiscard]] BufferId FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible); void SynchronizeBufferWrites(Buffer& buffer); @@ -604,6 +632,29 @@ private: // Covers the device address space (MaxwellDeviceTraits::device_virtual_bits). std::array> CACHING_PAGEBITS)> page_table; Common::ScratchBuffer tmp_buffer; + + /// A command list page (Nx2CopyListPages): the GPU writes noted to it and the last copy. + using Nx2Staging = decltype(std::declval().DownloadStagingBuffer(size_t{}, true)); + struct Nx2ListPage { + u64 writes{}; ///< GPU writes noted to the page + u64 copy_writes{}; ///< `writes` when the copy was made: the copy is current while equal + u64 copy_tick{}; ///< GPU tick that completes the copy + /// `writes` when the front last flushed the page: guest memory holds it while equal + u64 synced_writes{~u64{0}}; + u32 copy_begin{}; ///< Bytes of the page the copy holds (a buffer can end inside it) + u32 copy_end{}; + /// Written outside a compute dispatch or a DMA copy (a draw's storage writes) at least + /// once: the front can't tell from what it sent whether such a write is still queued, so + /// the page's lists always drain the split instead of reading a copy. + bool draw_written{}; + std::optional copy; + }; + std::mutex nx2_list_mutex; ///< nx2_list_pages: the split's front reads the copies + std::unordered_map nx2_list_pages; + /// Copies replaced while the GPU may still write them: freed once it is past their tick. + std::vector> nx2_retired_list_copies; + boost::container::small_vector, 8> nx2_list_copy_requests; + bool nx2_collect_list_copies{}; }; } // namespace VideoCommon diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 1144488c91..6b93bb7f60 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -9,6 +9,7 @@ #include #include +#include "common/scope_exit.h" #include "common/settings.h" #include "core/core.h" #include "core/memory.h" @@ -178,6 +179,11 @@ bool DmaPusher::Step() { RecordParsedEntry(header, commands); command_resume_offset += ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); + } else if (ReadGpuWrittenCommands(dma_state.dma_get, static_cast(header.size))) { + const std::span commands{command_headers.data(), header.size}; + RecordParsedEntry(header, commands); + command_resume_offset += + ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); } else { FlushGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); Tegra::Memory::GpuGuestMemoryheaders(memory_manager, dma_state.dma_get, header.size, &command_headers); @@ -382,6 +388,15 @@ void DmaPusher::CallMethod(u32 argument) { ReportUnboundSubchannel(); return; } + // Compute dispatches and DMA copies write the command lists ReadGpuWrittenCommands reads. + const auto engine_type = subchannel_type[dma_state.subchannel]; + const bool writer = engine_type == Engines::EngineTypes::KeplerCompute || + engine_type == Engines::EngineTypes::MaxwellDMA; + SCOPE_EXIT { + if (writer) { + split.NoteWriter(); + } + }; if (split.Active()) { const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset; if (split.FrontMacros()) { @@ -432,6 +447,15 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { ReportUnboundSubchannel(); return; } + // Compute dispatches and DMA copies write the command lists ReadGpuWrittenCommands reads. + const auto engine_type = subchannel_type[dma_state.subchannel]; + const bool writer = engine_type == Engines::EngineTypes::KeplerCompute || + engine_type == Engines::EngineTypes::MaxwellDMA; + SCOPE_EXIT { + if (writer) { + split.NoteWriter(); + } + }; if (split.Active()) { const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset; if (split.FrontMacros()) { @@ -476,18 +500,86 @@ bool DmaPusher::IsGpuWrittenCommands(GPUVAddr address, size_t size) { return false; } +bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { + // The pages are marked as command list pages: from the next write on, the buffer cache copies + // them to host memory right after the dispatch or copy that writes them and submits. The + // front then waits only for the back to run that work and for the GPU to finish the copy, not + // for everything queued and a cache flush (FlushGpuWrittenCommands: ~400 GPU round trips a + // second in a Mario Kart World race, during which the back had nothing to do). Pages + // FlushGpuWrittenCommands flushed whole and no GPU work wrote since are read from guest + // memory. + if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || count == 0 || + rasterizer == nullptr) { + return false; + } + const GPUVAddr end = address + u64{count} * sizeof(u32); + bool written = false; + for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < end; page += 0x1000) { + if (const auto device = memory_manager.GpuToCpuAddress(page); + device && Tegra::Nx2IsGpuWritten(*device)) { + Tegra::Nx2MarkCommandListPage(*device); + written = true; + } + } + if (!written) { + return false; + } + if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { + split.WaitForWriters(); + } + using GpuWrittenList = VideoCore::RasterizerInterface::GpuWrittenList; + command_headers.resize_destructive(count); + u8* const out = reinterpret_cast(command_headers.data()); + for (GPUVAddr chunk = address; chunk < end;) { + const GPUVAddr chunk_end = (std::min)(end, (chunk & ~GPUVAddr{0xfff}) + 0x1000); + const std::span part{out + (chunk - address), static_cast(chunk_end - chunk)}; + const auto device = memory_manager.GpuToCpuAddress(chunk); + if (!device) { + return false; + } + GpuWrittenList where = GpuWrittenList::Synced; + if (Tegra::Nx2IsGpuWritten(*device)) { + where = rasterizer->ReadGpuWrittenCommandList(*device, part); + } + if (where == GpuWrittenList::Unknown) { + return false; + } + if (where == GpuWrittenList::Synced) { + memory_manager.ReadBlockUnsafe(chunk, part.data(), part.size()); + } + chunk = chunk_end; + } + return true; +} + void DmaPusher::FlushGpuWrittenCommands(GPUVAddr address, size_t size) { // Title 1 runs small command lists its culling dispatches fill in each frame. Accurate GPU // mode reads every command list through the caches (safe reads); here only the ones on pages // the GPU has written are, or their draws run with an older frame's words (trees vanish). + // These are the lists ReadGpuWrittenCommands can't take from a copy or guest memory (a page's + // first list, or pages draws write too). if (!IsGpuWrittenCommands(address, size)) { return; } // The caches belong to the split's back: it first runs what the front queued (the dispatch - // that writes these words among it). + // that writes these words among it). The whole pages are flushed: their next lists read + // guest memory until GPU work writes them again. VideoCommon::GpuSplit::Get().Drain(); - if (memory_manager.IsMemoryDirty(address, size, VideoCommon::CacheType::BufferCache)) { - memory_manager.FlushRegion(address, size, VideoCommon::CacheType::BufferCache); + const GPUVAddr first_page = address & ~GPUVAddr{0xfff}; + const GPUVAddr end_page = (address + size + 0xfff) & ~GPUVAddr{0xfff}; + if (memory_manager.IsMemoryDirty(first_page, end_page - first_page, + VideoCommon::CacheType::BufferCache)) { + memory_manager.FlushRegion(first_page, end_page - first_page, + VideoCommon::CacheType::BufferCache); + } + if (rasterizer == nullptr) { + return; + } + for (GPUVAddr page = first_page; page < end_page; page += 0x1000) { + if (const auto device = memory_manager.GpuToCpuAddress(page); + device && Tegra::Nx2IsCommandListPage(*device)) { + rasterizer->NoteGpuWrittenCommandListSynced(*device); + } } } diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index e9dd5e2b6b..289786bb9e 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -188,6 +188,11 @@ public: void RecordParsedEntry(const CommandListHeader& header, std::span commands); /// Whether the entry lies on device pages the GPU has written (NX2). bool IsGpuWrittenCommands(GPUVAddr address, size_t size); + /// Reads a command list on pages the GPU wrote into command_headers, from the copies the + /// buffer cache made after the work that wrote them or from guest memory for pages flushed + /// since. False when not all are current (or the list isn't GPU-written): the caller drains, + /// flushes and reads guest memory. + bool ReadGpuWrittenCommands(GPUVAddr address, u32 count); /// Brings command list words the GPU wrote (still only in host buffers) to guest memory. void FlushGpuWrittenCommands(GPUVAddr address, size_t size); diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index ac0d3f25f1..ef0cdae123 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -138,6 +138,7 @@ void GpuSplit::Stop() { // never ran, it waited on its syncpoint forever), and the engines it held were freed. ring.reset(); write_position = 0; + last_writer_position = 0; published_local = 0; consumed_cache = 0; publish_every = PublishEvery; @@ -216,6 +217,28 @@ void GpuSplit::Drain() { } } +void GpuSplit::WaitForWriters() { + const u64 target = std::min(last_writer_position, write_position); + consumed_cache = consumed.load(std::memory_order_acquire); + if (consumed_cache >= target) { + return; + } + wait_target.store(target, std::memory_order_seq_cst); + Publish(); + for (u32 spins = 0;; ++spins) { + consumed_cache = consumed.load(std::memory_order_acquire); + if (consumed_cache >= target) { + wait_target.store(~u64{0}, std::memory_order_relaxed); + return; + } + if (spins < 16384) { + Pause(); + } else { + std::this_thread::yield(); + } + } +} + void GpuSplit::SwitchEngine(Tegra::Engines::EngineInterface* engine) { front_engine = engine; u64* const entry = Reserve(2); @@ -346,8 +369,11 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { const size_t index = read_position & RingMask; const size_t size = Execute(&ring[index]); read_position += size != 0 ? size : RingWords - index; - // Hand ring space back now and then while working through a long batch. - if (read_position - last_store >= 4096 && read_position != available) { + // Hand ring space back now and then while working through a long batch, and at once + // when the front waits for this position (WaitForWriters). + if ((read_position - last_store >= 4096 || + read_position >= wait_target.load(std::memory_order_relaxed)) && + read_position != available) { consumed.store(read_position, std::memory_order_release); last_store = read_position; } diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index d6f91e4141..c7c7118481 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -100,6 +100,13 @@ public: /// Waits until the back executed everything sent so far. void Drain(); + /// Compute or copy work was just sent: what can write GPU-written command lists. + void NoteWriter() { + last_writer_position = write_position; + } + /// Waits until the back executed the compute and copy work sent so far (not what came after). + void WaitForWriters(); + private: enum class Kind : u64 { Wrap, ///< The rest of the ring is unused: continue at its start @@ -156,6 +163,7 @@ private: // Front side. u64 write_position{}; + u64 last_writer_position{}; u64 published_local{}; u64 consumed_cache{}; /// Words between publishes: small right after a drain, while the back has nothing to do. @@ -169,6 +177,8 @@ private: alignas(64) std::atomic published{}; alignas(64) std::atomic consumed{}; + /// A position the front waits for (WaitForWriters): the back stores `consumed` once past it. + alignas(64) std::atomic wait_target{~u64{0}}; alignas(64) std::atomic back_waiting{}; std::atomic wake{}; std::jthread back_thread; diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index db8c92adfd..92714d52d2 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -43,11 +43,43 @@ inline void Nx2MarkGpuWritten(u64 device_address, u64 size) { } } +// GPU-written device pages the front has read command lists from (same layout as above). The +// buffer cache copies these pages into host memory right after the compute or copy work that +// writes them and submits, so the front reads the copy once the GPU is done instead of draining +// the split and flushing the caches (BufferCache::Nx2ReadListCopy). +inline std::array, (u64{1} << (36 - Nx2WrittenPageBits)) / 64> g_nx2_command_list_pages{}; +inline std::atomic g_nx2_has_command_list_pages{false}; + +inline void Nx2MarkCommandListPage(u64 device_address) { + const u64 page = device_address >> Nx2WrittenPageBits; + if (page >= g_nx2_command_list_pages.size() * 64) { + return; + } + auto& word = g_nx2_command_list_pages[page / 64]; + const u64 bit = u64{1} << (page % 64); + if ((word.load(std::memory_order_relaxed) & bit) == 0) { + word.fetch_or(bit, std::memory_order_relaxed); + g_nx2_has_command_list_pages.store(true, std::memory_order_relaxed); + } +} + +[[nodiscard]] inline bool Nx2IsCommandListPage(u64 device_address) { + const u64 page = device_address >> Nx2WrittenPageBits; + if (page >= g_nx2_command_list_pages.size() * 64) { + return false; + } + return (g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> (page % 64)) & 1; +} + /// A new emulation session starts: no page has been written by its GPU yet. inline void Nx2ClearGpuWrittenPages() { for (auto& word : g_nx2_gpu_written_pages) { word.store(0, std::memory_order_relaxed); } + for (auto& word : g_nx2_command_list_pages) { + word.store(0, std::memory_order_relaxed); + } + g_nx2_has_command_list_pages.store(false, std::memory_order_relaxed); } [[nodiscard]] inline bool Nx2IsGpuWritten(u64 device_address) { diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 8486d44ac9..e9eaa2b3c2 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -94,6 +94,20 @@ public: virtual void FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) = 0; + /// Switch 2, from the GPU thread's front: where a GPU-written command list's words are. + enum class GpuWrittenList { + Unknown, ///< Neither: the caller drains, flushes the page and notes it synced + Copied, ///< `out` (within one 4 KiB page) is filled from the copy made after its writer + Synced, ///< Guest memory holds the page: no GPU work wrote it since it was flushed + }; + virtual GpuWrittenList ReadGpuWrittenCommandList([[maybe_unused]] DAddr addr, + [[maybe_unused]] std::span out) { + return GpuWrittenList::Unknown; + } + + /// Switch 2: the front flushed this command list page with the split drained. + virtual void NoteGpuWrittenCommandListSynced([[maybe_unused]] DAddr page) {} + /// Check if the the specified memory area requires flushing to CPU Memory. virtual bool MustFlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) = 0; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index fe0ab3a28e..2449265509 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -453,6 +453,10 @@ void BufferCacheRuntime::Wait(u64 tick) { scheduler.Wait(tick); } +void BufferCacheRuntime::WaitSubmittedTick(u64 tick) { + scheduler.GetMasterSemaphore().Wait(tick); +} + void BufferCacheRuntime::Finish() { scheduler.Finish(); } diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index ff854445b5..d3295ec6ed 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -122,6 +122,9 @@ public: void Wait(u64 tick); + /// Waits for a submitted tick without flushing: safe on threads other than the GPU thread. + void WaitSubmittedTick(u64 tick); + void Finish(); u64 GetDeviceLocalMemory() const; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index d3dbba6163..978f8be547 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -922,6 +922,11 @@ void RasterizerVulkan::DispatchCompute() { return; } std::scoped_lock lock{texture_cache.mutex, buffer_cache.mutex}; + // Command list pages the dispatch writes are copied for the split's front right after it. + buffer_cache.Nx2CollectListCopies(true); + SCOPE_EXIT { + buffer_cache.Nx2CollectListCopies(false); + }; if (!pipeline->Configure(*kepler_compute, *gpu_memory, scheduler, buffer_cache, texture_cache)) { if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute resources unavailable"); @@ -944,6 +949,7 @@ void RasterizerVulkan::DispatchCompute() { } cmdbuf.DispatchIndirect(indirect_buffer, indirect_offset); }); + SubmitNx2ListCopies(); return; } const std::array dim{qmd.grid_dim_x, qmd.grid_dim_y, qmd.grid_dim_z}; @@ -966,6 +972,7 @@ void RasterizerVulkan::DispatchCompute() { } cmdbuf.Dispatch(dim[0], dim[1], dim[2]); }); + SubmitNx2ListCopies(); // Log compute dispatch if (GPU::Logging::IsActive() && @@ -1007,6 +1014,29 @@ void Vulkan::RasterizerVulkan::DisableGraphicsUniformBuffer(size_t stage, u32 in void RasterizerVulkan::FlushAll() {} +void RasterizerVulkan::SubmitNx2ListCopies() { + if (buffer_cache.Nx2CopyListPages(scheduler.CurrentTick())) { + scheduler.Flush(); + } +} + +VideoCore::RasterizerInterface::GpuWrittenList RasterizerVulkan::ReadGpuWrittenCommandList( + DAddr addr, std::span out) { + // No cache lock: the front only reads the copies, under their own lock. + switch (buffer_cache.Nx2ReadListCopy(addr, out)) { + case BufferCache::Nx2ListRead::Copied: + return GpuWrittenList::Copied; + case BufferCache::Nx2ListRead::Synced: + return GpuWrittenList::Synced; + default: + return GpuWrittenList::Unknown; + } +} + +void RasterizerVulkan::NoteGpuWrittenCommandListSynced(DAddr page) { + buffer_cache.Nx2NoteListSynced(page); +} + void RasterizerVulkan::FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which) { if (addr == 0 || size == 0) { return; @@ -1319,6 +1349,8 @@ void RasterizerVulkan::AccelerateInlineToMemory(GPUVAddr address, size_t copy_si bool uploaded; { std::scoped_lock lock{buffer_cache.mutex}; + // Guest memory has the words now: a command list copy of the page no longer does. + buffer_cache.Nx2NoteListHostWrite(*cpu_addr, copy_size); uploaded = buffer_cache.InlineMemory(*cpu_addr, copy_size, memory); if (!uploaded && !defer) { buffer_cache.WriteMemory(*cpu_addr, copy_size); @@ -1414,7 +1446,13 @@ bool AccelerateDMA::BufferClear(GPUVAddr src_address, u64 amount, u32 value) { bool AccelerateDMA::BufferCopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount) { std::scoped_lock lock{buffer_cache.mutex}; - return buffer_cache.DMACopy(src_address, dest_address, amount); + buffer_cache.Nx2CollectListCopies(true); + const bool copied = buffer_cache.DMACopy(src_address, dest_address, amount); + if (buffer_cache.Nx2CopyListPages(scheduler.CurrentTick())) { + scheduler.Flush(); + } + buffer_cache.Nx2CollectListCopies(false); + return copied; } template diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index fc282fbe8d..dbb6ebdca8 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -99,6 +99,8 @@ public: void FlushAll() override; void FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; + GpuWrittenList ReadGpuWrittenCommandList(DAddr addr, std::span out) override; + void NoteGpuWrittenCommandListSynced(DAddr page) override; bool MustFlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; VideoCore::RasterizerDownloadArea GetFlushArea(DAddr addr, u64 size) override; @@ -165,6 +167,10 @@ private: void FlushWork(); + /// Records the copies of the command list pages the work just recorded wrote and submits, so + /// the GPU makes them now (BufferCache::Nx2CopyListPages). + void SubmitNx2ListCopies(); + void UpdateDynamicStates(); void HandleTransformFeedback(); -- cgit v1.2.3