From bef001c1515a13a9b8e361a5a7d372fc2e01094c Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:39:08 -0400 Subject: nx2: read command lists the GPU wrote from the GPU's copy Mario Kart World runs small command lists (20-byte entries) that its culling dispatches fill in each frame. Accurate GPU mode reads every command list through the caches; Fast mode reads guest memory directly (on the split's front thread too), where those entries were still the words of an older frame, so the draws behind them were left out: the trees around the courses vanished in Fast mode only. The buffer cache now also marks the device pages the GPU writes (shader stores, copies of GPU-written data) in a bitmap the split's front can read. A command list on such a page waits for the back to run what was queued before it and is flushed when its words are still only in host buffers. In a race that is ~5% of command lists (13,500 a second of 262,000) and ~400 flushes a second. --- src/video_core/buffer_cache/buffer_cache.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/video_core/buffer_cache/buffer_cache.h') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 42d7ba066d..a8f1edc68d 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -17,6 +17,7 @@ #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/guest_memory.h" #include "video_core/host1x/gpu_device_memory_manager.h" +#include "video_core/nx2.h" #include "video_core/nx2_profile.h" #include "video_core/texture_cache/util.h" @@ -289,6 +290,9 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am const bool has_new_downloads = tmp_intervals.size() != 0; for (const auto& pair : tmp_intervals) { gpu_modified_ranges.Add(pair.first, pair.second); + if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { + Tegra::Nx2MarkGpuWritten(pair.first, pair.second); + } } const auto& copy = copies[0]; src_buffer.MarkUsage(copy.src_offset, copy.size); @@ -1606,6 +1610,9 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 if (AsyncBufferDownloads()) { uncommitted_gpu_modified_ranges.Add(device_addr, size); } + if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { + Tegra::Nx2MarkGpuWritten(device_addr, size); + } } template -- cgit v1.2.3 From 594e07ee5302098e417858608dd03c46336274f5 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:11:09 -0400 Subject: buffer_cache: collect Switch 2 buffers from 512 MiB The collector ran on Switch 2 titles only once their buffers reached 1 GiB (aggressively at 2 GiB), so geometry of scenes long gone stayed in VRAM: Mario Kart World's title screen grew from 0.2 to 0.9 GB of buffers in ten minutes, most of a ~1.4 GB climb in VRAM in use. It now starts at 512 MiB (aggressive at 1 GiB); it only evicts buffers unused for 120 frames or more. (cherry picked from commit fb5d797636f2269591b68fda7c8d65787a9e3a9f) --- src/video_core/buffer_cache/buffer_cache.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/video_core/buffer_cache/buffer_cache.h') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e90af3214d..53874e75cc 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -141,9 +141,10 @@ void BufferCache

::TickFrame() { // title 1's buffers were ~200 MiB of an 8 GB GPU's ~6.8 GB in use, yet this collector // ran aggressively every frame on the device-wide figure, evicting up to 64 buffers (and // downloading the GPU-written ones, ~5% of the GPU thread) that were recreated later. - // Collect when the buffers themselves are large. - if (buffer_bytes >= 1_GiB) { - RunGarbageCollector(buffer_bytes >= 2_GiB); + // Collect when the buffers themselves are large. At 1 GiB it held geometry of scenes + // long gone: title 1's title screen grew to ~0.9 GB of buffers in ten minutes. + if (buffer_bytes >= 512_MiB) { + RunGarbageCollector(buffer_bytes >= 1_GiB); } } else if (total_used_memory >= minimum_memory) { RunGarbageCollector(total_used_memory >= critical_memory); -- cgit v1.2.3 From 86968cb5ce3195cb131eeb0ef68d2d8a4860f7df Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:42:26 -0500 Subject: buffer_cache: mark guest memory large buffer uploads came from cold Like the texture cache's uploads: guest ranges of 256 KiB or more uploaded to the host GPU are mostly geometry the guest rarely reads again, so every 600 frames they are marked cold and the OS pages them out first when it needs RAM. (cherry picked from commit 5c942a5158a0f9769553eeba21b6121f277d0dc5) --- src/video_core/buffer_cache/buffer_cache.h | 11 +++++++++++ src/video_core/buffer_cache/buffer_cache_base.h | 2 ++ 2 files changed, 13 insertions(+) (limited to 'src/video_core/buffer_cache/buffer_cache.h') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 42d7ba066d..633f8760e4 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -100,6 +100,12 @@ void BufferCache

::TickFrame() { return; } runtime.TickFrame(slot_buffers); + if (frame_tick % 600 == 0 && !uploaded_buffer_ranges.empty()) { + for (const auto& [address, size] : uploaded_buffer_ranges) { + device_memory.HintCold(address, size); + } + uploaded_buffer_ranges.clear(); + } if (static u64 frames{}; ++frames % 600 == 0) { u64 total{}; size_t count{}; @@ -1923,6 +1929,11 @@ void BufferCache

::MappedUploadMemory([[maybe_unused]] Buffer& buffer, u8* const src_pointer = staging_pointer.data() + copy.src_offset; const DAddr device_addr = buffer.CpuAddr() + copy.dst_offset; device_memory.ReadBlockUnsafe(device_addr, src_pointer, copy.size); + // Large uploads are mostly geometry the host GPU now has its own copy of: TickFrame + // marks their guest memory cold (see TextureCache::RefreshContents). + if (copy.size >= 256_KiB) { + uploaded_buffer_ranges.emplace_back(device_addr, copy.size); + } // Apply the staging offset copy.src_offset += upload_staging.offset; } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 45a1836341..0db8bdb1eb 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -549,6 +549,8 @@ private: size_t immediate_buffer_capacity = 0; Common::ScratchBuffer immediate_buffer_alloc; + /// Guest memory large uploads came from since the last cold hint (device address, size). + std::vector> uploaded_buffer_ranges; struct LRUItemParams { using ObjectType = BufferId; -- cgit v1.2.3 From 808feaf3bf34b3c108ff30baae1d2261f216f4f3 Mon Sep 17 00:00:00 2001 From: Duck912 <350528965+Duck912@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:42:19 -0500 Subject: video_core: keep root tables and bind table 0 as constant bank 17 The 3D methods 0x504/0x508 write eight 256-byte root tables, which shaders read as constant banks 24-31; NVN2 keeps push constants in table 0 (bank 17 after the Ampere lowering). Fire Emblem: Fortune's Weave keeps its material constants there and never binds bank 17, and the blit constants pass replaced those reads with render-target sizes, flooding its 3D scenes red. Maxwell3D now keeps the tables, the buffer cache streams table 0 as bank 17 when the game has not bound one, and the blit constants pass leaves shaders that read bank 17 outside its parameter block alone. Assisted-by: Claude Opus 5.5 (debugging, implementation and drafting) Signed-off-by: Duck912 <350528965+Duck912@users.noreply.github.com> --- src/shader_recompiler/ir_opt/position_pass.cpp | 21 +++++++++++++++++++++ src/video_core/buffer_cache/buffer_cache.h | 20 ++++++++++++++++++++ src/video_core/engines/maxwell_3d.cpp | 15 +++++++++++++++ src/video_core/engines/maxwell_3d.h | 12 ++++++++++++ 4 files changed, 68 insertions(+) (limited to 'src/video_core/buffer_cache/buffer_cache.h') diff --git a/src/shader_recompiler/ir_opt/position_pass.cpp b/src/shader_recompiler/ir_opt/position_pass.cpp index 4890346944..23b1200cfc 100644 --- a/src/shader_recompiler/ir_opt/position_pass.cpp +++ b/src/shader_recompiler/ir_opt/position_pass.cpp @@ -83,6 +83,27 @@ void BlitConstantsPass(Environment& env, IR::Program& program) { if (env.ShaderStage() != Stage::VertexB && env.ShaderStage() != Stage::Fragment) { return; } + // Bank 24 is root table 0, which games may load themselves (3D methods 0x504/0x508; the buffer + // cache binds it as bank 17): Fire Emblem: Fortune's Weave keeps its material push constants + // there. A shader reading bank 17 outside the blit block reads such a table and keeps its reads. + const auto in_blit_block = [](u32 offset) { + return offset < 0x20 || (offset >= 0x30 && offset < 0x4c) || (offset >= 0x50 && offset < 0x60); + }; + for (IR::Block* const block : program.post_order_blocks) { + for (IR::Inst& inst : block->Instructions()) { + const IR::Opcode opcode{inst.GetOpcode()}; + if ((opcode != IR::Opcode::GetCbufU32 && opcode != IR::Opcode::GetCbufF32 && + opcode != IR::Opcode::GetCbufU32x2) || + !inst.Arg(0).IsImmediate() || inst.Arg(0).U32() != 17 || !inst.Arg(1).IsImmediate()) { + continue; + } + const u32 offset{inst.Arg(1).U32()}; + const bool pair{opcode == IR::Opcode::GetCbufU32x2}; + if (!in_blit_block(offset) || (pair && !in_blit_block(offset + 4))) { + return; + } + } + } bool used{}; for (IR::Block* const block : program.post_order_blocks) { for (IR::Inst& inst : block->Instructions()) { diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e90af3214d..2fc74cb6c8 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -994,6 +994,26 @@ void BufferCache

::BindHostGraphicsUniformBuffers(size_t stage) { template void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind) { ++channel_state->uniform_cache_shots[0]; + // Bank 24 (root table 0, NVN2's push constants) is read from bank 17, which such games do not + // bind: stream the table's current words as that buffer. + if (index == 17 && maxwell3d->root_tables.loaded && channel_state->uniform_buffers[stage][index].size == 0) { + const auto& words = maxwell3d->root_tables.tables[0]; + const u32 size = (*channel_state->uniform_buffer_sizes)[stage][index]; + const u32 copied = (std::min)(size, static_cast(sizeof(words))); + if constexpr (IS_OPENGL) { + std::array data{}; + std::memcpy(data.data(), words.data(), copied); + runtime.BindFastUniformBuffer(stage, binding_index, copied); + runtime.PushFastUniformBuffer(stage, binding_index, std::span(data.data(), copied)); + } else { + const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + std::memset(span.data(), 0, span.size()); + std::memcpy(span.data(), words.data(), copied); + } + channel_state->fast_bound_uniform_buffers[stage] |= 1u << binding_index; + channel_state->uniform_buffer_binding_sizes[stage][binding_index] = size; + return; + } const Binding& binding = channel_state->uniform_buffers[stage][index]; const DAddr device_addr = binding.device_addr; const u32 size = (std::min)(binding.size, (*channel_state->uniform_buffer_sizes)[stage][index]); diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 6adfcf89b8..4001284127 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -164,6 +164,8 @@ bool Maxwell3D::IsMethodExecutable(u32 method) { case MAXWELL3D_REG_INDEX(vertex_array_instance_first): case MAXWELL3D_REG_INDEX(vertex_array_instance_subsequent): case MAXWELL3D_REG_INDEX(draw_texture.src_y0): + case RootTables::SelectMethod: + case RootTables::LoadMethod: case MAXWELL3D_REG_INDEX(wait_for_idle): case MAXWELL3D_REG_INDEX(shadow_ram_control): case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): @@ -346,6 +348,17 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume LOG_DEBUG(HW_GPU, "Native GPU surface clear: value={:#x}", argument); } switch (method) { + case RootTables::SelectMethod: + root_tables.selected = argument & 7; + root_tables.offset = (argument >> 8) & 0xff; + return; + case RootTables::LoadMethod: + if (root_tables.offset < 256 && root_tables.offset % 4 == 0) { + root_tables.tables[root_tables.selected][root_tables.offset / 4] = argument; + root_tables.loaded = true; + } + root_tables.offset += 4; + return; case MAXWELL3D_REG_INDEX(wait_for_idle): return rasterizer->WaitForIdle(); case MAXWELL3D_REG_INDEX(shadow_ram_control): @@ -819,6 +832,8 @@ bool FrontHandledOutsideDrawManager(u32 method) { return true; } switch (method) { + case Maxwell3D::RootTables::SelectMethod: + case Maxwell3D::RootTables::LoadMethod: case MAXWELL3D_REG_INDEX(wait_for_idle): case MAXWELL3D_REG_INDEX(shadow_ram_control): case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 0121f1a01c..a32b31fd1e 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3160,6 +3160,18 @@ public: std::array& GetNativeDataRam() { return native_data_ram; } std::span GetNativeMacroPositions() { return native_macro_context ? std::span{native_macro_context->positions} : std::span{macro_positions}; } + /// Root tables: eight 256-byte tables written inline (method 0x504 selects a table, bits 0-2, + /// and a byte offset, bits 8-15; each 0x508 writes the next word) that shaders read as constant + /// banks 24-31. NVN2 keeps push constants in bank 24, which the Ampere lowering reads as bank 17. + struct RootTables { + static constexpr u32 SelectMethod = 0x504 / 4; + static constexpr u32 LoadMethod = 0x508 / 4; + std::array, 8> tables{}; + u32 selected{}; + u32 offset{}; + bool loaded{}; + } root_tables; + /// Write the value to the register identified by method. void CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) override; -- cgit v1.2.3 From 46b2577bf78870ed85cbb790a9dc3c82cc77f4a7 Mon Sep 17 00:00:00 2001 From: ymatrix1 <10088631+ymatrix1@users.noreply.github.com> Date: Mon, 2 Nov 2026 04:41:15 -0300 Subject: video_core/buffer_cache: fast-guard stage buffer updates and uniform buffer disable (#264) --- src/video_core/buffer_cache/buffer_cache.h | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'src/video_core/buffer_cache/buffer_cache.h') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 2fc74cb6c8..e09a435dcc 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -379,6 +379,10 @@ template void BufferCache

::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr, u32 size) { const std::optional device_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + if (!device_addr) { + channel_state->uniform_buffers[stage][index] = NULL_BINDING; + return; + } const Binding binding{ .device_addr = *device_addr, .size = size, @@ -389,7 +393,11 @@ void BufferCache

::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr template void BufferCache

::DisableGraphicsUniformBuffer(size_t stage, u32 index) { - channel_state->uniform_buffers[stage][index] = NULL_BINDING; + auto& current = channel_state->uniform_buffers[stage][index]; + if (current.device_addr == 0 && current.size == 0 && !current.buffer_id) [[likely]] { + return; + } + current = NULL_BINDING; } template @@ -1514,6 +1522,9 @@ void BufferCache

::UpdateDrawIndirect() { template void BufferCache

::UpdateUniformBuffers(size_t stage) { + if (channel_state->enabled_uniform_buffer_masks[stage] == 0) [[likely]] { + return; + } ForEachEnabledBit(channel_state->enabled_uniform_buffer_masks[stage], [&](u32 index) { Binding& binding = channel_state->uniform_buffers[stage][index]; if (binding.buffer_id) { @@ -1531,6 +1542,9 @@ void BufferCache

::UpdateUniformBuffers(size_t stage) { template void BufferCache

::UpdateStorageBuffers(size_t stage) { + if (channel_state->enabled_storage_buffers[stage] == 0) [[likely]] { + return; + } ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { // Resolve buffer Binding& binding = channel_state->storage_buffers[stage][index]; @@ -1543,6 +1557,9 @@ void BufferCache

::UpdateStorageBuffers(size_t stage) { template void BufferCache

::UpdateTextureBuffers(size_t stage) { + if (channel_state->enabled_texture_buffers[stage] == 0) [[likely]] { + return; + } ForEachEnabledBit(channel_state->enabled_texture_buffers[stage], [&](u32 index) { Binding& binding = channel_state->texture_buffers[stage][index]; binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); -- cgit v1.2.3 From a407703f6d29a5ab73328b4ededb12cf29b2758d Mon Sep 17 00:00:00 2001 From: Whovian64 <86082882+whovian64@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:45:39 -0500 Subject: Linux and AMD: BC3 flashing on RADV, ASTC decode stalls, lower RAM usage, a buffer cache crash (#260) * host_memory: punch holes in the backing on Linux instead of zero filling freed memory Freeing guest memory zero filled the memfd backing with memset, which commits every page it touches. fallocate(PUNCH_HOLE) hands the pages back to the kernel instead, and they read as zero afterwards. Only the partial pages at the ends are still cleared by hand. Mario Kart World on Linux (RX 9070 XT, 31 GB): peak RSS 23.1 -> 15.9 GB. EDEN_PUNCH_HOLE=0 restores the memset. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * fiber: don't zero fill the stacks, untouched pages stay unbacked The value initialised stack arrays were zero filled when each fiber was made, which commits the whole stack. Left uninitialised, the pages a fiber never touches stay unbacked. Together with the descriptor ring change, about 1.2 GB less RSS in Mario Kart World. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * vulkan: don't zero fill the descriptor update ring Same as the fiber stacks: make_unique value initialised the whole ring for every frame in flight. Entries are always written before they are read, so allocate it without initialising it. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * vulkan: wait for the GPU after an ASTC transcode decode like the other ASTC decodes TranscodeAstcToBcn (astc_recompression Bc1/Bc3 with GPU decode) was the only ASTC decode recorded with wait = false. On RADV (RX 9070 XT, Mesa) Mario Kart World with BC3 flashed heavily all over the screen, while the uncompressed GPU decode, which waits, did not. CPU decode with BC3 does not flash either. Waiting after the transcode removes the flashing; full pipeline barriers around its copy did not, so the wait hides a race somewhere else rather than fixing one in the transcode itself. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * vulkan: submit after an ASTC decode instead of waiting for the GPU by default Every ASTC decode ended with scheduler.Finish(). Over a 19 minute Mario Kart World Grand Prix that stalled the GPU thread 30.7 s in total, up to 1.2 s out of every 5 s while new textures stream in (track sections, race starts), which is where the hitches were. Flushing keeps the ordering that stops the BC3 flashing on RADV without the CPU waiting: no flashing in play, and frames over 100 ms went from dozens to 3 in about 7 minutes. EDEN_NX2_ASTC_FLUSH=0 restores the wait. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * texture_cache: over the expected use, evict costly images unused for 1800 frames first The collector never deletes CostlyLoad images outside aggressive mode, so once a cup moves on, the previous tracks' textures stay resident and the cache sits over its expected use deleting the cheap images still in use. When over the expected use, costly images unused for 1800 frames (half a minute) now go first. Mario Kart World Grand Prix on an RX 9070 XT: frames over 20 ms 8.8% -> 4.0%, 58.7 fps average. EDEN_NX2_TC_OLD_AGE= sets the age, 0 turns it off. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * buffer_cache: no buffer for device addresses past the page table FindBuffer indexed page_table, which covers 2^36 bytes, with any address. Mario Kart World crashed after about 4.5 minutes with a uniform buffer at 0x72253b2f2890 (FindBuffer <- UpdateUniformBuffers <- DrawEnd from a macro), reading far past the table. Treat such an address as no buffer. Where that address comes from is still unknown; this only stops the crash. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) --- src/common/fiber.cpp | 4 ++-- src/common/host_memory.cpp | 22 ++++++++++++++++++++++ src/video_core/buffer_cache/buffer_cache.h | 4 ++-- src/video_core/renderer_vulkan/vk_compute_pass.cpp | 6 +++--- .../renderer_vulkan/vk_texture_cache.cpp | 2 +- .../renderer_vulkan/vk_update_descriptor.cpp | 8 ++++---- .../renderer_vulkan/vk_update_descriptor.h | 3 ++- src/video_core/texture_cache/texture_cache.h | 16 ++++++++++++++-- 8 files changed, 50 insertions(+), 15 deletions(-) (limited to 'src/video_core/buffer_cache/buffer_cache.h') diff --git a/src/common/fiber.cpp b/src/common/fiber.cpp index 0e7f28c5b9..08558a0026 100644 --- a/src/common/fiber.cpp +++ b/src/common/fiber.cpp @@ -25,8 +25,8 @@ struct Fiber::FiberImpl { FiberImpl() {} u32 canary_1 = CANARY_VALUE; - std::array stack{}; - std::array rewind_stack{}; + std::array stack; + std::array rewind_stack; u32 canary_2 = CANARY_VALUE; boost::context::detail::fcontext_t context{}; diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 75a9da80e9..147eba8477 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -51,6 +51,7 @@ #endif // ^^^ POSIX ^^^ +#include #include #include @@ -517,6 +518,14 @@ public: , virtual_size{virtual_size_} {} +#ifdef __linux__ + bool PunchHole(size_t offset, size_t length) { + return fd >= 0 && + fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, static_cast(offset), + static_cast(length)) == 0; + } +#endif + bool Init() { long page_size = sysconf(_SC_PAGESIZE); ASSERT_MSG(page_size == 0x1000, "page size {:#x} is incompatible with 4K paging", page_size); @@ -791,6 +800,19 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, MemoryPermission } void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value) { +#ifdef __linux__ + const size_t first = AlignUp(physical_offset, PageAlignment); + const size_t last = AlignDown(physical_offset + length, PageAlignment); + static const bool punch = [] { + const char* value = std::getenv("EDEN_PUNCH_HOLE"); + return value == nullptr || value[0] != '0'; + }(); + if (punch && fill_value == 0 && impl && last > first && impl->PunchHole(first, last - first)) { + std::memset(backing_base + physical_offset, 0, first - physical_offset); + std::memset(backing_base + last, 0, physical_offset + length - last); + return; + } +#endif std::memset(backing_base + physical_offset, fill_value, length); } diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e09a435dcc..a3358048dc 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1651,10 +1651,10 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 template BufferId BufferCache

::FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible) { - if (device_addr == 0) { + const u64 page = device_addr >> CACHING_PAGEBITS; + if (device_addr == 0 || page >= page_table.size()) { return NULL_BUFFER_ID; } - const u64 page = device_addr >> CACHING_PAGEBITS; const BufferId buffer_id = page_table[page]; if (buffer_id) { Buffer& buffer = slot_buffers[buffer_id]; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index 691be728d2..cb8dcde42f 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -801,11 +801,11 @@ void ASTCDecoderPass::AssembleInto(const VideoCommon::ImageInfo& info, u32 skip, cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier); }); - // Each decode ends with the GPU thread waiting for the host GPU to go idle. Measured here - // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=1 submits without waiting instead. + // Each decode ends with a submit to the host GPU. Measured here + // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=0 waits for the host GPU to go idle instead. static const bool flush_only = [] { const char* value = std::getenv("EDEN_NX2_ASTC_FLUSH"); - return value != nullptr && value[0] == '1'; + return value == nullptr || value[0] != '0'; }(); if (!target.wait) { return; diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 4e428229f6..b0e5eacb32 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -4267,7 +4267,7 @@ void TextureCacheRuntime::TranscodeAstcToBcn( .aspect_mask = VK_IMAGE_ASPECT_COLOR_BIT, .initialized = false, .level_view = [&pending](s32 level) { return *pending.views[level]; }, - .wait = false, + .wait = true, }, map, swizzles); diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp index 742a8d9969..29f3ee71a1 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp @@ -20,10 +20,10 @@ UpdateDescriptorQueue::UpdateDescriptorQueue(const Device& device_, size_t frame bool supports_descriptor_buffer_) : device{device_}, frame_payload_size{frame_payload_size_}, supports_descriptor_buffer{supports_descriptor_buffer_}, - payload(frame_payload_size_ * FRAMES_IN_FLIGHT) + payload(std::make_unique_for_overwrite(frame_payload_size_ * FRAMES_IN_FLIGHT)) { - payload_start = payload.data(); - payload_cursor = payload.data(); + payload_start = payload.get(); + payload_cursor = payload.get(); } UpdateDescriptorQueue::~UpdateDescriptorQueue() = default; @@ -32,7 +32,7 @@ void UpdateDescriptorQueue::TickFrame() { if (++frame_index >= FRAMES_IN_FLIGHT) { frame_index = 0; } - payload_start = payload.data() + frame_index * frame_payload_size; + payload_start = payload.get() + frame_index * frame_payload_size; payload_cursor = payload_start; } diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.h b/src/video_core/renderer_vulkan/vk_update_descriptor.h index 9f30e65dbd..4d393376b4 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.h +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -122,7 +123,7 @@ private: DescriptorUpdateEntry* payload_cursor = nullptr; DescriptorUpdateEntry* payload_start = nullptr; const DescriptorUpdateEntry* upload_start = nullptr; - std::vector payload; + std::unique_ptr payload; }; // TODO: should these be separate classes instead? diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 32775b8719..55ec4c64df 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -125,6 +125,7 @@ template void TextureCache

::RunGarbageCollector() { bool high_priority_mode = false; bool aggressive_mode = false; + bool evict_old_costly = false; u64 ticks_to_destroy = 0; size_t num_iterations = 0; const auto Configure = [&](bool allow_aggressive) { @@ -139,7 +140,8 @@ void TextureCache

::RunGarbageCollector() { ticks_to_destroy = (std::max)(ticks_to_destroy, min_age); num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10); }; - const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) { + const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode, + &evict_old_costly](ImageId image_id) { if (num_iterations == 0) { return true; } @@ -149,7 +151,8 @@ void TextureCache

::RunGarbageCollector() { return false; } const bool must_download = IsDownloadable(image) && False(image.flags & ImageFlagBits::BadOverlap); - if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) { + if ((!aggressive_mode && !evict_old_costly && True(image.flags & ImageFlagBits::CostlyLoad)) || + (!high_priority_mode && must_download)) { return false; } if (must_download) { @@ -176,6 +179,15 @@ void TextureCache

::RunGarbageCollector() { return false; }; Configure(false); + static const u64 old_age = [] { + const char* value = std::getenv("EDEN_NX2_TC_OLD_AGE"); + return value != nullptr ? std::strtoull(value, nullptr, 10) : 1800ULL; + }(); + if (high_priority_mode && old_age != 0 && frame_tick > old_age) { + evict_old_costly = true; + lru_cache.ForEachItemBelow(frame_tick - old_age, Cleanup); + evict_old_costly = false; + } lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); if (total_used_memory >= critical_memory) { Configure(true); -- cgit v1.2.3 From 8f1435f8db104351fa3388551eee9d12d2043b40 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:24:19 -0500 Subject: nx2: read GPU-written command lists from copies instead of draining The trees fix (#52) drains the split and flushes for every command list on a GPU-written page: in a Mario Kart World race ~13,500 lists and ~400 GPU round trips a second, during which the back has nothing to do. The replay ran at ~40 fps instead of ~45. The front now marks the pages those lists are on. The buffer cache notes writes to them; after a compute dispatch or DMA copy that writes one it copies the page into host memory and submits. The front waits for the back only up to the last compute or copy work it sent (not everything queued), then reads the copy once the GPU is done with it. Pages the GPU wrote once, long before, and the CPU now writes lists to have no copy: their first list drains and flushes the whole page, which is then read from guest memory until GPU work writes it again. CPU writes and inline uploads reach guest memory, so they keep such a page synced but make a copy stale. A page a draw writes always drains. Mario Kart World time-trial replay, Radeon 780M: ~40 -> ~45 fps, trees drawn, no list drains left. --- src/video_core/buffer_cache/buffer_cache.h | 191 +++++++++++++++++++++ src/video_core/buffer_cache/buffer_cache_base.h | 51 ++++++ src/video_core/dma_pusher.cpp | 98 ++++++++++- src/video_core/dma_pusher.h | 5 + src/video_core/gpu_split.cpp | 30 +++- src/video_core/gpu_split.h | 10 ++ src/video_core/nx2.h | 32 ++++ src/video_core/rasterizer_interface.h | 14 ++ src/video_core/renderer_vulkan/vk_buffer_cache.cpp | 4 + src/video_core/renderer_vulkan/vk_buffer_cache.h | 3 + src/video_core/renderer_vulkan/vk_rasterizer.cpp | 40 ++++- src/video_core/renderer_vulkan/vk_rasterizer.h | 6 + 12 files changed, 478 insertions(+), 6 deletions(-) (limited to 'src/video_core/buffer_cache/buffer_cache.h') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 53bbae7610..cd176dec27 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -174,6 +174,7 @@ void BufferCache

::UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size template void BufferCache

::WriteMemory(DAddr device_addr, u64 size) { + Nx2NoteListHostWrite(device_addr, size); if (IsRegionGpuModified(device_addr, size)) { ClearDownload(device_addr, size); gpu_modified_ranges.Subtract(device_addr, size); @@ -194,6 +195,7 @@ void BufferCache

::CachedWriteMemory(DAddr device_addr, u64 size) { return; } + Nx2NoteListHostWrite(device_addr, size); tmp_buffer.resize_destructive(size); device_memory.ReadBlockUnsafe(device_addr, tmp_buffer.data(), size); @@ -301,6 +303,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am gpu_modified_ranges.Add(pair.first, pair.second); if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { Tegra::Nx2MarkGpuWritten(pair.first, pair.second); + Nx2NoteListWrite(buffer_b, pair.first, pair.second); } } const auto& copy = copies[0]; @@ -1660,6 +1663,194 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 } if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { Tegra::Nx2MarkGpuWritten(device_addr, size); + Nx2NoteListWrite(buffer_id, device_addr, size); + } +} + +template +void BufferCache

::Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 size) { + if constexpr (!IS_OPENGL) { + if (size == 0 || !Tegra::g_nx2_has_command_list_pages.load(std::memory_order_relaxed)) { + return; + } + constexpr u64 page_bits = Tegra::Nx2WrittenPageBits; + const u64 last = (std::min)((device_addr + size - 1) >> page_bits, + Tegra::g_nx2_command_list_pages.size() * 64 - 1); + for (u64 page = device_addr >> page_bits; page <= last;) { + // Storage buffers can be large: skip 64 pages at a time where none is a list page. + const u64 word = + Tegra::g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> + (page % 64); + if (word == 0) { + page = (page / 64 + 1) * 64; + continue; + } + if (word & 1) { + const DAddr page_addr = page << page_bits; + { + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page_addr]; + ++entry.writes; + if (!nx2_collect_list_copies && !entry.draw_written) { + entry.draw_written = true; + LOG_INFO(HW_GPU, "Command list page {:#x} is written by draws too: its " + "lists drain the split", + page_addr); + } + } + if (nx2_collect_list_copies) { + nx2_list_copy_requests.emplace_back(buffer_id, page_addr); + } + } + ++page; + } + } +} + +template +bool BufferCache

::Nx2CopyListPages(u64 tick) { + if constexpr (IS_OPENGL) { + return false; + } else { + if (nx2_list_copy_requests.empty()) { + return false; + } + std::ranges::sort(nx2_list_copy_requests, {}, &std::pair::second); + constexpr DAddr page_size = DAddr{1} << Tegra::Nx2WrittenPageBits; + bool recorded = false; + for (size_t i = 0; i < nx2_list_copy_requests.size(); ++i) { + const auto [buffer_id, page] = nx2_list_copy_requests[i]; + if (i + 1 < nx2_list_copy_requests.size() && + nx2_list_copy_requests[i + 1].second == page) { + continue; + } + Buffer& buffer = slot_buffers[buffer_id]; + const DAddr begin = (std::max)(page, buffer.CpuAddr()); + const DAddr end = + (std::min)(page + page_size, buffer.CpuAddr() + buffer.SizeBytes()); + if (begin >= end) { + continue; + } + Nx2Staging staging = runtime.DownloadStagingBuffer(page_size, true); + const BufferCopy copy{ + .src_offset = buffer.Offset(begin), + .dst_offset = staging.offset + (begin - page), + .size = end - begin, + }; + buffer.MarkUsage(copy.src_offset, copy.size); + runtime.CopyBuffer(staging.buffer, buffer, std::span{©, 1}, true); + recorded = true; + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page]; + if (entry.copy) { + nx2_retired_list_copies.emplace_back(entry.copy_tick, std::move(*entry.copy)); + } + entry.copy = staging; + entry.copy_writes = entry.writes; + entry.copy_tick = tick; + entry.copy_begin = static_cast(begin - page); + entry.copy_end = static_cast(end - page); + } + nx2_list_copy_requests.clear(); + std::erase_if(nx2_retired_list_copies, [this](auto& retired) { + if (!runtime.IsFree(retired.first)) { + return false; + } + runtime.FreeDeferredStagingBuffer(retired.second); + return true; + }); + return recorded; + } +} + +template +auto BufferCache

::Nx2ReadListCopy(DAddr device_addr, std::span out) -> Nx2ListRead { + if constexpr (IS_OPENGL) { + return Nx2ListRead::Unknown; + } else { + constexpr DAddr page_size = DAddr{1} << Tegra::Nx2WrittenPageBits; + const DAddr page = device_addr & ~(page_size - 1); + const u64 offset = device_addr - page; + if (offset + out.size() > page_size) { + return Nx2ListRead::Unknown; + } + const auto current = [&](const Nx2ListPage& entry) { + return entry.copy && !entry.draw_written && entry.copy_writes == entry.writes && + offset >= entry.copy_begin && offset + out.size() <= entry.copy_end; + }; + u64 tick{}; + u64 writes{}; + { + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page); + if (it == nx2_list_pages.end()) { + return Nx2ListRead::Unknown; + } + const Nx2ListPage& entry = it->second; + // Pages the CPU writes the lists of (GPU-written once, long before) stay synced. + if (!entry.draw_written && entry.synced_writes == entry.writes) { + return Nx2ListRead::Synced; + } + if (!current(entry)) { + return Nx2ListRead::Unknown; + } + // The lists after the first of a batch find the copy done: read it at once. + if (runtime.IsFree(entry.copy_tick)) { + std::memcpy(out.data(), entry.copy->mapped_span.data() + offset, out.size()); + return Nx2ListRead::Copied; + } + tick = entry.copy_tick; + writes = entry.copy_writes; + } + // Submitted with the work that wrote the page (Nx2CopyListPages's caller flushes). + runtime.WaitSubmittedTick(tick); + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page); + if (it == nx2_list_pages.end() || !current(it->second) || it->second.copy_tick != tick || + it->second.copy_writes != writes) { + return Nx2ListRead::Unknown; + } + std::memcpy(out.data(), it->second.copy->mapped_span.data() + offset, out.size()); + return Nx2ListRead::Copied; + } +} + +template +void BufferCache

::Nx2NoteListSynced(DAddr page) { + if constexpr (!IS_OPENGL) { + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page]; + entry.synced_writes = entry.writes; + } +} + +template +void BufferCache

::Nx2NoteListHostWrite(DAddr device_addr, u64 size) { + if constexpr (!IS_OPENGL) { + if (size == 0 || !Tegra::g_nx2_has_command_list_pages.load(std::memory_order_relaxed)) { + return; + } + constexpr u64 page_bits = Tegra::Nx2WrittenPageBits; + const u64 last = (std::min)((device_addr + size - 1) >> page_bits, + Tegra::g_nx2_command_list_pages.size() * 64 - 1); + for (u64 page = device_addr >> page_bits; page <= last; ++page) { + if (!((Tegra::g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> + (page % 64)) & + 1)) { + continue; + } + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page << page_bits); + if (it == nx2_list_pages.end()) { + continue; + } + Nx2ListPage& entry = it->second; + const bool synced = entry.synced_writes == entry.writes; + ++entry.writes; + if (synced) { + entry.synced_writes = entry.writes; + } + } } } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 47f342932d..1f0c6cf10f 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -14,7 +14,9 @@ #include #include #include +#include #include +#include #include #include "common/container/unordered_map.h" @@ -321,6 +323,28 @@ public: bool DMAClear(GPUVAddr src_address, u64 amount, u32 value); + /// Switch 2 GPU-written command lists (nx2.h, g_nx2_command_list_pages). While collecting (a + /// compute dispatch or a DMA copy being recorded), writes to command list pages are noted; + /// Nx2CopyListPages then records copies of those pages into host memory after the work. + void Nx2CollectListCopies(bool collect) { + nx2_collect_list_copies = collect; + if (!collect) { + nx2_list_copy_requests.clear(); + } + } + /// True when it recorded copies, which the GPU completes at `tick` (the caller submits). + bool Nx2CopyListPages(u64 tick); + enum class Nx2ListRead { Unknown, Copied, Synced }; + /// Front thread: Synced when guest memory holds the page `device_addr` lies in (flushed and + /// written by no GPU work since), else Copied with `out` filled from the copy of its last + /// write once the GPU is done with it. Unknown otherwise: the caller drains and flushes. + Nx2ListRead Nx2ReadListCopy(DAddr device_addr, std::span out); + /// Front thread, the split drained: the command list page was flushed to guest memory. + void Nx2NoteListSynced(DAddr page); + /// CPU and inline writes: they reach guest memory, so a synced page stays synced, but its copy + /// no longer holds the page. + void Nx2NoteListHostWrite(DAddr device_addr, u64 size); + /// Return true when a CPU region is modified from the GPU [[nodiscard]] bool IsRegionGpuModified(DAddr addr, size_t size); @@ -440,6 +464,10 @@ private: void MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size, bool needs_sync = false); + /// Counts a GPU write to the command list pages in the range (invalidating their copies) and, + /// while collecting, asks for new copies. + void Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 size); + [[nodiscard]] BufferId FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible); void SynchronizeBufferWrites(Buffer& buffer); @@ -604,6 +632,29 @@ private: // Covers the device address space (MaxwellDeviceTraits::device_virtual_bits). std::array> CACHING_PAGEBITS)> page_table; Common::ScratchBuffer tmp_buffer; + + /// A command list page (Nx2CopyListPages): the GPU writes noted to it and the last copy. + using Nx2Staging = decltype(std::declval().DownloadStagingBuffer(size_t{}, true)); + struct Nx2ListPage { + u64 writes{}; ///< GPU writes noted to the page + u64 copy_writes{}; ///< `writes` when the copy was made: the copy is current while equal + u64 copy_tick{}; ///< GPU tick that completes the copy + /// `writes` when the front last flushed the page: guest memory holds it while equal + u64 synced_writes{~u64{0}}; + u32 copy_begin{}; ///< Bytes of the page the copy holds (a buffer can end inside it) + u32 copy_end{}; + /// Written outside a compute dispatch or a DMA copy (a draw's storage writes) at least + /// once: the front can't tell from what it sent whether such a write is still queued, so + /// the page's lists always drain the split instead of reading a copy. + bool draw_written{}; + std::optional copy; + }; + std::mutex nx2_list_mutex; ///< nx2_list_pages: the split's front reads the copies + std::unordered_map nx2_list_pages; + /// Copies replaced while the GPU may still write them: freed once it is past their tick. + std::vector> nx2_retired_list_copies; + boost::container::small_vector, 8> nx2_list_copy_requests; + bool nx2_collect_list_copies{}; }; } // namespace VideoCommon diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 1144488c91..6b93bb7f60 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -9,6 +9,7 @@ #include #include +#include "common/scope_exit.h" #include "common/settings.h" #include "core/core.h" #include "core/memory.h" @@ -178,6 +179,11 @@ bool DmaPusher::Step() { RecordParsedEntry(header, commands); command_resume_offset += ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); + } else if (ReadGpuWrittenCommands(dma_state.dma_get, static_cast(header.size))) { + const std::span commands{command_headers.data(), header.size}; + RecordParsedEntry(header, commands); + command_resume_offset += + ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); } else { FlushGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); Tegra::Memory::GpuGuestMemoryheaders(memory_manager, dma_state.dma_get, header.size, &command_headers); @@ -382,6 +388,15 @@ void DmaPusher::CallMethod(u32 argument) { ReportUnboundSubchannel(); return; } + // Compute dispatches and DMA copies write the command lists ReadGpuWrittenCommands reads. + const auto engine_type = subchannel_type[dma_state.subchannel]; + const bool writer = engine_type == Engines::EngineTypes::KeplerCompute || + engine_type == Engines::EngineTypes::MaxwellDMA; + SCOPE_EXIT { + if (writer) { + split.NoteWriter(); + } + }; if (split.Active()) { const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset; if (split.FrontMacros()) { @@ -432,6 +447,15 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { ReportUnboundSubchannel(); return; } + // Compute dispatches and DMA copies write the command lists ReadGpuWrittenCommands reads. + const auto engine_type = subchannel_type[dma_state.subchannel]; + const bool writer = engine_type == Engines::EngineTypes::KeplerCompute || + engine_type == Engines::EngineTypes::MaxwellDMA; + SCOPE_EXIT { + if (writer) { + split.NoteWriter(); + } + }; if (split.Active()) { const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset; if (split.FrontMacros()) { @@ -476,18 +500,86 @@ bool DmaPusher::IsGpuWrittenCommands(GPUVAddr address, size_t size) { return false; } +bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) { + // The pages are marked as command list pages: from the next write on, the buffer cache copies + // them to host memory right after the dispatch or copy that writes them and submits. The + // front then waits only for the back to run that work and for the GPU to finish the copy, not + // for everything queued and a cache flush (FlushGpuWrittenCommands: ~400 GPU round trips a + // second in a Mario Kart World race, during which the back had nothing to do). Pages + // FlushGpuWrittenCommands flushed whole and no GPU work wrote since are read from guest + // memory. + if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || count == 0 || + rasterizer == nullptr) { + return false; + } + const GPUVAddr end = address + u64{count} * sizeof(u32); + bool written = false; + for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < end; page += 0x1000) { + if (const auto device = memory_manager.GpuToCpuAddress(page); + device && Tegra::Nx2IsGpuWritten(*device)) { + Tegra::Nx2MarkCommandListPage(*device); + written = true; + } + } + if (!written) { + return false; + } + if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { + split.WaitForWriters(); + } + using GpuWrittenList = VideoCore::RasterizerInterface::GpuWrittenList; + command_headers.resize_destructive(count); + u8* const out = reinterpret_cast(command_headers.data()); + for (GPUVAddr chunk = address; chunk < end;) { + const GPUVAddr chunk_end = (std::min)(end, (chunk & ~GPUVAddr{0xfff}) + 0x1000); + const std::span part{out + (chunk - address), static_cast(chunk_end - chunk)}; + const auto device = memory_manager.GpuToCpuAddress(chunk); + if (!device) { + return false; + } + GpuWrittenList where = GpuWrittenList::Synced; + if (Tegra::Nx2IsGpuWritten(*device)) { + where = rasterizer->ReadGpuWrittenCommandList(*device, part); + } + if (where == GpuWrittenList::Unknown) { + return false; + } + if (where == GpuWrittenList::Synced) { + memory_manager.ReadBlockUnsafe(chunk, part.data(), part.size()); + } + chunk = chunk_end; + } + return true; +} + void DmaPusher::FlushGpuWrittenCommands(GPUVAddr address, size_t size) { // Title 1 runs small command lists its culling dispatches fill in each frame. Accurate GPU // mode reads every command list through the caches (safe reads); here only the ones on pages // the GPU has written are, or their draws run with an older frame's words (trees vanish). + // These are the lists ReadGpuWrittenCommands can't take from a copy or guest memory (a page's + // first list, or pages draws write too). if (!IsGpuWrittenCommands(address, size)) { return; } // The caches belong to the split's back: it first runs what the front queued (the dispatch - // that writes these words among it). + // that writes these words among it). The whole pages are flushed: their next lists read + // guest memory until GPU work writes them again. VideoCommon::GpuSplit::Get().Drain(); - if (memory_manager.IsMemoryDirty(address, size, VideoCommon::CacheType::BufferCache)) { - memory_manager.FlushRegion(address, size, VideoCommon::CacheType::BufferCache); + const GPUVAddr first_page = address & ~GPUVAddr{0xfff}; + const GPUVAddr end_page = (address + size + 0xfff) & ~GPUVAddr{0xfff}; + if (memory_manager.IsMemoryDirty(first_page, end_page - first_page, + VideoCommon::CacheType::BufferCache)) { + memory_manager.FlushRegion(first_page, end_page - first_page, + VideoCommon::CacheType::BufferCache); + } + if (rasterizer == nullptr) { + return; + } + for (GPUVAddr page = first_page; page < end_page; page += 0x1000) { + if (const auto device = memory_manager.GpuToCpuAddress(page); + device && Tegra::Nx2IsCommandListPage(*device)) { + rasterizer->NoteGpuWrittenCommandListSynced(*device); + } } } diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index e9dd5e2b6b..289786bb9e 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -188,6 +188,11 @@ public: void RecordParsedEntry(const CommandListHeader& header, std::span commands); /// Whether the entry lies on device pages the GPU has written (NX2). bool IsGpuWrittenCommands(GPUVAddr address, size_t size); + /// Reads a command list on pages the GPU wrote into command_headers, from the copies the + /// buffer cache made after the work that wrote them or from guest memory for pages flushed + /// since. False when not all are current (or the list isn't GPU-written): the caller drains, + /// flushes and reads guest memory. + bool ReadGpuWrittenCommands(GPUVAddr address, u32 count); /// Brings command list words the GPU wrote (still only in host buffers) to guest memory. void FlushGpuWrittenCommands(GPUVAddr address, size_t size); diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp index ac0d3f25f1..ef0cdae123 100644 --- a/src/video_core/gpu_split.cpp +++ b/src/video_core/gpu_split.cpp @@ -138,6 +138,7 @@ void GpuSplit::Stop() { // never ran, it waited on its syncpoint forever), and the engines it held were freed. ring.reset(); write_position = 0; + last_writer_position = 0; published_local = 0; consumed_cache = 0; publish_every = PublishEvery; @@ -216,6 +217,28 @@ void GpuSplit::Drain() { } } +void GpuSplit::WaitForWriters() { + const u64 target = std::min(last_writer_position, write_position); + consumed_cache = consumed.load(std::memory_order_acquire); + if (consumed_cache >= target) { + return; + } + wait_target.store(target, std::memory_order_seq_cst); + Publish(); + for (u32 spins = 0;; ++spins) { + consumed_cache = consumed.load(std::memory_order_acquire); + if (consumed_cache >= target) { + wait_target.store(~u64{0}, std::memory_order_relaxed); + return; + } + if (spins < 16384) { + Pause(); + } else { + std::this_thread::yield(); + } + } +} + void GpuSplit::SwitchEngine(Tegra::Engines::EngineInterface* engine) { front_engine = engine; u64* const entry = Reserve(2); @@ -346,8 +369,11 @@ void GpuSplit::BackLoop(std::stop_token stop_token) { const size_t index = read_position & RingMask; const size_t size = Execute(&ring[index]); read_position += size != 0 ? size : RingWords - index; - // Hand ring space back now and then while working through a long batch. - if (read_position - last_store >= 4096 && read_position != available) { + // Hand ring space back now and then while working through a long batch, and at once + // when the front waits for this position (WaitForWriters). + if ((read_position - last_store >= 4096 || + read_position >= wait_target.load(std::memory_order_relaxed)) && + read_position != available) { consumed.store(read_position, std::memory_order_release); last_store = read_position; } diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h index d6f91e4141..c7c7118481 100644 --- a/src/video_core/gpu_split.h +++ b/src/video_core/gpu_split.h @@ -100,6 +100,13 @@ public: /// Waits until the back executed everything sent so far. void Drain(); + /// Compute or copy work was just sent: what can write GPU-written command lists. + void NoteWriter() { + last_writer_position = write_position; + } + /// Waits until the back executed the compute and copy work sent so far (not what came after). + void WaitForWriters(); + private: enum class Kind : u64 { Wrap, ///< The rest of the ring is unused: continue at its start @@ -156,6 +163,7 @@ private: // Front side. u64 write_position{}; + u64 last_writer_position{}; u64 published_local{}; u64 consumed_cache{}; /// Words between publishes: small right after a drain, while the back has nothing to do. @@ -169,6 +177,8 @@ private: alignas(64) std::atomic published{}; alignas(64) std::atomic consumed{}; + /// A position the front waits for (WaitForWriters): the back stores `consumed` once past it. + alignas(64) std::atomic wait_target{~u64{0}}; alignas(64) std::atomic back_waiting{}; std::atomic wake{}; std::jthread back_thread; diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index db8c92adfd..92714d52d2 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -43,11 +43,43 @@ inline void Nx2MarkGpuWritten(u64 device_address, u64 size) { } } +// GPU-written device pages the front has read command lists from (same layout as above). The +// buffer cache copies these pages into host memory right after the compute or copy work that +// writes them and submits, so the front reads the copy once the GPU is done instead of draining +// the split and flushing the caches (BufferCache::Nx2ReadListCopy). +inline std::array, (u64{1} << (36 - Nx2WrittenPageBits)) / 64> g_nx2_command_list_pages{}; +inline std::atomic g_nx2_has_command_list_pages{false}; + +inline void Nx2MarkCommandListPage(u64 device_address) { + const u64 page = device_address >> Nx2WrittenPageBits; + if (page >= g_nx2_command_list_pages.size() * 64) { + return; + } + auto& word = g_nx2_command_list_pages[page / 64]; + const u64 bit = u64{1} << (page % 64); + if ((word.load(std::memory_order_relaxed) & bit) == 0) { + word.fetch_or(bit, std::memory_order_relaxed); + g_nx2_has_command_list_pages.store(true, std::memory_order_relaxed); + } +} + +[[nodiscard]] inline bool Nx2IsCommandListPage(u64 device_address) { + const u64 page = device_address >> Nx2WrittenPageBits; + if (page >= g_nx2_command_list_pages.size() * 64) { + return false; + } + return (g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> (page % 64)) & 1; +} + /// A new emulation session starts: no page has been written by its GPU yet. inline void Nx2ClearGpuWrittenPages() { for (auto& word : g_nx2_gpu_written_pages) { word.store(0, std::memory_order_relaxed); } + for (auto& word : g_nx2_command_list_pages) { + word.store(0, std::memory_order_relaxed); + } + g_nx2_has_command_list_pages.store(false, std::memory_order_relaxed); } [[nodiscard]] inline bool Nx2IsGpuWritten(u64 device_address) { diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 8486d44ac9..e9eaa2b3c2 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -94,6 +94,20 @@ public: virtual void FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) = 0; + /// Switch 2, from the GPU thread's front: where a GPU-written command list's words are. + enum class GpuWrittenList { + Unknown, ///< Neither: the caller drains, flushes the page and notes it synced + Copied, ///< `out` (within one 4 KiB page) is filled from the copy made after its writer + Synced, ///< Guest memory holds the page: no GPU work wrote it since it was flushed + }; + virtual GpuWrittenList ReadGpuWrittenCommandList([[maybe_unused]] DAddr addr, + [[maybe_unused]] std::span out) { + return GpuWrittenList::Unknown; + } + + /// Switch 2: the front flushed this command list page with the split drained. + virtual void NoteGpuWrittenCommandListSynced([[maybe_unused]] DAddr page) {} + /// Check if the the specified memory area requires flushing to CPU Memory. virtual bool MustFlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) = 0; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index fe0ab3a28e..2449265509 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -453,6 +453,10 @@ void BufferCacheRuntime::Wait(u64 tick) { scheduler.Wait(tick); } +void BufferCacheRuntime::WaitSubmittedTick(u64 tick) { + scheduler.GetMasterSemaphore().Wait(tick); +} + void BufferCacheRuntime::Finish() { scheduler.Finish(); } diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index ff854445b5..d3295ec6ed 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -122,6 +122,9 @@ public: void Wait(u64 tick); + /// Waits for a submitted tick without flushing: safe on threads other than the GPU thread. + void WaitSubmittedTick(u64 tick); + void Finish(); u64 GetDeviceLocalMemory() const; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index d3dbba6163..978f8be547 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -922,6 +922,11 @@ void RasterizerVulkan::DispatchCompute() { return; } std::scoped_lock lock{texture_cache.mutex, buffer_cache.mutex}; + // Command list pages the dispatch writes are copied for the split's front right after it. + buffer_cache.Nx2CollectListCopies(true); + SCOPE_EXIT { + buffer_cache.Nx2CollectListCopies(false); + }; if (!pipeline->Configure(*kepler_compute, *gpu_memory, scheduler, buffer_cache, texture_cache)) { if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute resources unavailable"); @@ -944,6 +949,7 @@ void RasterizerVulkan::DispatchCompute() { } cmdbuf.DispatchIndirect(indirect_buffer, indirect_offset); }); + SubmitNx2ListCopies(); return; } const std::array dim{qmd.grid_dim_x, qmd.grid_dim_y, qmd.grid_dim_z}; @@ -966,6 +972,7 @@ void RasterizerVulkan::DispatchCompute() { } cmdbuf.Dispatch(dim[0], dim[1], dim[2]); }); + SubmitNx2ListCopies(); // Log compute dispatch if (GPU::Logging::IsActive() && @@ -1007,6 +1014,29 @@ void Vulkan::RasterizerVulkan::DisableGraphicsUniformBuffer(size_t stage, u32 in void RasterizerVulkan::FlushAll() {} +void RasterizerVulkan::SubmitNx2ListCopies() { + if (buffer_cache.Nx2CopyListPages(scheduler.CurrentTick())) { + scheduler.Flush(); + } +} + +VideoCore::RasterizerInterface::GpuWrittenList RasterizerVulkan::ReadGpuWrittenCommandList( + DAddr addr, std::span out) { + // No cache lock: the front only reads the copies, under their own lock. + switch (buffer_cache.Nx2ReadListCopy(addr, out)) { + case BufferCache::Nx2ListRead::Copied: + return GpuWrittenList::Copied; + case BufferCache::Nx2ListRead::Synced: + return GpuWrittenList::Synced; + default: + return GpuWrittenList::Unknown; + } +} + +void RasterizerVulkan::NoteGpuWrittenCommandListSynced(DAddr page) { + buffer_cache.Nx2NoteListSynced(page); +} + void RasterizerVulkan::FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which) { if (addr == 0 || size == 0) { return; @@ -1319,6 +1349,8 @@ void RasterizerVulkan::AccelerateInlineToMemory(GPUVAddr address, size_t copy_si bool uploaded; { std::scoped_lock lock{buffer_cache.mutex}; + // Guest memory has the words now: a command list copy of the page no longer does. + buffer_cache.Nx2NoteListHostWrite(*cpu_addr, copy_size); uploaded = buffer_cache.InlineMemory(*cpu_addr, copy_size, memory); if (!uploaded && !defer) { buffer_cache.WriteMemory(*cpu_addr, copy_size); @@ -1414,7 +1446,13 @@ bool AccelerateDMA::BufferClear(GPUVAddr src_address, u64 amount, u32 value) { bool AccelerateDMA::BufferCopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount) { std::scoped_lock lock{buffer_cache.mutex}; - return buffer_cache.DMACopy(src_address, dest_address, amount); + buffer_cache.Nx2CollectListCopies(true); + const bool copied = buffer_cache.DMACopy(src_address, dest_address, amount); + if (buffer_cache.Nx2CopyListPages(scheduler.CurrentTick())) { + scheduler.Flush(); + } + buffer_cache.Nx2CollectListCopies(false); + return copied; } template diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index fc282fbe8d..dbb6ebdca8 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -99,6 +99,8 @@ public: void FlushAll() override; void FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; + GpuWrittenList ReadGpuWrittenCommandList(DAddr addr, std::span out) override; + void NoteGpuWrittenCommandListSynced(DAddr page) override; bool MustFlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override; VideoCore::RasterizerDownloadArea GetFlushArea(DAddr addr, u64 size) override; @@ -165,6 +167,10 @@ private: void FlushWork(); + /// Records the copies of the command list pages the work just recorded wrote and submits, so + /// the GPU makes them now (BufferCache::Nx2CopyListPages). + void SubmitNx2ListCopies(); + void UpdateDynamicStates(); void HandleTransformFeedback(); -- cgit v1.2.3 From 202c6b0356e1fa9366e5b9000df35a63f1eb9dc0 Mon Sep 17 00:00:00 2001 From: gabrielskaminski Date: Sun, 1 Nov 2026 01:49:26 -0300 Subject: video_core/buffer_cache: reuse unchanged small uniform buffers within command buffer (cherry picked from commit 4aa5cd89fd1862751b3297036f2d21a0dc924b23) (cherry picked from commit f1bdd5b13a1bdca81243b4aa1d8529f0485d3671) --- src/video_core/buffer_cache/buffer_cache.h | 20 +++- src/video_core/buffer_cache/uniform_stream_reuse.h | 121 +++++++++++++++++++++ src/video_core/renderer_vulkan/vk_buffer_cache.h | 31 +++++- 3 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 src/video_core/buffer_cache/uniform_stream_reuse.h (limited to 'src/video_core/buffer_cache/buffer_cache.h') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index cd176dec27..481b2f0f32 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -15,6 +15,7 @@ #include "common/range_sets.inc" #include "video_core/buffer_cache/buffer_cache_base.h" +#include "video_core/buffer_cache/uniform_stream_reuse.h" #include "video_core/guest_memory.h" #include "video_core/host1x/gpu_device_memory_manager.h" #include "video_core/nx2.h" @@ -28,6 +29,9 @@ using Core::DEVICE_PAGESIZE; template BufferCache

::BufferCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, Runtime& runtime_) : runtime{runtime_}, device_memory{device_memory_}, memory_tracker{device_memory} { + // A new runtime has a new scheduler (its ticks restart) and a new stream buffer: whatever the + // stream uniform reuse remembers refers to the old ones. Before the early return below. + VideoCommon::ResetUniformStreamReuse(); // Ensure the first slot is used for the null buffer void(slot_buffers.insert(runtime, NullBufferParams{})); gpu_modified_ranges.Clear(); @@ -1078,8 +1082,20 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 channel_state->fast_bound_uniform_buffers[stage] |= 1u << binding_index; channel_state->uniform_buffer_binding_sizes[stage][binding_index] = size; // Stream buffer path to avoid stalling on non-Nvidia drivers or Vulkan - const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); - device_memory.ReadBlockUnsafe(device_addr, span.data(), size); + bool streamed = false; + if constexpr (!IS_OPENGL && VideoCommon::StreamUniformReusable) { + // The same bytes as this slot's previous bind, in the same command buffer: bind that + // stream allocation again instead of allocating and copying (uniform_stream_reuse.h). + // EDEN_NX2_UBOREUSE=0 turns it off. + if (VideoCommon::UniformStreamReuseEnabled()) { + streamed = VideoCommon::BindSmallUniformReusing( + runtime, device_memory, stage, index, binding_index, device_addr, size); + } + } + if (!streamed) { + const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + device_memory.ReadBlockUnsafe(device_addr, span.data(), size); + } return; } // Classic cached path diff --git a/src/video_core/buffer_cache/uniform_stream_reuse.h b/src/video_core/buffer_cache/uniform_stream_reuse.h new file mode 100644 index 0000000000..7ba31c4717 --- /dev/null +++ b/src/video_core/buffer_cache/uniform_stream_reuse.h @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "common/common_types.h" + +namespace VideoCommon { + +/// What a buffer cache runtime offers for stream uniform reuse. +/// - StreamUniformRef: a small trivially copyable description of one stream allocation (the +/// host buffer, its offset and whatever else the runtime needs to add it to a descriptor +/// payload again, e.g. the device address of the descriptor buffer path). +/// - LastStreamUniformRef(): the allocation made by the latest BindMappedUniformBuffer call. +/// - RebindStreamUniform(ref, size): adds `ref` to the descriptor payload exactly as +/// BindMappedUniformBuffer added the allocation the first time. +template +concept StreamUniformReusable = requires(R& runtime, u32 size) { + typename R::StreamUniformRef; + requires std::is_trivially_copyable_v; + { runtime.LastStreamUniformRef() } -> std::same_as; + runtime.RebindStreamUniform(runtime.LastStreamUniformRef(), size); + { runtime.CurrentTick() } -> std::convertible_to; +}; + +/// EDEN_NX2_UBOREUSE=0 turns the reuse off (every small uniform buffer is copied, as before). +[[nodiscard]] inline bool UniformStreamReuseEnabled() noexcept { + static const bool enabled = [] { + const char* const value = std::getenv("EDEN_NX2_UBOREUSE"); + return value == nullptr || value[0] != '0'; + }(); + return enabled; +} + +template +class UniformStreamReuse { +public: + static constexpr u32 MaxBytes = 4096; // BufferCache's DEFAULT_SKIP_CACHE_SIZE + static constexpr size_t NumStages = 5; + static constexpr size_t NumSlots = 18; // Maxwell constant buffers per stage + + struct Slot { + Ref ref{}; + u64 tick{}; + u32 size{}; // 0: nothing streamed yet + std::array bytes{}; + }; + + void Reset() noexcept { + for (auto& stage : slots) { + for (Slot& slot : stage) { + slot.size = 0; + slot.tick = 0; + } + } + } + + std::array, NumStages> slots{}; + std::array scratch{}; +}; + +template +[[nodiscard]] inline UniformStreamReuse& UniformStreamReuseState() noexcept { + static UniformStreamReuse state; + return state; +} + +/// Forgets every slot. Call it when a buffer cache (and so its runtime, scheduler and stream +/// buffer) is created: ticks restart with a new scheduler and the old references are dead. +template +inline void ResetUniformStreamReuse() noexcept { + if constexpr (StreamUniformReusable) { + UniformStreamReuseState().Reset(); + } +} + +/// Binds the small uniform buffer at `device_addr` for (stage, cbuf `index`) through the stream +/// buffer, reusing the slot's previous allocation when the bytes and the command buffer are the +/// same. Returns false (nothing bound) when the buffer does not qualify, so the caller binds it +/// the classic way. +template +[[nodiscard]] bool BindSmallUniformReusing(Runtime& runtime, DeviceMemory& device_memory, + size_t stage, u32 index, u32 binding_index, + u64 device_addr, u32 size) { + using Reuse = UniformStreamReuse; + if (size == 0 || size > Reuse::MaxBytes || stage >= Reuse::NumStages || + index >= Reuse::NumSlots) { + return false; + } + Reuse& reuse = UniformStreamReuseState(); + typename Reuse::Slot& slot = reuse.slots[stage][index]; + u8* const scratch = reuse.scratch.data(); + device_memory.ReadBlockUnsafe(device_addr, scratch, size); + + const bool same_bytes = + slot.size == size && std::memcmp(slot.bytes.data(), scratch, size) == 0; + if (same_bytes && slot.tick == runtime.CurrentTick()) { + runtime.RebindStreamUniform(slot.ref, size); + return true; + } + const std::span span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + std::memcpy(span.data(), scratch, size); + slot.ref = runtime.LastStreamUniformRef(); + // After the allocation: the tick the stream region is tagged with is the one current now. + slot.tick = runtime.CurrentTick(); + if (!same_bytes) { + slot.size = size; + std::memcpy(slot.bytes.data(), scratch, size); + } + return true; +} + +} // namespace VideoCommon diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index d3295ec6ed..0587f28c64 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -168,15 +168,40 @@ public: void BindTransformFeedbackBuffers(VideoCommon::HostBindings& bindings); + /// This and the two members below are the hooks of VideoCommon::StreamUniformReusable + /// (buffer_cache/uniform_stream_reuse.h). + struct StreamUniformRef { + VkBuffer buffer; + VkDeviceAddress device_address; // base of `buffer`, 0 without buffer device address + u32 offset; + }; + std::span BindMappedUniformBuffer([[maybe_unused]] size_t stage, [[maybe_unused]] u32 binding_index, u32 size) { const StagingBufferRef ref = staging_pool.Request(size, MemoryUsage::Upload); - guest_descriptor_queue.AddBuffer(ref.buffer, ref.device_address, - static_cast(ref.offset), size); + last_stream_uniform = StreamUniformRef{ + .buffer = ref.buffer, + .device_address = ref.device_address, + .offset = static_cast(ref.offset), + }; + RebindStreamUniform(last_stream_uniform, size); return ref.mapped_span; } + /// The allocation of the latest BindMappedUniformBuffer call. + [[nodiscard]] StreamUniformRef LastStreamUniformRef() const noexcept { + return last_stream_uniform; + } + + /// Adds `ref` to the descriptor payload exactly as BindMappedUniformBuffer did when it made + /// the allocation (it is the very same call). The caller guarantees the allocation is still + /// live: made on the scheduler tick that is current now, so the staging pool has not recycled + /// it and the GPU has not even started on it. + void RebindStreamUniform(const StreamUniformRef& ref, u32 size) { + guest_descriptor_queue.AddBuffer(ref.buffer, ref.device_address, ref.offset, size); + } + [[nodiscard]] VkDeviceSize SparseAlignmentFor(bool sparse_compatible) const noexcept { if (!sparse_compatible || !multi_range_buffers.use_sparse) { return 0; @@ -276,6 +301,8 @@ private: boost::container::small_vector multi_range_sources; VkDeviceSize multi_range_total{}; + StreamUniformRef last_stream_uniform{}; + bool limit_dynamic_storage_buffers = false; u32 max_dynamic_storage_buffers = (std::numeric_limits::max)(); }; -- cgit v1.2.3 From 13fc5fda26e6b5a9af67f5a654e831187bfa1962 Mon Sep 17 00:00:00 2001 From: Sssnowyyy <78863999+Sssnowyyy@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:38:11 -0500 Subject: vulkan: repack 69-byte vertices only when needed SetVertexRepackLayout returned old_mask | new_mask, which stays non-zero on every draw that uses a 69-byte binding. PrepareDraw then re-dirtied the vertex buffers and vertex input each time, so the repack compute pass ran and the render pass ended once per draw, which is costly on tiled GPUs. Return old_mask ^ new_mask so only a layout change re-dirties the bindings. The repacked copy only stays valid while the source data is unchanged, so the buffer cache now flags the vertex buffers overlapping a CPU write, inline or DMA write, or GPU write, and re-dirties the repacked ones before binding. Command buffer submission already invalidates all vertex buffer state, which keeps the staging copy within its tick. Also pass the source buffer size to the repack pass and clamp the word-rounded storage descriptor range to it; it could run up to three bytes past the end of the buffer. A binding that does not fit is skipped like one over the device limits. Follow-up to e38b738b84. Assisted-by: Claude Sonnet 5.5 (implementation and commit message) --- src/tests/video_core/vertex_repack.cpp | 26 ++++++++++----- src/video_core/buffer_cache/buffer_cache.h | 37 ++++++++++++++++++++++ src/video_core/buffer_cache/buffer_cache_base.h | 8 +++++ src/video_core/renderer_vulkan/vertex_repack.h | 14 +++++--- src/video_core/renderer_vulkan/vk_buffer_cache.cpp | 8 +++-- src/video_core/renderer_vulkan/vk_buffer_cache.h | 1 + src/video_core/renderer_vulkan/vk_compute_pass.cpp | 5 +-- src/video_core/renderer_vulkan/vk_compute_pass.h | 7 ++-- 8 files changed, 86 insertions(+), 20 deletions(-) (limited to 'src/video_core/buffer_cache/buffer_cache.h') diff --git a/src/tests/video_core/vertex_repack.cpp b/src/tests/video_core/vertex_repack.cpp index 6409be3ca4..3705a97926 100644 --- a/src/tests/video_core/vertex_repack.cpp +++ b/src/tests/video_core/vertex_repack.cpp @@ -85,7 +85,8 @@ TEST_CASE("Static vertex repacking requires explicit experimental opt-in", "[ver TEST_CASE("Vertex repack planning keeps descriptor offsets aligned", "[vertex_repack]") { for (const u32 alignment : {4U, 16U, 64U, 256U, 4096U}) { for (const u32 offset : {0U, 1U, 3U, 255U, 4099U}) { - const auto plan = Vulkan::MakeVertexRepackParameters(offset, 70, alignment, 1U << 20, 65535); + const auto plan = Vulkan::MakeVertexRepackParameters(offset, 70, alignment, 1U << 20, + 65535, 1U << 20); REQUIRE(plan.has_value()); REQUIRE(plan->source_base % alignment == 0); REQUIRE(plan->source_base + plan->source_bias == offset); @@ -99,13 +100,22 @@ TEST_CASE("Vertex repack planning keeps descriptor offsets aligned", "[vertex_re } TEST_CASE("Vertex repack planning rejects overflow and device-limit violations", "[vertex_repack]") { - REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 0, 256, 1U << 20, 65535)); - REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 69, 0, 1U << 20, 65535)); - REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 69, 256, 71, 65535)); - REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(255, 69, 256, 323, 65535)); - REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 69, 256, 1U << 20, 0)); - REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 0xffffffffU, 256, ~u64{0}, 0xffffffffU)); - const auto one = Vulkan::MakeVertexRepackParameters(0, 69, 256, 72, 1); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 0, 256, 1U << 20, 65535, 1U << 20)); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 69, 0, 1U << 20, 65535, 1U << 20)); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 69, 256, 71, 65535, 1U << 20)); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(255, 69, 256, 323, 65535, 1U << 20)); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 69, 256, 1U << 20, 0, 1U << 20)); + REQUIRE_FALSE( + Vulkan::MakeVertexRepackParameters(0, 0xffffffffU, 256, ~u64{0}, 0xffffffffU, ~u64{0})); + const auto one = Vulkan::MakeVertexRepackParameters(0, 69, 256, 72, 1, 72); REQUIRE(one.has_value()); REQUIRE(one->source_range == 72); } + +TEST_CASE("Vertex repack planning keeps the source range inside the buffer", "[vertex_repack]") { + const auto clamped = Vulkan::MakeVertexRepackParameters(0, 69, 256, 1U << 20, 65535, 69); + REQUIRE(clamped.has_value()); + REQUIRE(clamped->source_range == 69); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 69, 256, 1U << 20, 65535, 68)); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(300, 69, 256, 1U << 20, 65535, 256)); +} diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index cd176dec27..6878780866 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -180,6 +181,7 @@ void BufferCache

::WriteMemory(DAddr device_addr, u64 size) { gpu_modified_ranges.Subtract(device_addr, size); } memory_tracker.MarkRegionAsCpuModified(device_addr, size); + MarkVertexBuffersStale(device_addr, size); } template @@ -265,6 +267,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am } ClearDownload(*cpu_dest_address, amount); + MarkVertexBuffersStale(*cpu_dest_address, amount); BufferId buffer_a; BufferId buffer_b; @@ -892,8 +895,40 @@ void BufferCache

::UpdateVertexBufferSlot(u32 index, const Binding& binding) { } } +template +void BufferCache

::MarkVertexBuffersStale(DAddr device_addr, u64 size) { + if constexpr (requires { runtime.IsVertexRepacked(u32{}); }) { + u32 mask = enabled_vertex_buffers_mask; + while (mask != 0) { + const u32 index = std::countr_zero(mask); + mask &= mask - 1; + const Binding& binding = v_buffer[index]; + if (device_addr < binding.device_addr + binding.size && + binding.device_addr < device_addr + size) { + stale_vertex_buffers_mask |= 1U << index; + } + } + } +} + +template +void BufferCache

::RefreshRepackedVertexBuffers() { + if constexpr (requires { runtime.IsVertexRepacked(u32{}); }) { + u32 mask = std::exchange(stale_vertex_buffers_mask, 0) & enabled_vertex_buffers_mask; + auto& flags = maxwell3d->dirty.flags; + while (mask != 0) { + const u32 index = std::countr_zero(mask); + mask &= mask - 1; + if (runtime.IsVertexRepacked(index)) { + flags[Dirty::VertexBuffer0 + index] = true; + } + } + } +} + template void BufferCache

::BindHostVertexBuffers() { + RefreshRepackedVertexBuffers(); #ifdef __ANDROID__ const bool use_optimized_vertex_buffers = Settings::values.use_optimized_vertex_buffers.GetValue(); @@ -1655,6 +1690,7 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 } } memory_tracker.MarkRegionAsGpuModified(device_addr, size); + MarkVertexBuffersStale(device_addr, size); gpu_modified_ranges.Add(device_addr, size); if (DownloadsAtFence(device_addr, size)) { uncommitted_gpu_modified_ranges.Add(device_addr, size); @@ -2205,6 +2241,7 @@ void BufferCache

::InlineMemoryImplementation(DAddr dest_address, size_t copy_ std::span inlined_buffer) { ClearDownload(dest_address, copy_size); gpu_modified_ranges.Subtract(dest_address, copy_size); + MarkVertexBuffersStale(dest_address, copy_size); BufferId buffer_id = FindBuffer(dest_address, static_cast(copy_size), false); auto& buffer = slot_buffers[buffer_id]; diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 1f0c6cf10f..59be418030 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -525,6 +525,13 @@ private: void UpdateVertexBufferSlot(u32 index, const Binding& binding); + /// Flags the vertex buffers overlapping a written range so host copies derived from them are + /// rebuilt. + void MarkVertexBuffersStale(DAddr device_addr, u64 size); + + /// Re-dirties repacked vertex buffers whose source data was written since they were bound. + void RefreshRepackedVertexBuffers(); + void ClearDownload(DAddr base_addr, u64 size); void InlineMemoryImplementation(DAddr dest_address, size_t copy_size, @@ -545,6 +552,7 @@ private: u32 last_index_count = 0; u32 enabled_vertex_buffers_mask = 0; + u32 stale_vertex_buffers_mask = 0; u64 vertex_buffers_serial = 0; std::array v_buffer{}; diff --git a/src/video_core/renderer_vulkan/vertex_repack.h b/src/video_core/renderer_vulkan/vertex_repack.h index b0849c25c7..456d1c6f20 100644 --- a/src/video_core/renderer_vulkan/vertex_repack.h +++ b/src/video_core/renderer_vulkan/vertex_repack.h @@ -2,6 +2,7 @@ #pragma once +#include #include #include #include @@ -31,8 +32,11 @@ struct VertexRepackParameters { u32 dispatch_groups; }; -inline std::optional MakeVertexRepackParameters( - u32 offset, u32 size, u64 alignment, u64 max_storage_range, u32 max_dispatch_groups) { +inline std::optional MakeVertexRepackParameters(u32 offset, u32 size, + u64 alignment, + u64 max_storage_range, + u32 max_dispatch_groups, + u64 buffer_size) { if (size == 0 || alignment == 0) { return std::nullopt; } @@ -40,9 +44,11 @@ inline std::optional MakeVertexRepackParameters( const u64 output_size = vertices * 72; const u64 base = u64{offset} / alignment * alignment; const u64 bias = offset - base; - const u64 range = (bias + size + 3) / 4 * 4; + // The word-rounded range must not run past the end of the source buffer + const u64 available = buffer_size > base ? buffer_size - base : 0; + const u64 range = (std::min)((bias + size + 3) / 4 * 4, available); const u64 groups = (vertices * 18 + 127) / 128; - if (output_size > std::numeric_limits::max() || + if (range < bias + size || output_size > std::numeric_limits::max() || range > std::numeric_limits::max() || output_size > max_storage_range || range > max_storage_range || groups > max_dispatch_groups) { return std::nullopt; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 2449265509..4d5ccb6520 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -394,7 +394,7 @@ u32 BufferCacheRuntime::SetVertexRepackLayout(const Tegra::Engines::Maxwell3D::R LOG_DEBUG(Render_Vulkan, "Vertex repack selection: mask={:#x}, dynamic_input={}", vertex_repack_mask, dynamic_input); } - return old_mask | vertex_repack_mask; + return old_mask ^ vertex_repack_mask; } StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) { @@ -657,7 +657,8 @@ void BufferCacheRuntime::BindVertexBuffer(u32 index, VkBuffer buffer, u32 offset return; } if (buffer != VK_NULL_HANDLE && size != 0 && IsVertexRepacked(index)) { - const auto repacked = vertex_repack_pass->Assemble(buffer, offset, size); + const auto repacked = + vertex_repack_pass->Assemble(buffer, offset, size, u64{offset} + size); if (!repacked) { return; } @@ -702,7 +703,8 @@ void BufferCacheRuntime::BindVertexBuffers(VideoCommon::HostBindings& bi if (handle != VK_NULL_HANDLE && IsVertexRepacked(binding) && bindings.strides[i] == 69 && bindings.sizes[i] > 0) { const auto repacked = vertex_repack_pass->Assemble( - handle, static_cast(bindings.offsets[i]), static_cast(bindings.sizes[i])); + handle, static_cast(bindings.offsets[i]), static_cast(bindings.sizes[i]), + bindings.buffers[i]->SizeBytes()); if (!repacked) { continue; } diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index d3295ec6ed..a8d0274016 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -110,6 +110,7 @@ public: void TickFrame(Common::SlotVector& slot_buffers) noexcept; + /// Returns the mask of bindings whose repacked state changed. u32 SetVertexRepackLayout(const Tegra::Engines::Maxwell3D::Regs& regs, bool dynamic_input); bool IsVertexRepacked(u32 binding) const { diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index cb8dcde42f..5902b7d8f5 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -335,10 +335,11 @@ VertexRepackPass::VertexRepackPass(const Device& device_, Scheduler& scheduler_, scheduler{scheduler_}, staging_pool{staging_pool_}, descriptor_queue{descriptor_queue_} {} std::optional> VertexRepackPass::Assemble(VkBuffer source, - u32 offset, u32 size) { + u32 offset, u32 size, + u64 buffer_size) { const auto parameters = MakeVertexRepackParameters( offset, size, device.GetStorageBufferAlignment(), device.GetMaxStorageBufferRange(), - device.GetMaxComputeWorkGroupCount()[0]); + device.GetMaxComputeWorkGroupCount()[0], buffer_size); if (!parameters) { LOG_ERROR(Render_Vulkan, "Vertex repack exceeds device limits: offset={}, size={}", offset, size); return std::nullopt; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h index 27fffbbbb7..9687bfff91 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.h +++ b/src/video_core/renderer_vulkan/vk_compute_pass.h @@ -60,9 +60,10 @@ class VertexRepackPass final : public ComputePass { public: VertexRepackPass(const Device& device, Scheduler& scheduler, DescriptorPool& descriptor_pool, StagingBufferPool& staging_pool, ComputePassDescriptorQueue& descriptor_queue); - /// Returns nothing when the range exceeds the device limits; the binding is then left as is. - std::optional> Assemble(VkBuffer source, u32 offset, - u32 size); + /// Returns nothing when the range exceeds the device limits or the source buffer; the binding + /// is then left as is. + std::optional> Assemble(VkBuffer source, u32 offset, u32 size, + u64 buffer_size); private: Scheduler& scheduler; -- cgit v1.2.3 From 8556e27015ded665ca9a5522071cdeeec20a1575 Mon Sep 17 00:00:00 2001 From: Sssnowyyy <78863999+Sssnowyyy@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:39:27 -0500 Subject: buffer_cache: fix the already-disabled check for uniform buffers DisableGraphicsUniformBuffer tested !current.buffer_id to detect a binding that is already NULL_BINDING. SlotId converts to true unless it is the invalid id, and NULL_BUFFER_ID is BufferId{0}, so the check was false for NULL_BINDING and the early return never triggered. Compare against NULL_BUFFER_ID instead. Follow-up to 46b2577bf7. Assisted-by: Claude Sonnet 5.5 (implementation and commit message) --- src/video_core/buffer_cache/buffer_cache.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/video_core/buffer_cache/buffer_cache.h') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 6878780866..2d90a949c3 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -411,7 +411,8 @@ void BufferCache

::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr template void BufferCache

::DisableGraphicsUniformBuffer(size_t stage, u32 index) { auto& current = channel_state->uniform_buffers[stage][index]; - if (current.device_addr == 0 && current.size == 0 && !current.buffer_id) [[likely]] { + if (current.device_addr == 0 && current.size == 0 && current.buffer_id == NULL_BUFFER_ID) + [[likely]] { return; } current = NULL_BINDING; -- cgit v1.2.3 From 06e211e0d0a1634d16f1d74d75682092fc92a5de Mon Sep 17 00:00:00 2001 From: snowstormxd <214507868+snowstormxd@users.noreply.github.com> Date: Sun, 1 Nov 2026 14:21:03 +0000 Subject: nx2: rearm clean read areas when the GPU writes them again Port the coherency changes from alarok/Eglantine 77fc365f2 without its diagnostic framework or shutdown changes. Only GPU-dirty CPU reads become permanently preflushable, clean areas can be trapped on the next write, non-core readers avoid indexing core arrays, and collected CPU writes precede readback. Assisted-by: Codex (source review, selective integration and verification) (cherry picked from commit b64e211957180aa218ec6d9a3bcb020332279c3f) --- src/core/memory.cpp | 15 +++++++++++++++ src/video_core/buffer_cache/buffer_cache.h | 13 +++++++++++-- src/video_core/gpu.cpp | 4 +++- 3 files changed, 29 insertions(+), 3 deletions(-) (limited to 'src/video_core/buffer_cache/buffer_cache.h') diff --git a/src/core/memory.cpp b/src/core/memory.cpp index cfc39916db..1f7315f3c1 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -549,6 +549,8 @@ struct Memory::Impl { } void RasterizerArmReadTraps(u64 vaddr, u64 size) { + // Clean cached read areas also need invalidation when reactive flushing is enabled. + read_trap_generation.fetch_add(1, std::memory_order_release); if (vaddr == 0 || size == 0 || !AddressSpaceContains(*current_page_table, vaddr, size) || !ReadTrapsEnabled()) { return; @@ -781,6 +783,19 @@ struct Memory::Impl { gpu_device_memory = &system.Host1x().MemoryManager(); } const size_t core = system.GetCurrentHostThreadID(); + if (core >= Core::Hardware::NUM_CPU_CORES) { + // A service thread has no slot in the arrays below, and it does not use a slot of a + // core. It keeps no area, so it asks on each read. + if (static std::atomic_flag logged; !logged.test_and_set(std::memory_order_relaxed)) { + LOG_INFO(HW_Memory, "Read of cached guest memory from a thread that is not a core"); + } + Common::ScratchBuffer scratch; + gpu_device_memory->ApplyOpOnPointer(p, scratch, [&](DAddr address) { + DisarmReadTraps(v_address, size); + (void)system.GPU().OnCPURead(address, size); + }); + return; + } auto& current_area = rasterizer_read_areas[core]; if (const u64 generation = read_trap_generation.load(std::memory_order_acquire); read_area_generations[core] != generation) { diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 481b2f0f32..f6d394ce81 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -234,8 +234,17 @@ std::optional BufferCache

::GetFlushArea(DA }; area->preemtive = !IsRegionGpuModified(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned); - memory_tracker.MarkRegionAsPreflushable(device_addr_start_aligned, - device_addr_end_aligned - device_addr_start_aligned); + // Mark only data that the GPU wrote. A mark stays for the session and sends later GPU writes + // in the range to a download at each fence. + if (!area->preemtive) { + memory_tracker.MarkRegionAsPreflushable(device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned); + } else { + // The core keeps this area and has no mark. The next GPU write here must arm the range + // again, which ends the area and sets the trap. + read_trapped_ranges.Subtract(device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned); + } return area; } diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 53f8f063f7..681a4a232d 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -228,7 +228,9 @@ struct GPU::Impl { raster_area.start_address, raster_area.end_address, reads.load(std::memory_order_relaxed)); } - const u64 fence = RequestSyncOperation([this, &raster_area]() { + const u64 fence = RequestSyncOperation([this, raster_area]() { + // Apply collected guest writes before downloading GPU data over guest memory. + InvalidateGPUCache(); renderer->ReadRasterizer()->FlushRegion(raster_area.start_address, raster_area.end_address - raster_area.start_address); }); gpu_thread.TickGPU(is_async); -- cgit v1.2.3