diff options
Diffstat (limited to 'src/video_core/buffer_cache')
| -rw-r--r-- | src/video_core/buffer_cache/buffer_cache.h | 330 | ||||
| -rw-r--r-- | src/video_core/buffer_cache/buffer_cache_base.h | 64 | ||||
| -rw-r--r-- | src/video_core/buffer_cache/uniform_stream_reuse.h | 121 |
3 files changed, 504 insertions, 11 deletions
diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e90af3214d..40369b16ef 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -10,13 +10,16 @@ #include <memory> #include <numeric> #include <stdexcept> +#include <utility> #include <vector> #include <fmt/ranges.h> #include "common/range_sets.inc" #include "video_core/buffer_cache/buffer_cache_base.h" +#include "video_core/buffer_cache/uniform_stream_reuse.h" #include "video_core/guest_memory.h" #include "video_core/host1x/gpu_device_memory_manager.h" +#include "video_core/nx2.h" #include "video_core/nx2_profile.h" #include "video_core/texture_cache/util.h" @@ -27,6 +30,9 @@ using Core::DEVICE_PAGESIZE; template <class P> BufferCache<P>::BufferCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, Runtime& runtime_) : runtime{runtime_}, device_memory{device_memory_}, memory_tracker{device_memory} { + // A new runtime has a new scheduler (its ticks restart) and a new stream buffer: whatever the + // stream uniform reuse remembers refers to the old ones. Before the early return below. + VideoCommon::ResetUniformStreamReuse<Runtime>(); // Ensure the first slot is used for the null buffer void(slot_buffers.insert(runtime, NullBufferParams{})); gpu_modified_ranges.Clear(); @@ -100,6 +106,12 @@ void BufferCache<P>::TickFrame() { return; } runtime.TickFrame(slot_buffers); + if (frame_tick % 600 == 0 && !uploaded_buffer_ranges.empty()) { + for (const auto& [address, size] : uploaded_buffer_ranges) { + device_memory.HintCold(address, size); + } + uploaded_buffer_ranges.clear(); + } if (static u64 frames{}; ++frames % 600 == 0) { u64 total{}; size_t count{}; @@ -141,9 +153,10 @@ void BufferCache<P>::TickFrame() { // title 1's buffers were ~200 MiB of an 8 GB GPU's ~6.8 GB in use, yet this collector // ran aggressively every frame on the device-wide figure, evicting up to 64 buffers (and // downloading the GPU-written ones, ~5% of the GPU thread) that were recreated later. - // Collect when the buffers themselves are large. - if (buffer_bytes >= 1_GiB) { - RunGarbageCollector(buffer_bytes >= 2_GiB); + // Collect when the buffers themselves are large. At 1 GiB it held geometry of scenes + // long gone: title 1's title screen grew to ~0.9 GB of buffers in ten minutes. + if (buffer_bytes >= 512_MiB) { + RunGarbageCollector(buffer_bytes >= 1_GiB); } } else if (total_used_memory >= minimum_memory) { RunGarbageCollector(total_used_memory >= critical_memory); @@ -166,11 +179,13 @@ void BufferCache<P>::UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size template <class P> void BufferCache<P>::WriteMemory(DAddr device_addr, u64 size) { + Nx2NoteListHostWrite(device_addr, size); if (IsRegionGpuModified(device_addr, size)) { ClearDownload(device_addr, size); gpu_modified_ranges.Subtract(device_addr, size); } memory_tracker.MarkRegionAsCpuModified(device_addr, size); + MarkVertexBuffersStale(device_addr, size); } template <class P> @@ -186,6 +201,7 @@ void BufferCache<P>::CachedWriteMemory(DAddr device_addr, u64 size) { return; } + Nx2NoteListHostWrite(device_addr, size); tmp_buffer.resize_destructive(size); device_memory.ReadBlockUnsafe(device_addr, tmp_buffer.data(), size); @@ -220,8 +236,17 @@ std::optional<VideoCore::RasterizerDownloadArea> BufferCache<P>::GetFlushArea(DA }; area->preemtive = !IsRegionGpuModified(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned); - memory_tracker.MarkRegionAsPreflushable(device_addr_start_aligned, - device_addr_end_aligned - device_addr_start_aligned); + // Mark only data that the GPU wrote. A mark stays for the session and sends later GPU writes + // in the range to a download at each fence. + if (!area->preemtive) { + memory_tracker.MarkRegionAsPreflushable(device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned); + } else { + // The core keeps this area and has no mark. The next GPU write here must arm the range + // again, which ends the area and sets the trap. + read_trapped_ranges.Subtract(device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned); + } return area; } @@ -255,6 +280,7 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am } ClearDownload(*cpu_dest_address, amount); + MarkVertexBuffersStale(*cpu_dest_address, amount); BufferId buffer_a; BufferId buffer_b; @@ -291,6 +317,10 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am const bool has_new_downloads = tmp_intervals.size() != 0; for (const auto& pair : tmp_intervals) { gpu_modified_ranges.Add(pair.first, pair.second); + if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { + Tegra::Nx2MarkGpuWritten(pair.first, pair.second); + Nx2NoteListWrite(buffer_b, pair.first, pair.second); + } } const auto& copy = copies[0]; src_buffer.MarkUsage(copy.src_offset, copy.size); @@ -379,6 +409,10 @@ template <class P> void BufferCache<P>::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr, u32 size) { const std::optional<DAddr> device_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + if (!device_addr) { + channel_state->uniform_buffers[stage][index] = NULL_BINDING; + return; + } const Binding binding{ .device_addr = *device_addr, .size = size, @@ -389,7 +423,12 @@ void BufferCache<P>::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr template <class P> void BufferCache<P>::DisableGraphicsUniformBuffer(size_t stage, u32 index) { - channel_state->uniform_buffers[stage][index] = NULL_BINDING; + auto& current = channel_state->uniform_buffers[stage][index]; + if (current.device_addr == 0 && current.size == 0 && current.buffer_id == NULL_BUFFER_ID) + [[likely]] { + return; + } + current = NULL_BINDING; } template <class P> @@ -871,7 +910,39 @@ void BufferCache<P>::UpdateVertexBufferSlot(u32 index, const Binding& binding) { } template <class P> +void BufferCache<P>::MarkVertexBuffersStale(DAddr device_addr, u64 size) { + if constexpr (requires { runtime.IsVertexRepacked(u32{}); }) { + u32 mask = enabled_vertex_buffers_mask; + while (mask != 0) { + const u32 index = std::countr_zero(mask); + mask &= mask - 1; + const Binding& binding = v_buffer[index]; + if (device_addr < binding.device_addr + binding.size && + binding.device_addr < device_addr + size) { + stale_vertex_buffers_mask |= 1U << index; + } + } + } +} + +template <class P> +void BufferCache<P>::RefreshRepackedVertexBuffers() { + if constexpr (requires { runtime.IsVertexRepacked(u32{}); }) { + u32 mask = std::exchange(stale_vertex_buffers_mask, 0) & enabled_vertex_buffers_mask; + auto& flags = maxwell3d->dirty.flags; + while (mask != 0) { + const u32 index = std::countr_zero(mask); + mask &= mask - 1; + if (runtime.IsVertexRepacked(index)) { + flags[Dirty::VertexBuffer0 + index] = true; + } + } + } +} + +template <class P> void BufferCache<P>::BindHostVertexBuffers() { + RefreshRepackedVertexBuffers(); #ifdef __ANDROID__ const bool use_optimized_vertex_buffers = Settings::values.use_optimized_vertex_buffers.GetValue(); @@ -994,6 +1065,26 @@ void BufferCache<P>::BindHostGraphicsUniformBuffers(size_t stage) { template <class P> void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind) { ++channel_state->uniform_cache_shots[0]; + // Bank 24 (root table 0, NVN2's push constants) is read from bank 17, which such games do not + // bind: stream the table's current words as that buffer. + if (index == 17 && maxwell3d->root_tables.loaded && channel_state->uniform_buffers[stage][index].size == 0) { + const auto& words = maxwell3d->root_tables.tables[0]; + const u32 size = (*channel_state->uniform_buffer_sizes)[stage][index]; + const u32 copied = (std::min)(size, static_cast<u32>(sizeof(words))); + if constexpr (IS_OPENGL) { + std::array<u8, sizeof(words)> data{}; + std::memcpy(data.data(), words.data(), copied); + runtime.BindFastUniformBuffer(stage, binding_index, copied); + runtime.PushFastUniformBuffer(stage, binding_index, std::span<const u8>(data.data(), copied)); + } else { + const std::span<u8> span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + std::memset(span.data(), 0, span.size()); + std::memcpy(span.data(), words.data(), copied); + } + channel_state->fast_bound_uniform_buffers[stage] |= 1u << binding_index; + channel_state->uniform_buffer_binding_sizes[stage][binding_index] = size; + return; + } const Binding& binding = channel_state->uniform_buffers[stage][index]; const DAddr device_addr = binding.device_addr; const u32 size = (std::min)(binding.size, (*channel_state->uniform_buffer_sizes)[stage][index]); @@ -1036,8 +1127,20 @@ void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 channel_state->fast_bound_uniform_buffers[stage] |= 1u << binding_index; channel_state->uniform_buffer_binding_sizes[stage][binding_index] = size; // Stream buffer path to avoid stalling on non-Nvidia drivers or Vulkan - const std::span<u8> span = runtime.BindMappedUniformBuffer(stage, binding_index, size); - device_memory.ReadBlockUnsafe(device_addr, span.data(), size); + bool streamed = false; + if constexpr (!IS_OPENGL && VideoCommon::StreamUniformReusable<Runtime>) { + // The same bytes as this slot's previous bind, in the same command buffer: bind that + // stream allocation again instead of allocating and copying (uniform_stream_reuse.h). + // EDEN_NX2_UBOREUSE=0 turns it off. + if (VideoCommon::UniformStreamReuseEnabled()) { + streamed = VideoCommon::BindSmallUniformReusing( + runtime, device_memory, stage, index, binding_index, device_addr, size); + } + } + if (!streamed) { + const std::span<u8> span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + device_memory.ReadBlockUnsafe(device_addr, span.data(), size); + } return; } // Classic cached path @@ -1494,6 +1597,9 @@ void BufferCache<P>::UpdateDrawIndirect() { template <class P> void BufferCache<P>::UpdateUniformBuffers(size_t stage) { + if (channel_state->enabled_uniform_buffer_masks[stage] == 0) [[likely]] { + return; + } ForEachEnabledBit(channel_state->enabled_uniform_buffer_masks[stage], [&](u32 index) { Binding& binding = channel_state->uniform_buffers[stage][index]; if (binding.buffer_id) { @@ -1511,6 +1617,9 @@ void BufferCache<P>::UpdateUniformBuffers(size_t stage) { template <class P> void BufferCache<P>::UpdateStorageBuffers(size_t stage) { + if (channel_state->enabled_storage_buffers[stage] == 0) [[likely]] { + return; + } ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { // Resolve buffer Binding& binding = channel_state->storage_buffers[stage][index]; @@ -1523,6 +1632,9 @@ void BufferCache<P>::UpdateStorageBuffers(size_t stage) { template <class P> void BufferCache<P>::UpdateTextureBuffers(size_t stage) { + if (channel_state->enabled_texture_buffers[stage] == 0) [[likely]] { + return; + } ForEachEnabledBit(channel_state->enabled_texture_buffers[stage], [&](u32 index) { Binding& binding = channel_state->texture_buffers[stage][index]; binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); @@ -1604,20 +1716,212 @@ void BufferCache<P>::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 } } memory_tracker.MarkRegionAsGpuModified(device_addr, size); + MarkVertexBuffersStale(device_addr, size); gpu_modified_ranges.Add(device_addr, size); if (DownloadsAtFence(device_addr, size)) { uncommitted_gpu_modified_ranges.Add(device_addr, size); } else { ArmReadTraps(device_addr, size); } + if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { + Tegra::Nx2MarkGpuWritten(device_addr, size); + Nx2NoteListWrite(buffer_id, device_addr, size); + } +} + +template <class P> +void BufferCache<P>::Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 size) { + if constexpr (!IS_OPENGL) { + if (size == 0 || !Tegra::g_nx2_has_command_list_pages.load(std::memory_order_relaxed)) { + return; + } + constexpr u64 page_bits = Tegra::Nx2WrittenPageBits; + const u64 last = (std::min<u64>)((device_addr + size - 1) >> page_bits, + Tegra::g_nx2_command_list_pages.size() * 64 - 1); + for (u64 page = device_addr >> page_bits; page <= last;) { + // Storage buffers can be large: skip 64 pages at a time where none is a list page. + const u64 word = + Tegra::g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> + (page % 64); + if (word == 0) { + page = (page / 64 + 1) * 64; + continue; + } + if (word & 1) { + const DAddr page_addr = page << page_bits; + { + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page_addr]; + ++entry.writes; + if (!nx2_collect_list_copies && !entry.draw_written) { + entry.draw_written = true; + LOG_INFO(HW_GPU, "Command list page {:#x} is written by draws too: its " + "lists drain the split", + page_addr); + } + } + if (nx2_collect_list_copies) { + nx2_list_copy_requests.emplace_back(buffer_id, page_addr); + } + } + ++page; + } + } +} + +template <class P> +bool BufferCache<P>::Nx2CopyListPages(u64 tick) { + if constexpr (IS_OPENGL) { + return false; + } else { + if (nx2_list_copy_requests.empty()) { + return false; + } + std::ranges::sort(nx2_list_copy_requests, {}, &std::pair<BufferId, DAddr>::second); + constexpr DAddr page_size = DAddr{1} << Tegra::Nx2WrittenPageBits; + bool recorded = false; + for (size_t i = 0; i < nx2_list_copy_requests.size(); ++i) { + const auto [buffer_id, page] = nx2_list_copy_requests[i]; + if (i + 1 < nx2_list_copy_requests.size() && + nx2_list_copy_requests[i + 1].second == page) { + continue; + } + Buffer& buffer = slot_buffers[buffer_id]; + const DAddr begin = (std::max<DAddr>)(page, buffer.CpuAddr()); + const DAddr end = + (std::min<DAddr>)(page + page_size, buffer.CpuAddr() + buffer.SizeBytes()); + if (begin >= end) { + continue; + } + Nx2Staging staging = runtime.DownloadStagingBuffer(page_size, true); + const BufferCopy copy{ + .src_offset = buffer.Offset(begin), + .dst_offset = staging.offset + (begin - page), + .size = end - begin, + }; + buffer.MarkUsage(copy.src_offset, copy.size); + runtime.CopyBuffer(staging.buffer, buffer, std::span{©, 1}, true); + recorded = true; + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page]; + if (entry.copy) { + nx2_retired_list_copies.emplace_back(entry.copy_tick, std::move(*entry.copy)); + } + entry.copy = staging; + entry.copy_writes = entry.writes; + entry.copy_tick = tick; + entry.copy_begin = static_cast<u32>(begin - page); + entry.copy_end = static_cast<u32>(end - page); + } + nx2_list_copy_requests.clear(); + std::erase_if(nx2_retired_list_copies, [this](auto& retired) { + if (!runtime.IsFree(retired.first)) { + return false; + } + runtime.FreeDeferredStagingBuffer(retired.second); + return true; + }); + return recorded; + } +} + +template <class P> +auto BufferCache<P>::Nx2ReadListCopy(DAddr device_addr, std::span<u8> out) -> Nx2ListRead { + if constexpr (IS_OPENGL) { + return Nx2ListRead::Unknown; + } else { + constexpr DAddr page_size = DAddr{1} << Tegra::Nx2WrittenPageBits; + const DAddr page = device_addr & ~(page_size - 1); + const u64 offset = device_addr - page; + if (offset + out.size() > page_size) { + return Nx2ListRead::Unknown; + } + const auto current = [&](const Nx2ListPage& entry) { + return entry.copy && !entry.draw_written && entry.copy_writes == entry.writes && + offset >= entry.copy_begin && offset + out.size() <= entry.copy_end; + }; + u64 tick{}; + u64 writes{}; + { + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page); + if (it == nx2_list_pages.end()) { + return Nx2ListRead::Unknown; + } + const Nx2ListPage& entry = it->second; + // Pages the CPU writes the lists of (GPU-written once, long before) stay synced. + if (!entry.draw_written && entry.synced_writes == entry.writes) { + return Nx2ListRead::Synced; + } + if (!current(entry)) { + return Nx2ListRead::Unknown; + } + // The lists after the first of a batch find the copy done: read it at once. + if (runtime.IsFree(entry.copy_tick)) { + std::memcpy(out.data(), entry.copy->mapped_span.data() + offset, out.size()); + return Nx2ListRead::Copied; + } + tick = entry.copy_tick; + writes = entry.copy_writes; + } + // Submitted with the work that wrote the page (Nx2CopyListPages's caller flushes). + runtime.WaitSubmittedTick(tick); + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page); + if (it == nx2_list_pages.end() || !current(it->second) || it->second.copy_tick != tick || + it->second.copy_writes != writes) { + return Nx2ListRead::Unknown; + } + std::memcpy(out.data(), it->second.copy->mapped_span.data() + offset, out.size()); + return Nx2ListRead::Copied; + } +} + +template <class P> +void BufferCache<P>::Nx2NoteListSynced(DAddr page) { + if constexpr (!IS_OPENGL) { + std::scoped_lock lock{nx2_list_mutex}; + Nx2ListPage& entry = nx2_list_pages[page]; + entry.synced_writes = entry.writes; + } +} + +template <class P> +void BufferCache<P>::Nx2NoteListHostWrite(DAddr device_addr, u64 size) { + if constexpr (!IS_OPENGL) { + if (size == 0 || !Tegra::g_nx2_has_command_list_pages.load(std::memory_order_relaxed)) { + return; + } + constexpr u64 page_bits = Tegra::Nx2WrittenPageBits; + const u64 last = (std::min<u64>)((device_addr + size - 1) >> page_bits, + Tegra::g_nx2_command_list_pages.size() * 64 - 1); + for (u64 page = device_addr >> page_bits; page <= last; ++page) { + if (!((Tegra::g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> + (page % 64)) & + 1)) { + continue; + } + std::scoped_lock lock{nx2_list_mutex}; + const auto it = nx2_list_pages.find(page << page_bits); + if (it == nx2_list_pages.end()) { + continue; + } + Nx2ListPage& entry = it->second; + const bool synced = entry.synced_writes == entry.writes; + ++entry.writes; + if (synced) { + entry.synced_writes = entry.writes; + } + } + } } template <class P> BufferId BufferCache<P>::FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible) { - if (device_addr == 0) { + const u64 page = device_addr >> CACHING_PAGEBITS; + if (device_addr == 0 || page >= page_table.size()) { return NULL_BUFFER_ID; } - const u64 page = device_addr >> CACHING_PAGEBITS; const BufferId buffer_id = page_table[page]; if (buffer_id) { Buffer& buffer = slot_buffers[buffer_id]; @@ -1927,6 +2231,11 @@ void BufferCache<P>::MappedUploadMemory([[maybe_unused]] Buffer& buffer, u8* const src_pointer = staging_pointer.data() + copy.src_offset; const DAddr device_addr = buffer.CpuAddr() + copy.dst_offset; device_memory.ReadBlockUnsafe(device_addr, src_pointer, copy.size); + // Large uploads are mostly geometry the host GPU now has its own copy of: TickFrame + // marks their guest memory cold (see TextureCache::RefreshContents). + if (copy.size >= 256_KiB) { + uploaded_buffer_ranges.emplace_back(device_addr, copy.size); + } // Apply the staging offset copy.src_offset += upload_staging.offset; } @@ -1958,6 +2267,7 @@ void BufferCache<P>::InlineMemoryImplementation(DAddr dest_address, size_t copy_ std::span<const u8> inlined_buffer) { ClearDownload(dest_address, copy_size); gpu_modified_ranges.Subtract(dest_address, copy_size); + MarkVertexBuffersStale(dest_address, copy_size); BufferId buffer_id = FindBuffer(dest_address, static_cast<u32>(copy_size), false); auto& buffer = slot_buffers[buffer_id]; diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index d8a61b7913..86c05c8f22 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -14,7 +14,9 @@ #include <memory> #include <mutex> #include <numeric> +#include <optional> #include <span> +#include <unordered_map> #include <vector> #include "common/container/unordered_map.h" @@ -57,7 +59,8 @@ constexpr u32 NUM_VERTEX_BUFFERS = 32; constexpr u32 NUM_TRANSFORM_FEEDBACK_BUFFERS = 4; constexpr u32 NUM_GRAPHICS_UNIFORM_BUFFERS = 18; constexpr u32 NUM_COMPUTE_UNIFORM_BUFFERS = 8; -constexpr u32 NUM_STORAGE_BUFFERS = 16; +// Native compute shaders bind up to 18 global memory ranges; the enable masks are 32 bits wide. +constexpr u32 NUM_STORAGE_BUFFERS = 32; constexpr u32 NUM_TEXTURE_BUFFERS = 32; constexpr u32 NUM_STAGES = 5; @@ -321,6 +324,28 @@ public: bool DMAClear(GPUVAddr src_address, u64 amount, u32 value); + /// Switch 2 GPU-written command lists (nx2.h, g_nx2_command_list_pages). While collecting (a + /// compute dispatch or a DMA copy being recorded), writes to command list pages are noted; + /// Nx2CopyListPages then records copies of those pages into host memory after the work. + void Nx2CollectListCopies(bool collect) { + nx2_collect_list_copies = collect; + if (!collect) { + nx2_list_copy_requests.clear(); + } + } + /// True when it recorded copies, which the GPU completes at `tick` (the caller submits). + bool Nx2CopyListPages(u64 tick); + enum class Nx2ListRead { Unknown, Copied, Synced }; + /// Front thread: Synced when guest memory holds the page `device_addr` lies in (flushed and + /// written by no GPU work since), else Copied with `out` filled from the copy of its last + /// write once the GPU is done with it. Unknown otherwise: the caller drains and flushes. + Nx2ListRead Nx2ReadListCopy(DAddr device_addr, std::span<u8> out); + /// Front thread, the split drained: the command list page was flushed to guest memory. + void Nx2NoteListSynced(DAddr page); + /// CPU and inline writes: they reach guest memory, so a synced page stays synced, but its copy + /// no longer holds the page. + void Nx2NoteListHostWrite(DAddr device_addr, u64 size); + /// Return true when a CPU region is modified from the GPU [[nodiscard]] bool IsRegionGpuModified(DAddr addr, size_t size); @@ -440,6 +465,10 @@ private: void MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size, bool needs_sync = false); + /// Counts a GPU write to the command list pages in the range (invalidating their copies) and, + /// while collecting, asks for new copies. + void Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 size); + [[nodiscard]] BufferId FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible); void SynchronizeBufferWrites(Buffer& buffer); @@ -497,6 +526,13 @@ private: void UpdateVertexBufferSlot(u32 index, const Binding& binding); + /// Flags the vertex buffers overlapping a written range so host copies derived from them are + /// rebuilt. + void MarkVertexBuffersStale(DAddr device_addr, u64 size); + + /// Re-dirties repacked vertex buffers whose source data was written since they were bound. + void RefreshRepackedVertexBuffers(); + void ClearDownload(DAddr base_addr, u64 size); void InlineMemoryImplementation(DAddr dest_address, size_t copy_size, @@ -517,6 +553,7 @@ private: u32 last_index_count = 0; u32 enabled_vertex_buffers_mask = 0; + u32 stale_vertex_buffers_mask = 0; u64 vertex_buffers_serial = 0; std::array<Binding, 32> v_buffer{}; @@ -580,6 +617,8 @@ private: size_t immediate_buffer_capacity = 0; Common::ScratchBuffer<u8> immediate_buffer_alloc; + /// Guest memory large uploads came from since the last cold hint (device address, size). + std::vector<std::pair<DAddr, u64>> uploaded_buffer_ranges; struct LRUItemParams { using ObjectType = BufferId; @@ -602,6 +641,29 @@ private: // Covers the device address space (MaxwellDeviceTraits::device_virtual_bits). std::array<BufferId, ((1ULL << 36) >> CACHING_PAGEBITS)> page_table; Common::ScratchBuffer<u8> tmp_buffer; + + /// A command list page (Nx2CopyListPages): the GPU writes noted to it and the last copy. + using Nx2Staging = decltype(std::declval<Runtime&>().DownloadStagingBuffer(size_t{}, true)); + struct Nx2ListPage { + u64 writes{}; ///< GPU writes noted to the page + u64 copy_writes{}; ///< `writes` when the copy was made: the copy is current while equal + u64 copy_tick{}; ///< GPU tick that completes the copy + /// `writes` when the front last flushed the page: guest memory holds it while equal + u64 synced_writes{~u64{0}}; + u32 copy_begin{}; ///< Bytes of the page the copy holds (a buffer can end inside it) + u32 copy_end{}; + /// Written outside a compute dispatch or a DMA copy (a draw's storage writes) at least + /// once: the front can't tell from what it sent whether such a write is still queued, so + /// the page's lists always drain the split instead of reading a copy. + bool draw_written{}; + std::optional<Nx2Staging> copy; + }; + std::mutex nx2_list_mutex; ///< nx2_list_pages: the split's front reads the copies + std::unordered_map<DAddr, Nx2ListPage> nx2_list_pages; + /// Copies replaced while the GPU may still write them: freed once it is past their tick. + std::vector<std::pair<u64, Nx2Staging>> nx2_retired_list_copies; + boost::container::small_vector<std::pair<BufferId, DAddr>, 8> nx2_list_copy_requests; + bool nx2_collect_list_copies{}; }; } // namespace VideoCommon diff --git a/src/video_core/buffer_cache/uniform_stream_reuse.h b/src/video_core/buffer_cache/uniform_stream_reuse.h new file mode 100644 index 0000000000..7ba31c4717 --- /dev/null +++ b/src/video_core/buffer_cache/uniform_stream_reuse.h @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <array> +#include <concepts> +#include <cstddef> +#include <cstdlib> +#include <cstring> +#include <span> +#include <type_traits> + +#include "common/common_types.h" + +namespace VideoCommon { + +/// What a buffer cache runtime offers for stream uniform reuse. +/// - StreamUniformRef: a small trivially copyable description of one stream allocation (the +/// host buffer, its offset and whatever else the runtime needs to add it to a descriptor +/// payload again, e.g. the device address of the descriptor buffer path). +/// - LastStreamUniformRef(): the allocation made by the latest BindMappedUniformBuffer call. +/// - RebindStreamUniform(ref, size): adds `ref` to the descriptor payload exactly as +/// BindMappedUniformBuffer added the allocation the first time. +template <class R> +concept StreamUniformReusable = requires(R& runtime, u32 size) { + typename R::StreamUniformRef; + requires std::is_trivially_copyable_v<typename R::StreamUniformRef>; + { runtime.LastStreamUniformRef() } -> std::same_as<typename R::StreamUniformRef>; + runtime.RebindStreamUniform(runtime.LastStreamUniformRef(), size); + { runtime.CurrentTick() } -> std::convertible_to<u64>; +}; + +/// EDEN_NX2_UBOREUSE=0 turns the reuse off (every small uniform buffer is copied, as before). +[[nodiscard]] inline bool UniformStreamReuseEnabled() noexcept { + static const bool enabled = [] { + const char* const value = std::getenv("EDEN_NX2_UBOREUSE"); + return value == nullptr || value[0] != '0'; + }(); + return enabled; +} + +template <class Ref> +class UniformStreamReuse { +public: + static constexpr u32 MaxBytes = 4096; // BufferCache's DEFAULT_SKIP_CACHE_SIZE + static constexpr size_t NumStages = 5; + static constexpr size_t NumSlots = 18; // Maxwell constant buffers per stage + + struct Slot { + Ref ref{}; + u64 tick{}; + u32 size{}; // 0: nothing streamed yet + std::array<u8, MaxBytes> bytes{}; + }; + + void Reset() noexcept { + for (auto& stage : slots) { + for (Slot& slot : stage) { + slot.size = 0; + slot.tick = 0; + } + } + } + + std::array<std::array<Slot, NumSlots>, NumStages> slots{}; + std::array<u8, MaxBytes> scratch{}; +}; + +template <class Ref> +[[nodiscard]] inline UniformStreamReuse<Ref>& UniformStreamReuseState() noexcept { + static UniformStreamReuse<Ref> state; + return state; +} + +/// Forgets every slot. Call it when a buffer cache (and so its runtime, scheduler and stream +/// buffer) is created: ticks restart with a new scheduler and the old references are dead. +template <class Runtime> +inline void ResetUniformStreamReuse() noexcept { + if constexpr (StreamUniformReusable<Runtime>) { + UniformStreamReuseState<typename Runtime::StreamUniformRef>().Reset(); + } +} + +/// Binds the small uniform buffer at `device_addr` for (stage, cbuf `index`) through the stream +/// buffer, reusing the slot's previous allocation when the bytes and the command buffer are the +/// same. Returns false (nothing bound) when the buffer does not qualify, so the caller binds it +/// the classic way. +template <class Runtime, class DeviceMemory> +[[nodiscard]] bool BindSmallUniformReusing(Runtime& runtime, DeviceMemory& device_memory, + size_t stage, u32 index, u32 binding_index, + u64 device_addr, u32 size) { + using Reuse = UniformStreamReuse<typename Runtime::StreamUniformRef>; + if (size == 0 || size > Reuse::MaxBytes || stage >= Reuse::NumStages || + index >= Reuse::NumSlots) { + return false; + } + Reuse& reuse = UniformStreamReuseState<typename Runtime::StreamUniformRef>(); + typename Reuse::Slot& slot = reuse.slots[stage][index]; + u8* const scratch = reuse.scratch.data(); + device_memory.ReadBlockUnsafe(device_addr, scratch, size); + + const bool same_bytes = + slot.size == size && std::memcmp(slot.bytes.data(), scratch, size) == 0; + if (same_bytes && slot.tick == runtime.CurrentTick()) { + runtime.RebindStreamUniform(slot.ref, size); + return true; + } + const std::span<u8> span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + std::memcpy(span.data(), scratch, size); + slot.ref = runtime.LastStreamUniformRef(); + // After the allocation: the tick the stream region is tagged with is the one current now. + slot.tick = runtime.CurrentTick(); + if (!same_bytes) { + slot.size = size; + std::memcpy(slot.bytes.data(), scratch, size); + } + return true; +} + +} // namespace VideoCommon |