From 202c6b0356e1fa9366e5b9000df35a63f1eb9dc0 Mon Sep 17 00:00:00 2001 From: gabrielskaminski Date: Sun, 1 Nov 2026 01:49:26 -0300 Subject: video_core/buffer_cache: reuse unchanged small uniform buffers within command buffer (cherry picked from commit 4aa5cd89fd1862751b3297036f2d21a0dc924b23) (cherry picked from commit f1bdd5b13a1bdca81243b4aa1d8529f0485d3671) --- src/video_core/renderer_vulkan/vk_buffer_cache.h | 31 ++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) (limited to 'src/video_core/renderer_vulkan') diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index d3295ec6ed..0587f28c64 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -168,15 +168,40 @@ public: void BindTransformFeedbackBuffers(VideoCommon::HostBindings& bindings); + /// This and the two members below are the hooks of VideoCommon::StreamUniformReusable + /// (buffer_cache/uniform_stream_reuse.h). + struct StreamUniformRef { + VkBuffer buffer; + VkDeviceAddress device_address; // base of `buffer`, 0 without buffer device address + u32 offset; + }; + std::span BindMappedUniformBuffer([[maybe_unused]] size_t stage, [[maybe_unused]] u32 binding_index, u32 size) { const StagingBufferRef ref = staging_pool.Request(size, MemoryUsage::Upload); - guest_descriptor_queue.AddBuffer(ref.buffer, ref.device_address, - static_cast(ref.offset), size); + last_stream_uniform = StreamUniformRef{ + .buffer = ref.buffer, + .device_address = ref.device_address, + .offset = static_cast(ref.offset), + }; + RebindStreamUniform(last_stream_uniform, size); return ref.mapped_span; } + /// The allocation of the latest BindMappedUniformBuffer call. + [[nodiscard]] StreamUniformRef LastStreamUniformRef() const noexcept { + return last_stream_uniform; + } + + /// Adds `ref` to the descriptor payload exactly as BindMappedUniformBuffer did when it made + /// the allocation (it is the very same call). The caller guarantees the allocation is still + /// live: made on the scheduler tick that is current now, so the staging pool has not recycled + /// it and the GPU has not even started on it. + void RebindStreamUniform(const StreamUniformRef& ref, u32 size) { + guest_descriptor_queue.AddBuffer(ref.buffer, ref.device_address, ref.offset, size); + } + [[nodiscard]] VkDeviceSize SparseAlignmentFor(bool sparse_compatible) const noexcept { if (!sparse_compatible || !multi_range_buffers.use_sparse) { return 0; @@ -276,6 +301,8 @@ private: boost::container::small_vector multi_range_sources; VkDeviceSize multi_range_total{}; + StreamUniformRef last_stream_uniform{}; + bool limit_dynamic_storage_buffers = false; u32 max_dynamic_storage_buffers = (std::numeric_limits::max)(); }; -- cgit v1.2.3 From b2430b4d90016eb6a221224687459c1876bc5552 Mon Sep 17 00:00:00 2001 From: snowstormxd <214507868+snowstormxd@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:32:04 +0000 Subject: shader_cache: batch memory-write invalidations without skipping ranges Adapt the batching from ymatrix1/Eglantine PR #237, retaining all CPU invalidations. Acquire the lock once and remove pending shaders once per write batch. Compare real cache contents with sequential invalidation across randomized overlapping writes and pending removals. Assisted-by: Codex (implementation, review, commit text and automated validation) (cherry picked from commit b65f818d7789ffadfb58fe4543b18060fd88ce53) --- src/tests/CMakeLists.txt | 1 + src/tests/video_core/shader_invalidation.cpp | 120 +++++++++++++++++++++++ src/video_core/renderer_vulkan/vk_rasterizer.cpp | 8 +- src/video_core/shader_cache.cpp | 11 +++ src/video_core/shader_cache.h | 4 + 5 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 src/tests/video_core/shader_invalidation.cpp (limited to 'src/video_core/renderer_vulkan') diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 49fc646dea..be5528ef46 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -23,6 +23,7 @@ add_executable(tests video_core/vertex_repack.cpp video_core/ampere_returns.cpp video_core/native_macro_fifo.cpp + video_core/shader_invalidation.cpp input_common/calibration_configuration_job.cpp ) diff --git a/src/tests/video_core/shader_invalidation.cpp b/src/tests/video_core/shader_invalidation.cpp new file mode 100644 index 0000000000..806fb5c58d --- /dev/null +++ b/src/tests/video_core/shader_invalidation.cpp @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include +#include +#include +#include +#include +#include "core/device_memory.h" +#include "video_core/shader_cache.h" + +namespace VideoCommon { +struct ShaderCacheTestAccess { + static std::unique_ptr Create(Tegra::MaxwellDeviceMemoryManager& memory) { + return std::unique_ptr{new ShaderCache{memory}}; + } + static void Register(ShaderCache& cache, VAddr address, size_t size, u64 hash) { + auto data = std::make_unique(); + data->unique_hash = hash; + cache.Register(std::move(data), address, size); + } + static const ShaderInfo* Find(const ShaderCache& cache, VAddr address) { + return cache.TryGet(address); + } +}; +} // namespace VideoCommon + +using VideoCommon::ShaderCacheTestAccess; + +TEST_CASE("Shader batch invalidation matches sequential cache contents", "[shader_invalidation]") { + Core::DeviceMemory memory; + Tegra::MaxwellDeviceMemoryManager device_memory{memory}; + auto batched = ShaderCacheTestAccess::Create(device_memory); + auto sequential = ShaderCacheTestAccess::Create(device_memory); + std::mt19937 random{0x16'67}; + constexpr VAddr base = 0x100000; + for (unsigned iteration = 0; iteration < 600; ++iteration) { + for (unsigned slot = 0; slot < 128; ++slot) { + const auto address = base + slot * 4096; + if (!ShaderCacheTestAccess::Find(*batched, address)) { + const auto size = 1 + random() % 65536; + ShaderCacheTestAccess::Register(*batched, address, size, iteration + slot); + ShaderCacheTestAccess::Register(*sequential, address, size, iteration + slot); + } + } + std::vector> writes; + for (unsigned i = 0, count = random() % 24; i < count; ++i) { + writes.emplace_back(base + random() % (128 * 4096), random() % 32768); + if (i % 4 == 0) { + writes.push_back(writes.back()); + } + } + for (const auto& [address, size] : writes) { + sequential->InvalidateRegion(address, size); + } + batched->InvalidateRegions(writes); + for (unsigned slot = 0; slot < 128; ++slot) { + const auto address = base + slot * 4096; + const auto actual = ShaderCacheTestAccess::Find(*batched, address); + const auto expected = ShaderCacheTestAccess::Find(*sequential, address); + REQUIRE(bool(actual) == bool(expected)); + if (actual) { + REQUIRE(actual->unique_hash == expected->unique_hash); + } + } + } +} + +TEST_CASE("Shader empty batches preserve pending invalidations", "[shader_invalidation]") { + Core::DeviceMemory memory; + Tegra::MaxwellDeviceMemoryManager device_memory{memory}; + auto cache = ShaderCacheTestAccess::Create(device_memory); + constexpr VAddr address = 0x100000; + ShaderCacheTestAccess::Register(*cache, address, 32768, 123); + cache->OnCacheInvalidation(address, 1); + cache->InvalidateRegions({}); + REQUIRE(ShaderCacheTestAccess::Find(*cache, address)); + const std::array unrelated{std::pair{address + 0x100000, 4096}}; + cache->InvalidateRegions(unrelated); + REQUIRE_FALSE(ShaderCacheTestAccess::Find(*cache, address)); +} + +TEST_CASE("Shader invalidation CPU benchmark", "[.][cache_bench]") { + Core::DeviceMemory memory; + Tegra::MaxwellDeviceMemoryManager device_memory{memory}; + auto batched = ShaderCacheTestAccess::Create(device_memory); + auto sequential = ShaderCacheTestAccess::Create(device_memory); + std::vector> writes; + constexpr VAddr base = 0x100000; + for (unsigned slot = 0; slot < 768; ++slot) { + writes.emplace_back(base + slot * 65536, 1); + } + double sequential_ms{}, batched_ms{}; + for (unsigned repeat = 0; repeat < 20; ++repeat) { + for (unsigned slot = 0; slot < 1024; ++slot) { + const auto address = base + slot * 65536; + if (!ShaderCacheTestAccess::Find(*batched, address)) { + ShaderCacheTestAccess::Register(*batched, address, 64, slot); + ShaderCacheTestAccess::Register(*sequential, address, 64, slot); + } + } + auto start = std::chrono::steady_clock::now(); + for (const auto& [address, size] : writes) { + sequential->InvalidateRegion(address, size); + } + sequential_ms += + std::chrono::duration(std::chrono::steady_clock::now() - start) + .count(); + start = std::chrono::steady_clock::now(); + batched->InvalidateRegions(writes); + batched_ms += + std::chrono::duration(std::chrono::steady_clock::now() - start) + .count(); + REQUIRE_FALSE(ShaderCacheTestAccess::Find(*batched, base)); + REQUIRE(ShaderCacheTestAccess::Find(*batched, base + 1023 * 65536)); + } + std::cout << "Shader invalidation: sequential=" << sequential_ms << " ms batched=" << batched_ms + << " ms (CPU only, not gameplay FPS)\n"; +} diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 978f8be547..dc34260de5 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1127,12 +1127,10 @@ void RasterizerVulkan::InnerInvalidation(std::span> sequences) { + if (sequences.empty()) { + return; + } + std::scoped_lock lock{invalidation_mutex}; + for (const auto& [addr, size] : sequences) { + InvalidatePagesInRegion(addr, size); + } + RemovePendingShaders(); +} + void ShaderCache::OnCacheInvalidation(VAddr addr, size_t size) { std::scoped_lock lock{invalidation_mutex}; InvalidatePagesInRegion(addr, size); diff --git a/src/video_core/shader_cache.h b/src/video_core/shader_cache.h index 281a90cb6f..7c370e217c 100644 --- a/src/video_core/shader_cache.h +++ b/src/video_core/shader_cache.h @@ -67,6 +67,8 @@ public: /// @param size Number of bytes of the invalidation void InvalidateRegion(VAddr addr, size_t size); + void InvalidateRegions(std::span> sequences); + /// @brief Unmarks a memory region as cached and marks it for removal /// @param addr Start address of the CPU write operation /// @param size Number of bytes of the CPU write operation @@ -104,6 +106,8 @@ protected: bool last_shaders_valid = false; private: + friend struct ShaderCacheTestAccess; + /// @brief Tries to obtain a cached shader starting in a given address /// @note Doesn't check for ranges, the given address has to be the start of the shader /// @param addr Start address of the shader, this doesn't cache for region -- cgit v1.2.3 From 226162edf525f13bb0c1fa5c4788e510c474989d Mon Sep 17 00:00:00 2001 From: snowstormxd <214507868+snowstormxd@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:07:53 +0000 Subject: vulkan: submit native work sooner and use the active draw count Adapted from sylvieistaken/selevet-eglantine performance builds, including 8b076b167ffc5ca1ba8ffedcd00e255accbe018c and d8c95fef. Preserve upstream command-list coherency and submission ordering. Assisted by Codex. --- src/video_core/engines/draw_manager.cpp | 3 +++ src/video_core/engines/maxwell_3d.h | 1 + .../renderer_vulkan/vk_pipeline_cache.cpp | 2 +- src/video_core/renderer_vulkan/vk_rasterizer.cpp | 21 +++++++++++++++++++-- 4 files changed, 24 insertions(+), 3 deletions(-) (limited to 'src/video_core/renderer_vulkan') diff --git a/src/video_core/engines/draw_manager.cpp b/src/video_core/engines/draw_manager.cpp index 038e5a2e69..9419dd5758 100644 --- a/src/video_core/engines/draw_manager.cpp +++ b/src/video_core/engines/draw_manager.cpp @@ -301,6 +301,8 @@ void Maxwell3D::DrawManager::UpdateTopology(Maxwell3D& maxwell3d) { } void Maxwell3D::DrawManager::ProcessDraw(Maxwell3D& maxwell3d, bool draw_indexed, u32 instance_count) { + draw_state.draw_element_count = + draw_indexed ? draw_state.index_buffer.count : draw_state.vertex_buffer.count; LOG_TRACE(HW_GPU, "called, topology={}, count={}", draw_state.topology, draw_indexed ? draw_state.index_buffer.count : draw_state.vertex_buffer.count); UpdateTopology(maxwell3d); if (maxwell3d.ShouldExecute()) { @@ -309,6 +311,7 @@ void Maxwell3D::DrawManager::ProcessDraw(Maxwell3D& maxwell3d, bool draw_indexed } void Maxwell3D::DrawManager::ProcessDrawIndirect(Maxwell3D& maxwell3d) { + draw_state.draw_element_count = 0; LOG_TRACE(HW_GPU, "called, topology={}, is_indexed={}, includes_count={}, buffer_size={}, max_draw_count={}", draw_state.topology, indirect_state.is_indexed, indirect_state.include_count, indirect_state.buffer_size, indirect_state.max_draw_counts); UpdateTopology(maxwell3d); if (maxwell3d.ShouldExecute()) { diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 7b38e176b6..553760e843 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3064,6 +3064,7 @@ public: Maxwell3D::Regs::PrimitiveTopology topology{}; DrawMode draw_mode{}; bool draw_indexed{}; + u32 draw_element_count{}; // Zero for indirect draws whose count is not known yet. u32 base_index{}; Maxwell3D::Regs::VertexBuffer vertex_buffer; Maxwell3D::Regs::IndexBuffer index_buffer; diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index ff8e84b019..22e54ad75c 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -1056,7 +1056,7 @@ GraphicsPipeline* PipelineCache::BuiltPipeline(GraphicsPipeline* pipeline) const // Usually these shaders are only used once for building textures so we can assume they // can't be built async const auto& draw_state = maxwell3d->draw_manager.draw_state; - if (draw_state.index_buffer.count <= 6 || draw_state.vertex_buffer.count <= 6) { + if (draw_state.draw_element_count <= 6) { return pipeline; } return nullptr; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index dc34260de5..3898c571f5 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1422,10 +1422,27 @@ void RasterizerVulkan::FlushWork() { #endif // __ANDROID__ static_assert(DRAWS_TO_DISPATCH % (CHECK_MASK + 1) == 0); - if ((++draw_counter & CHECK_MASK) != CHECK_MASK) { + if ((++draw_counter & CHECK_MASK) != 0) { return; } - if (draw_counter < DRAWS_TO_DISPATCH) { + // Keep the host GPU working while native commands are still being recorded. + static const u32 native_flush_draws = [] { + constexpr u32 fallback = 128; + const char* value = std::getenv("EDEN_NX2_FLUSH_DRAWS"); + if (value == nullptr || value[0] < '0' || value[0] > '9') { + return fallback; + } + char* end = nullptr; + const unsigned long parsed = std::strtoul(value, &end, 10); + if (*end != '\0' || parsed == 0) { + return fallback; + } + return static_cast(std::clamp(parsed, 32UL, 1'048'576UL)); + }(); + const u32 draws_to_dispatch = Tegra::g_nx2_ampere.load(std::memory_order_relaxed) + ? native_flush_draws + : DRAWS_TO_DISPATCH; + if (draw_counter < draws_to_dispatch) { scheduler.DispatchWork(); return; } -- cgit v1.2.3