From 4d8a9629a6f49f57adfa420a7196e5967677f10f Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:30:46 -0400 Subject: texture_cache: collect from 5 GiB of device memory on every GPU The collector's thresholds follow the card's size and budget (critical at budget - 768 MiB), so the cache keeps textures of earlier scenes until the card is nearly full: Mario Kart World used 6.7 GB of an 8 GB card, and more on larger cards. Cap the critical threshold at 5 GiB (and expected/minimum below it) whatever the card. Same build, back to back: peak device memory 6.7 -> 5.6 GB, frames over 100 ms 114 -> 64 in six minutes, average frame time unchanged. The EDEN_NX2_TC_*_MIB overrides still apply. --- src/video_core/texture_cache/texture_cache.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 63abffed0f..bd5f7debfb 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -185,6 +185,14 @@ void TextureCache

::RunGarbageCollector() { template void TextureCache

::ApplyThresholdOverrides(bool log) { + // On every GPU: past what a frame needs, more device memory only keeps the textures of earlier + // scenes (title 1 filled 6.7 GB of an 8 GB card and draws its races in ~5 GB), so collect from + // MaxCriticalMemory on rather than near the card's budget. + critical_memory = (std::min)(critical_memory, MaxCriticalMemory); + expected_memory = + (std::min)(expected_memory, critical_memory - (std::min)(critical_memory, u64{768_MiB})); + minimum_memory = + (std::min)(minimum_memory, expected_memory - (std::min)(expected_memory, u64{1_GiB})); // EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB override the collector's thresholds. for (const auto& [name, target] : {std::pair{"EDEN_NX2_TC_EXPECTED_MIB", &expected_memory}, std::pair{"EDEN_NX2_TC_CRITICAL_MIB", &critical_memory}}) { -- cgit v1.2.3 From 049c9633edd9b3f7eaed6b4b3a18a4ead0cd18b7 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:31:41 -0400 Subject: nx2: read texture handles that hold a header byte offset Switch Sports' scenery shaders take their 64-bit texture handles from a uniform register, and the texture part of those handles is the header's byte offset in the pool (index * 32), not its index: 0x96e46000 names header 8960 (a 640x152 2D texture), while the plain reading, 286720, runs far past the 33067-entry pool. Eden skipped them at draw time (null view) and built the shaders for a garbage header, so the island's ground, cliffs and buildings drew black. When a handle's index falls past the pool but index / 32 lands inside it, take that header, both when building the shader and when binding the view. Handles that already resolve keep their meaning. --- src/video_core/shader_environment.cpp | 5 +++-- src/video_core/texture_cache/texture_cache.h | 1 + src/video_core/textures/texture.h | 11 +++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp index 9d189ad4f0..532b1f97f4 100644 --- a/src/video_core/shader_environment.cpp +++ b/src/video_core/shader_environment.cpp @@ -315,8 +315,9 @@ std::optional GenericEnvironment::TryFindSize() { Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw) { const auto handle{Tegra::Texture::TexturePair(raw, via_header_index)}; - ASSERT(handle.first <= tic_limit); - const GPUVAddr descriptor_addr{tic_addr + handle.first * sizeof(Tegra::Texture::TICEntry)}; + const u32 index{Tegra::Texture::TextureHeaderIndex(handle.first, tic_limit)}; + ASSERT(index <= tic_limit); + const GPUVAddr descriptor_addr{tic_addr + index * sizeof(Tegra::Texture::TICEntry)}; Tegra::Texture::TICEntry entry; gpu_memory->ReadBlock(descriptor_addr, &entry, sizeof(entry)); return entry; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 63abffed0f..b0fa8ac1ce 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -673,6 +673,7 @@ typename P::Framebuffer* TextureCache

::GetFramebuffer() { template ImageViewId TextureCache

::VisitImageView(u32 index, bool compute) { auto& table = compute ? channel_state->compute_image_table : channel_state->graphics_image_table; + index = Tegra::Texture::TextureHeaderIndex(index, table.current_limit); if (index > table.current_limit) { LOG_DEBUG(HW_GPU, "Invalid image view index={}", index); return NULL_IMAGE_VIEW_ID; diff --git a/src/video_core/textures/texture.h b/src/video_core/textures/texture.h index 9fc6a631bd..79396d8a68 100644 --- a/src/video_core/textures/texture.h +++ b/src/video_core/textures/texture.h @@ -165,6 +165,17 @@ static_assert(sizeof(TextureHandle) == 4, "TextureHandle has wrong size"); } } +/// Some Switch 2 bindless handles (Switch Sports' scenery, read from a uniform register) hold +/// the texture header's byte offset in the pool, index * 32, where the handle normally has the +/// index. Read it as an offset only when the plain index falls past the pool and the offset +/// lands inside it, so handles that already resolve are left alone. +[[nodiscard]] constexpr u32 TextureHeaderIndex(u32 index, u32 limit) { + if (index > limit && index % 32 == 0 && index / 32 <= limit) { + return index / 32; + } + return index; +} + struct TICEntry { union { struct { -- cgit v1.2.3 From 75385e994dc8cc8699a918180ee566e8723b8394 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:05:55 -0400 Subject: texture_cache: don't count deleted images still waiting to be destroyed The collector deletes images into a ring that destroys them TICKS_TO_DESTROY frames later, and until then the driver still counts their memory. Reading the driver's usage every frame, the collector saw no drop after deleting and kept deleting 40+ images a frame in aggressive mode long after it had freed enough: once a session reached the critical threshold, frames took over 100 ms for seconds at a time. Keep the bytes deleted into each slot of the ring and leave them out of the reported usage until the ring destroys them. --- src/video_core/texture_cache/texture_cache.h | 11 +++++++++++ src/video_core/texture_cache/texture_cache_base.h | 4 ++++ 2 files changed, 15 insertions(+) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index c8aaf8dff5..71f89db367 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -250,6 +250,12 @@ void TextureCache

::TickFrame() { // If we can obtain the memory info, use it instead of the estimate. if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); + // Images the collector deleted keep their memory for TICKS_TO_DESTROY frames: counting + // it kept the collector in aggressive mode, deleting 40+ images a frame for frames after + // it had freed enough (title 1: frames over 100 ms for seconds at a time). + for (const u64 bytes : sentenced_image_bytes) { + total_used_memory -= (std::min)(total_used_memory, bytes); + } } // Every 600 frames: device memory in use against the collector's thresholds, and what the // images themselves account for (VRAM pressure makes the collector thrash). @@ -316,6 +322,8 @@ void TextureCache

::TickFrame() { RunGarbageCollector(); } sentenced_images.Tick(); + sentenced_image_index = (sentenced_image_index + 1) % TICKS_TO_DESTROY; + sentenced_image_bytes[sentenced_image_index] = 0; sentenced_framebuffers.Tick(); sentenced_image_view.Tick(); TickAsyncDecode(); @@ -2832,6 +2840,8 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { } // TickFrame sets the total from the driver, which can be under the sum of these estimates. total_used_memory -= (std::min)(total_used_memory, ImageMemoryBytes(slot_images[image_id])); + const u64 freed_bytes = (image.HasScaled() ? GetScaledImageSizeBytes(image) : 0) + + ImageMemoryBytes(slot_images[image_id]); const GPUVAddr gpu_addr = image.gpu_addr; const auto alloc_it = image_allocs_table.find(gpu_addr); if (alloc_it == image_allocs_table.end()) { @@ -2897,6 +2907,7 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value()); } else if (!immediate_delete) { sentenced_images.Push(std::move(slot_images[image_id])); + sentenced_image_bytes[sentenced_image_index] += freed_bytes; } slot_images.erase(image_id); diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 94102cd261..d51eefeee8 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -552,6 +552,10 @@ private: static constexpr size_t TICKS_TO_DESTROY = 8; #endif DelayedDestructionRing sentenced_images; + /// Device memory of the images in each slot of sentenced_images (same index): deleted, but + /// counted in the driver's usage until the ring destroys them. + std::array sentenced_image_bytes{}; + size_t sentenced_image_index = 0; DelayedDestructionRing sentenced_image_view; DelayedDestructionRing sentenced_framebuffers; -- cgit v1.2.3 From dbd9572a5a03b1e0bb00946750cf866420a25596 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:04:22 -0400 Subject: texture_cache: delete images without walking every channel's tables Deleting an image stamped every entry of every channel's view-id map with CORRUPT_ID (validation, which is on in release builds) and cleared every channel's descriptor table bitmaps on the spot. On Mario Kart World's title screen (~5,400 entries in 5 channels) that took 362 us per deleted image, with dozens deleted a frame while the collector runs: ~10% of the time in its frames over 100 ms. The stamp is debug-only now. The tables are invalidated along with it, so every entry is written again before it is read. Invalidate() marks a table and its bitmap is cleared on the next read, once however many images were deleted in between. Same title screen: 3.7 us per deleted image. --- src/video_core/texture_cache/descriptor_table.h | 10 +++++++++- src/video_core/texture_cache/texture_cache.h | 10 ++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/descriptor_table.h b/src/video_core/texture_cache/descriptor_table.h index e40c128ab5..75383809c5 100644 --- a/src/video_core/texture_cache/descriptor_table.h +++ b/src/video_core/texture_cache/descriptor_table.h @@ -30,11 +30,17 @@ public: } void Invalidate() noexcept { - std::ranges::fill(read_descriptors, 0); + // Cleared on the next read: the texture cache invalidates every channel's tables for + // each image it deletes, dozens of times a frame while its collector runs. + invalidated = true; } [[nodiscard]] std::pair Read(Tegra::MemoryManager const& gpu_memory, u32 index) noexcept { DEBUG_ASSERT(index <= current_limit); + if (invalidated) [[unlikely]] { + std::ranges::fill(read_descriptors, 0); + invalidated = false; + } const GPUVAddr gpu_addr = current_gpu_addr + index * sizeof(T); std::pair result; gpu_memory.ReadBlockUnsafe(gpu_addr, std::addressof(result.first), sizeof(T)); @@ -60,6 +66,7 @@ public: read_descriptors.resize(Common::DivCeil(num_descriptors, 64U)); old_size = (std::min)(old_size, read_descriptors.size()); std::fill(read_descriptors.begin(), read_descriptors.begin() + old_size, 0); + invalidated = false; // descriptors.resize(num_descriptors); } @@ -68,6 +75,7 @@ public: std::vector descriptors; GPUVAddr current_gpu_addr{}; u32 current_limit{}; + bool invalidated{}; ///< read_descriptors is to be cleared before the next read }; } // namespace VideoCommon diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index c8aaf8dff5..559423dbee 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -1712,9 +1712,14 @@ void TextureCache

::InvalidateScale(Image& image) { for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; +#ifndef NDEBUG + // A walk over every entry (title 1: ~5,400 in 5 channels) for each deleted image: debug + // builds only. The tables are invalidated below, so each entry is written again before + // it is read. if constexpr (ENABLE_VALIDATION) for (auto& e : channel_info.image_view_ids) e.second = CORRUPT_ID; +#endif channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } @@ -2906,9 +2911,14 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { } for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; +#ifndef NDEBUG + // A walk over every entry (title 1: ~5,400 in 5 channels) for each deleted image: debug + // builds only. The tables are invalidated below, so each entry is written again before + // it is read. if constexpr (ENABLE_VALIDATION) for (auto& e : channel_info.image_view_ids) e.second = CORRUPT_ID; +#endif channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } -- cgit v1.2.3 From 446c34aa8b305deb9f0870721d2126da8c186368 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:24:25 -0400 Subject: texture_cache: free deleted images once the GPU is past them Deleted images kept their device memory for TICKS_TO_DESTROY (8) frames. While the collector deletes dozens a frame that held 1-1.5 GB of VRAM on top of its threshold, which the collector does not count (Mario Kart World on an 8 GB card: 6.6-7.0 GB in use against 5 GiB). On runtimes that report GPU progress (Vulkan) an image is destroyed as soon as the GPU is past it. Title screen, 10 minutes: VRAM in use 6.75 -> 6.34 GB on average, frames over 100 ms 22 -> 15. (cherry picked from commit 8c899541baf398deb892a857a932590f32d2aa4f) --- src/video_core/renderer_vulkan/vk_texture_cache.cpp | 8 ++++++++ src/video_core/renderer_vulkan/vk_texture_cache.h | 5 +++++ src/video_core/texture_cache/texture_cache.h | 18 ++++++++++++++++-- src/video_core/texture_cache/texture_cache_base.h | 11 +++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 5ba36feee9..b8d2e4edd4 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -1979,6 +1979,14 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 TextureCacheRuntime::CurrentTick() const noexcept { + return scheduler.CurrentTick(); +} + +bool TextureCacheRuntime::IsFree(u64 tick) const noexcept { + return scheduler.IsFree(tick); +} + u64 TextureCacheRuntime::GetDeviceMemoryBudget() const { return device.GetDeviceMemoryBudget(); } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 3944e183fd..49eba2d7d3 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -67,6 +67,11 @@ public: u64 GetDeviceMemoryUsage() const; + /// GPU progress: the texture cache frees a deleted image once the GPU is past its last use. + [[nodiscard]] u64 CurrentTick() const noexcept; + + [[nodiscard]] bool IsFree(u64 tick) const noexcept; + u64 GetDeviceMemoryBudget() const; bool CanReportMemoryUsage() const; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 71f89db367..46ebf6a25f 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -247,6 +247,13 @@ void TextureCache

::TickFrame() { } } } + if constexpr (requires { runtime.CurrentTick(); }) { + while (!gpu_sentenced_images.empty() && + runtime.IsFree(gpu_sentenced_images.front().tick)) { + gpu_sentenced_bytes -= gpu_sentenced_images.front().bytes; + gpu_sentenced_images.pop_front(); + } + } // If we can obtain the memory info, use it instead of the estimate. if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); @@ -256,6 +263,7 @@ void TextureCache

::TickFrame() { for (const u64 bytes : sentenced_image_bytes) { total_used_memory -= (std::min)(total_used_memory, bytes); } + total_used_memory -= (std::min)(total_used_memory, gpu_sentenced_bytes); } // Every 600 frames: device memory in use against the collector's thresholds, and what the // images themselves account for (VRAM pressure makes the collector thrash). @@ -2906,8 +2914,14 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { if (retained) { LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value()); } else if (!immediate_delete) { - sentenced_images.Push(std::move(slot_images[image_id])); - sentenced_image_bytes[sentenced_image_index] += freed_bytes; + if constexpr (requires { runtime.CurrentTick(); }) { + gpu_sentenced_images.push_back( + {runtime.CurrentTick(), freed_bytes, std::move(slot_images[image_id])}); + gpu_sentenced_bytes += freed_bytes; + } else { + sentenced_images.Push(std::move(slot_images[image_id])); + sentenced_image_bytes[sentenced_image_index] += freed_bytes; + } } slot_images.erase(image_id); diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index d51eefeee8..151dd9bfa1 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -556,6 +556,17 @@ private: /// counted in the driver's usage until the ring destroys them. std::array sentenced_image_bytes{}; size_t sentenced_image_index = 0; + /// Deleted images on runtimes that report GPU progress (CurrentTick, IsFree): destroyed, with + /// their device memory, as soon as the GPU is past them rather than TICKS_TO_DESTROY frames + /// later. While the collector deletes dozens a frame, eight frames of them held 1-1.5 GB of + /// VRAM above its threshold (title 1 on an 8 GB card: 6.6-7.0 GB in use against 5 GiB). + struct GpuSentencedImage { + u64 tick; + u64 bytes; + Image image; + }; + std::deque gpu_sentenced_images; + u64 gpu_sentenced_bytes{}; DelayedDestructionRing sentenced_image_view; DelayedDestructionRing sentenced_framebuffers; -- cgit v1.2.3 From 3871de9da994f9c0fa581ecd993db40081ce6b48 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:24:28 -0400 Subject: texture_cache: find a deleted image's view entries without a scan Deleting an image erased the channels' descriptor -> view entries pointing to its views by scanning every channel's map (Mario Kart World: ~5,000 entries, 50-130 us per deleted image, dozens a frame while the collector runs). The cache keeps, per view, the (channel, descriptor) entries that point to it and erases just those, skipping entries of channels that are gone or that point to another view since. (cherry picked from commit c3d22e8e09dacf39000c742d03dfc7517bd6f343) --- src/video_core/texture_cache/texture_cache.h | 31 +++++++++++++++-------- src/video_core/texture_cache/texture_cache_base.h | 6 +++++ 2 files changed, 27 insertions(+), 10 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index c8aaf8dff5..279668813f 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -689,8 +689,13 @@ ImageViewId TextureCache

::VisitImageView(u32 index, bool compute) { if (IsValidEntry(*gpu_memory, descriptor)) { // Is new (registered view) on the texture cache side? const auto [pair, is_new_tc] = channel_state->image_views.try_emplace(descriptor); - if (is_new_tc) + if (is_new_tc) { pair->second = CreateImageView(descriptor); + if (pair->second != NULL_IMAGE_VIEW_ID) { + image_view_descriptors[pair->second].emplace_back(current_channel_id, + descriptor); + } + } PrepareImageView(pair->second, false, false); channel_state->image_view_ids.insert_or_assign(map_index, pair->second); return pair->second; @@ -2917,17 +2922,23 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { template void TextureCache

::RemoveImageViewReferences(std::span removed_views) { - for (size_t c : active_channel_ids) { - auto& channel_info = channel_storage[c]; - auto it = channel_info.image_views.begin(); - while (it != channel_info.image_views.end()) { - const auto found = std::ranges::find(removed_views, it->second); - if (found != removed_views.end()) { - it = channel_info.image_views.erase(it); - } else { - ++it; + for (const ImageViewId removed : removed_views) { + const auto refs = image_view_descriptors.find(removed); + if (refs == image_view_descriptors.end()) { + continue; + } + for (const auto& [channel, descriptor] : refs->second) { + // The channel may be gone, or its entry point to another view since. + if (std::ranges::find(active_channel_ids, channel) == active_channel_ids.end()) { + continue; + } + auto& image_views = channel_storage[channel].image_views; + if (const auto it = image_views.find(descriptor); + it != image_views.end() && it->second == removed) { + image_views.erase(it); } } + image_view_descriptors.erase(refs); } } diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 94102cd261..628a5d8558 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -433,6 +433,12 @@ private: /// Remove image views references from the cache void RemoveImageViewReferences(std::span removed_views); + /// Per image view, the (channel, descriptor) entries of the channels' image_views maps that + /// point to it. Deleting an image erased them by scanning every map (title 1: ~5,000 entries, + /// 50-130 us for each deleted image, dozens a frame while the collector runs). + std::unordered_map, 2>> + image_view_descriptors; + /// Remove framebuffers using the given image views from the cache void RemoveFramebuffers(std::span removed_views); -- cgit v1.2.3 From e2c512916c6f5042d030fcf1708043233f8df08e Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:30:52 -0400 Subject: texture_cache: mark guest memory textures were uploaded from cold Once a texture is uploaded the host GPU has its own copy, and the guest rarely reads that memory again, but it stayed in the emulator's working set. Every 600 frames the guest ranges of images uploaded since (64 KiB or more, not GPU-written or sparse) are marked cold through HostMemory::HintCold, backing and CPU view, from a low-priority worker, so the OS pages them out before anything else when it needs RAM. Their contents stay; using them brings them back. Mario Kart World's title screen at ~10 minutes: game memory in RAM 4.8-5.5 -> 3.6 GB, the emulator's real RAM 10.6-11.9 -> 9.4 GB, frames over 100 ms not worse. (cherry picked from commit ea36b2c5ca352f8c8cdcc829a2d3bde9bdc06ece) --- src/common/host_memory.cpp | 2 +- src/common/host_memory.h | 3 +- src/core/device_memory_manager.h | 7 ++++ src/core/device_memory_manager.inc | 44 +++++++++++++++++++++++ src/video_core/texture_cache/texture_cache.h | 14 ++++++++ src/video_core/texture_cache/texture_cache_base.h | 2 ++ 6 files changed, 70 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index c0e7160dd4..2c45dfd3a5 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -791,7 +791,7 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, MemoryPermission } void HostMemory::HintCold(size_t physical_offset, size_t length, - std::optional virtual_offset) { + std::optional virtual_offset) const { if (length == 0 || backing_base == nullptr) { return; } diff --git a/src/common/host_memory.h b/src/common/host_memory.h index fe0938103c..4d8d15cf58 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -57,7 +57,8 @@ public: /// Memory written once and read again rarely: its pages leave the working set (backing and, /// when given, its view at virtual_offset), so the OS compresses or pages them out first when /// it needs RAM. The contents stay; using the pages brings them back. - void HintCold(size_t physical_offset, size_t length, std::optional virtual_offset); + void HintCold(size_t physical_offset, size_t length, + std::optional virtual_offset) const; [[nodiscard]] u8* BackingBasePointer() noexcept { return backing_base; diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 9c034f01a1..05a8136169 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -18,6 +18,7 @@ #include "common/common_types.h" #include "common/range_mutex.h" #include "common/scratch_buffer.h" +#include "common/thread_worker.h" #include "common/sparse_large_vector.h" namespace Core { @@ -53,6 +54,10 @@ public: void BindInterface(DeviceInterface* device_inter); + /// Marks the memory behind [address, address + size) cold (Common::HostMemory::HintCold), + /// from a low-priority thread: data the host GPU has its own copy of, e.g. uploaded textures. + void HintCold(DAddr address, size_t size); + DAddr Allocate(size_t size); void AllocateFixed(DAddr start, size_t size); void Free(DAddr start, size_t size); @@ -194,7 +199,9 @@ private: std::unique_ptr> impl; const uintptr_t physical_base; + const DeviceMemory* device_memory_ptr; DeviceInterface* device_inter; + std::unique_ptr cold_hint_worker; struct TrackedEntry { VAddr cpu_backing_address; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index c45bc71656..77ab95e07e 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -171,6 +171,7 @@ struct DeviceMemoryManagerAllocator { template DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memory_) : physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())} + , device_memory_ptr{&device_memory_} , device_inter{nullptr} , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS)) , tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS) @@ -182,6 +183,49 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo template DeviceMemoryManager::~DeviceMemoryManager() = default; +template +void DeviceMemoryManager::HintCold(DAddr address, size_t size) { + struct Run { + u64 physical_offset; + VAddr virtual_address; ///< 0: no CPU mapping known + u64 size; + }; + std::vector runs; + const size_t page_end = Common::DivCeil(address + size, Memory::YUZU_PAGESIZE); + for (size_t page = address >> Memory::YUZU_PAGEBITS; page < page_end; ++page) { + const u32 compressed = tracked_entries[page].compressed_physical_ptr; + if (compressed == 0) { + continue; + } + const u64 physical = u64{compressed - 1} << Memory::YUZU_PAGEBITS; + const VAddr vaddr = ExtractCPUBacking(page).second; + if (!runs.empty()) { + Run& last = runs.back(); + const bool contiguous_view = + last.virtual_address == 0 ? vaddr == 0 : last.virtual_address + last.size == vaddr; + if (last.physical_offset + last.size == physical && contiguous_view) { + last.size += Memory::YUZU_PAGESIZE; + continue; + } + } + runs.push_back({physical, vaddr, Memory::YUZU_PAGESIZE}); + } + if (runs.empty()) { + return; + } + if (!cold_hint_worker) { + cold_hint_worker = std::make_unique(1, "DeviceMemory:ColdHint"); + } + cold_hint_worker->QueueWork([this, runs = std::move(runs)] { + for (const Run& run : runs) { + device_memory_ptr->buffer.HintCold( + run.physical_offset, run.size, + run.virtual_address != 0 ? std::optional{run.virtual_address} + : std::nullopt); + } + }); +} + template void DeviceMemoryManager::BindInterface(DeviceInterface* device_inter_) { device_inter = device_inter_; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 63abffed0f..13637aa265 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -315,6 +315,12 @@ void TextureCache

::TickFrame() { sentenced_image_view.Tick(); TickAsyncDecode(); TickAsyncUnswizzle(); + if (frame_tick % 600 == 0 && !uploaded_ranges.empty()) { + for (const auto& [address, size] : uploaded_ranges) { + device_memory.HintCold(address, size); + } + uploaded_ranges.clear(); + } runtime.TickFrame(); ++frame_tick; @@ -1352,6 +1358,14 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image)); UploadImageContents(image, staging); runtime.InsertUploadMemoryBarrier(); + // The host GPU has its own copy now and the guest rarely reads this memory again: TickFrame + // marks it cold, so the OS pages it out before anything else when it needs RAM (title 1 + // streams several GB of textures through it). + constexpr u64 ColdHintMinBytes = 64_KiB; + if (image.guest_size_bytes >= ColdHintMinBytes && + False(image.flags & (ImageFlagBits::GpuModified | ImageFlagBits::Sparse))) { + uploaded_ranges.emplace_back(image.cpu_addr, image.guest_size_bytes); + } } template diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 11e95f98b3..26d989f2da 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -558,6 +558,8 @@ private: Common::ThreadWorker texture_decode_worker{1, "TextureDecoder", {}, Common::ThreadPlacement::Efficiency}; std::vector> async_decodes; + /// Guest memory images were uploaded from since the last cold hint (device address, size). + std::vector> uploaded_ranges; std::deque unswizzle_queue; u8 current_unswizzle_frame; -- cgit v1.2.3 From f398ff2e17f4e86701b698e68886e09d387889af Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:18:26 -0500 Subject: texture_cache: free the slots of deleted allocations and framebuffers Deleting the last image of an allocation erased its lookup but kept its slot, and removing a framebuffer moved it to the deletion queue but kept its slot, so both slot vectors grew with every allocation and framebuffer ever made (title 1: 8.3k allocations and 1k framebuffers made in its first 4 minutes, 5.2k and 600 alive). --- src/video_core/texture_cache/texture_cache.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 33ca051482..e0cc2fc517 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -2982,6 +2982,7 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { alloc_images.erase(alloc_image_it); if (alloc_images.empty()) { image_allocs_table.erase(alloc_it); + slot_image_allocs.erase(alloc_id); } for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; @@ -3023,6 +3024,7 @@ void TextureCache

::RemoveFramebuffers(std::span removed_vi last_framebuffer_serial = 0; } sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id])); + slot_framebuffers.erase(framebuffer_id); it = framebuffers.erase(it); } else { ++it; -- cgit v1.2.3 From ca1fac93fb4397a89424c6621be3515949b94729 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:11:11 -0400 Subject: texture_cache: cold-hint the guest memory of sparse images too Images whose guest memory is not contiguous were left out of the cold hint. With sparse tile remaps Mario Kart World maps its own memory into such images, and its game memory in RAM rose by ~1.9 GB in one run. Each mapped piece is now hinted. (cherry picked from commit 97f034615c174178301a4999c334fb39cd03e568) --- src/video_core/texture_cache/texture_cache.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 8ff3e012e1..249559e7be 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -1406,8 +1406,15 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { // streams several GB of textures through it). constexpr u64 ColdHintMinBytes = 64_KiB; if (image.guest_size_bytes >= ColdHintMinBytes && - False(image.flags & (ImageFlagBits::GpuModified | ImageFlagBits::Sparse))) { - uploaded_ranges.emplace_back(image.cpu_addr, image.guest_size_bytes); + False(image.flags & ImageFlagBits::GpuModified)) { + if (True(image.flags & ImageFlagBits::Sparse)) { + // Non-contiguous: each mapped piece (title 1 maps its own memory into these). + ForEachSparseSegment(image, [&](GPUVAddr, DAddr address, size_t size) { + uploaded_ranges.emplace_back(address, size); + }); + } else { + uploaded_ranges.emplace_back(image.cpu_addr, image.guest_size_bytes); + } } } -- cgit v1.2.3 From e37b4b9fa60f02fa77a4795cdd1bed0b8c71b933 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:02:00 -0500 Subject: texture_cache: collect images in the emptiest device memory blocks first Deleting by age alone left every block partly used: the collector deletes thousands of images a minute and new ones land in the holes, so blocks seldom empty (title 1: ~0.8 GB of holes in ~190 32 MiB blocks after ten minutes, counted by the driver as used). Among the oldest candidates, 8 per image a pass may delete, those in the emptiest blocks go first, so blocks drain and VMA returns them; placement already prefers the fullest blocks. (cherry picked from commit 2f8f0716fafd874cfbb3cc9086141aea61aff2df) --- src/video_core/renderer_vulkan/vk_texture_cache.h | 5 +++ src/video_core/texture_cache/texture_cache.h | 55 ++++++++++++++++++++++- src/video_core/texture_cache/texture_cache_base.h | 4 ++ src/video_core/vulkan_common/vulkan_wrapper.cpp | 13 ++++++ src/video_core/vulkan_common/vulkan_wrapper.h | 9 ++++ 5 files changed, 84 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index efdb705e93..e7ae9752a8 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -387,6 +387,11 @@ public: return (this->*current_image).UsageFlags(); } + /// Where the image's memory lies, for the collector (TextureCache::RunGarbageCollector). + [[nodiscard]] vk::Image::MemoryBlock MemoryBlock() const noexcept { + return original_image.GetMemoryBlock(); + } + /// Returns true when the image is already initialized and mark it as initialized [[nodiscard]] bool ExchangeInitialization() noexcept { return std::exchange(initialized, true); diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 249559e7be..c623c7d585 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -175,11 +175,62 @@ void TextureCache

::RunGarbageCollector() { } return false; }; + // Deleting by age alone left every device memory block partly used: the collector deletes + // thousands of images a minute and new ones land in the holes, so blocks seldom empty (title + // 1: ~0.8 GB of holes in ~190 32 MiB blocks after ten minutes, which the driver counts as + // used). Among the oldest candidates (8 per image this pass may delete), those in the + // emptiest blocks go first, so blocks drain and go back to the driver; placement already + // prefers the fullest blocks (best fit). Images with memory of their own sit in the middle. + const auto Collect = [&] { + if constexpr (requires(const Image& image) { image.MemoryBlock(); }) { + std::vector candidates; + const size_t limit = num_iterations * 8; + lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, [&](ImageId image_id) { + candidates.push_back(image_id); + return candidates.size() >= limit; + }); + if (candidates.empty()) { + return; + } + if (block_image_bytes_frame == ~0ULL || frame_tick - block_image_bytes_frame >= 60) { + block_image_bytes_frame = frame_tick; + block_image_bytes.clear(); + for (const auto [image_id, image] : slot_images) { + const auto block = image->MemoryBlock(); + if (block.block != 0) { + block_image_bytes[block.block] += block.size; + } + } + } + std::vector> ordered; + ordered.reserve(candidates.size()); + for (const ImageId image_id : candidates) { + const auto block = slot_images[image_id].MemoryBlock(); + double fill = 0.5; + if (block.block != 0 && block.block_size != 0) { + const auto it = block_image_bytes.find(block.block); + fill = it != block_image_bytes.end() + ? static_cast(it->second) / static_cast(block.block_size) + : 0.0; + } + ordered.emplace_back(fill, image_id); + } + std::stable_sort(ordered.begin(), ordered.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + for (const auto& [fill, image_id] : ordered) { + if (Cleanup(image_id)) { + break; + } + } + } else { + lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + } + }; Configure(false); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + Collect(); if (total_used_memory >= critical_memory) { Configure(true); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + Collect(); } } diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index af9d4f05a5..36702a9bff 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -588,6 +588,10 @@ private: }; std::deque gpu_sentenced_images; u64 gpu_sentenced_bytes{}; + + /// Bytes of images per device memory block, for the collector (refreshed every 60 frames). + std::unordered_map block_image_bytes; + u64 block_image_bytes_frame{~0ULL}; DelayedDestructionRing sentenced_image_view; DelayedDestructionRing sentenced_framebuffers; diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index ac67b95707..d0bca88125 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -526,6 +526,19 @@ void Image::Release() const noexcept { } } +Image::MemoryBlock Image::GetMemoryBlock() const noexcept { + if (allocation == nullptr) { + return {}; + } + VmaAllocationInfo2 info{}; + vmaGetAllocationInfo2(allocator, allocation, &info); + if (info.dedicatedMemory) { + return {0, info.blockSize, info.allocationInfo.size}; + } + return {reinterpret_cast(info.allocationInfo.deviceMemory), info.blockSize, + info.allocationInfo.size}; +} + void Buffer::Flush() const { if (!is_coherent) { vmaFlushAllocation(allocator, allocation, 0, VK_WHOLE_SIZE); diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index f9b3cc15df..f49e9518bb 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -737,6 +737,15 @@ public: return usage; } + /// Where the image's memory lies: the block (0 when the image has no memory or a block of + /// its own), the block's size and the image's part of it. + struct MemoryBlock { + u64 block; + u64 block_size; + u64 size; + }; + [[nodiscard]] MemoryBlock GetMemoryBlock() const noexcept; + private: void Release() const noexcept; -- cgit v1.2.3 From 16967eb30e97937f4ac3b200e681adde70089161 Mon Sep 17 00:00:00 2001 From: kraibse <33869366+kraibse@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:19:37 +0100 Subject: vulkan: don't make small images unbacked when device memory runs out When VRAM ran out, any image that failed to allocate became a sparse image with nothing bound, which AMD's drivers don't survive being sampled (an RX 6600 died mid-flood, an RX 7800 XT made 211 of them, 8x8 ones included). Try host memory first, and count the images that didn't fit: the texture cache then keeps more room under the budget (512 MiB a step, up to 3 GiB, decaying) so its collector runs earlier. The sparse fallback stays for images too large to back. --- src/video_core/renderer_vulkan/vk_texture_cache.h | 5 +++++ src/video_core/texture_cache/texture_cache.h | 23 ++++++++++++++++++++-- src/video_core/texture_cache/texture_cache_base.h | 3 +++ .../vulkan_common/vulkan_memory_allocator.cpp | 21 ++++++++++++++++++++ .../vulkan_common/vulkan_memory_allocator.h | 8 ++++++++ 5 files changed, 58 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index e7ae9752a8..5ac833e787 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -76,6 +76,11 @@ public: bool CanReportMemoryUsage() const; + /// Images that did not fit the device memory budget so far (see MemoryAllocator). + u32 MemoryPressureEvents() const noexcept { + return memory_allocator.MemoryPressureEvents(); + } + void LogMemoryStatistics() const; /// True when ASTC is re-compressed to BC3 and opaque images may use BC1 instead. diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index c623c7d585..cf042c1488 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -289,13 +289,32 @@ void TextureCache

::TickFrame() { // budget as the card fills (16 GB card: 13.5 GB left mid-race), hence the re-read every 30 // frames. EDEN_NX2_TC_*_MIB overrides still win. if constexpr (requires { runtime.GetDeviceMemoryBudget(); }) { + if constexpr (requires { runtime.MemoryPressureEvents(); }) { + // Images that did not fit (made in host memory or with nothing bound) mean the + // thresholds sit too high for this card and driver: leave more room until the + // pressure has been gone for a while. + if (native_thresholds_applied) { + const u32 events = runtime.MemoryPressureEvents(); + if (events != seen_pressure_events) { + seen_pressure_events = events; + pressure_margin = (std::min)(pressure_margin + 512_MiB, 3_GiB); + pressure_quiet_frames = 0; + budget_check_frames = 0; // thresholds are recomputed below + LOG_WARNING(HW_GPU, "Out of device memory: keeping {} MiB more free", + pressure_margin >> 20); + } else if (pressure_margin != 0 && ++pressure_quiet_frames % 600 == 0) { + pressure_margin -= (std::min)(pressure_margin, 128_MiB); + budget_check_frames = 0; + } + } + } if (native_thresholds_applied && budget_check_frames++ % 30 == 0) { const u64 budget = runtime.GetDeviceMemoryBudget(); if (budget > 2_GiB) { const u64 old_critical = critical_memory; const u64 margin = (std::min)(budget / 8, 1_GiB); - critical_memory = budget - margin; - expected_memory = budget - 2 * margin; + critical_memory = budget - (std::min)(margin + pressure_margin, budget / 2); + expected_memory = critical_memory - (std::min)(margin, critical_memory / 2); // Log the first values and real changes (the budget moves by a few MiB all // the time). const u64 change = critical_memory > old_critical diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 36702a9bff..1a6931d18a 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -136,6 +136,9 @@ class TextureCache : public VideoCommon::ChannelSetupCaches astc_scan_buffer; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 79fb331de0..a0a39abe39 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -245,11 +245,32 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, ci.arrayLayers, static_cast(ci.format)); } + pressure_events.fetch_add(1, std::memory_order_relaxed); VmaAllocationCreateInfo relaxed = alloc_ci; relaxed.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; result = vmaCreateImage(allocator, &ci, &relaxed, &handle, &allocation, &alloc_info); } + if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && !huge) { + // Still out of device memory: a real image in host memory (slow, but valid) beats one + // with nothing bound, which AMD's drivers do not survive being sampled. + VmaAllocationCreateInfo host = alloc_ci; + host.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + host.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST; + host.preferredFlags = 0; + result = vmaCreateImage(allocator, &ci, &host, &handle, &allocation, &alloc_info); + static u32 host_logged{}; + if (result == VK_SUCCESS && host_logged++ < 16) { + LOG_WARNING(Render_Vulkan, + "Image {}x{}x{} ({} levels, {} layers, format {}) is out of device memory; " + "placed in host memory", + ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, + ci.arrayLayers, static_cast(ci.format)); + } + } if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && ci.imageType != VK_IMAGE_TYPE_1D) { + if (!huge) { + pressure_events.fetch_add(1, std::memory_order_relaxed); + } // Too big to back at all, e.g. title 1's sparse virtual texture arrays (8192x8192, // 68 layers, ~23 GB): on the console only resident tiles have memory. Use an image with // nothing bound, so views and copies still work, instead of aborting. diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index ffca933ee7..8b9e72b3e3 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -126,6 +127,12 @@ namespace Vulkan { void FreeSparseMemory(VmaAllocation allocation) const; + /// Counts the images that did not fit the device memory budget (made anyway, in host + /// memory or with nothing bound): the texture cache collects harder when it grows. + u32 MemoryPressureEvents() const noexcept { + return pressure_events.load(std::memory_order_relaxed); + } + vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const; vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage, @@ -161,6 +168,7 @@ namespace Vulkan { const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties. VkDeviceSize buffer_image_granularity; ///< Adjacent buffer/image granularity u32 valid_memory_types{~0u}; + mutable std::atomic pressure_events{0}; mutable std::mutex buffer_type_mutex; /// Memory type of device-local buffers by usage and flags (CreateBufferInSeparatePool). mutable std::unordered_map buffer_memory_types; -- cgit v1.2.3 From 51d798ff0373a40098aa77ef13d54307f39e44ac Mon Sep 17 00:00:00 2001 From: kraibse <33869366+kraibse@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:38:05 +0100 Subject: texture_cache: keep the allocator's block slack out of the usable VRAM The collector's usage leaves out the unused part of VMA's blocks, but the driver counts it and a heap at its budget cannot give it to a new image. On an RX 7800 XT that slack was 1.06 GiB, the whole 1 GiB margin: the heap sat at its budget for a minute while the collector saw room, then images stopped fitting and the process died. Reserve the slack (capped at 1.5 GiB) on top of the margin: at once when it grows, slowly when it shrinks, so eviction leaving partly used blocks doesn't chase the threshold down. Assisted-by: Claude Sonnet 5.5 (diagnosis and code) (cherry picked from commit 73fd5a489ea28226fa16a55f5008e622d631b54c) --- .../renderer_vulkan/vk_texture_cache.cpp | 4 ++++ src/video_core/renderer_vulkan/vk_texture_cache.h | 2 ++ src/video_core/texture_cache/texture_cache.h | 23 ++++++++++++++++++++-- src/video_core/texture_cache/texture_cache_base.h | 1 + src/video_core/vulkan_common/vulkan_device.cpp | 10 ++++++++++ src/video_core/vulkan_common/vulkan_device.h | 8 +++++++- 6 files changed, 45 insertions(+), 3 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 09e7961ff9..cb0453c178 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -2082,6 +2082,10 @@ u64 TextureCacheRuntime::GetDeviceMemoryBudget() const { return device.GetDeviceMemoryBudget(); } +u64 TextureCacheRuntime::GetDeviceMemorySlack() const { + return device.GetDeviceMemorySlack(); +} + bool TextureCacheRuntime::CanDownloadMsaa(const VideoCommon::ImageInfo& info) const { const VkImageAspectFlags aspect_mask = ImageAspectMask(info.format); if ((aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0) { diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 5ac833e787..d5f8756486 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -74,6 +74,8 @@ public: u64 GetDeviceMemoryBudget() const; + u64 GetDeviceMemorySlack() const; + bool CanReportMemoryUsage() const; /// Images that did not fit the device memory budget so far (see MemoryAllocator). diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index cf042c1488..45bc35dee6 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -313,14 +313,33 @@ void TextureCache

::TickFrame() { if (budget > 2_GiB) { const u64 old_critical = critical_memory; const u64 margin = (std::min)(budget / 8, 1_GiB); - critical_memory = budget - (std::min)(margin + pressure_margin, budget / 2); + if constexpr (requires { runtime.GetDeviceMemorySlack(); }) { + // The usage counted here leaves out the unused part of the allocator's + // blocks, which the driver counts and a full heap cannot hand to a new + // image (RX 7800 XT: 1.06 GiB of it, the whole margin, with the heap at + // its budget for a minute before images stopped fitting). Reserve it + // too: at once when it grows, slowly when it shrinks, so evicting images + // (which leaves partly used blocks) does not chase the threshold down. + const u64 slack = (std::min)(runtime.GetDeviceMemorySlack(), 1536_MiB); + slack_margin = slack >= slack_margin + ? slack + : slack_margin - (std::min)(slack_margin - slack, 16_MiB); + } + critical_memory = + budget - (std::min)(margin + pressure_margin + slack_margin, budget / 2); expected_memory = critical_memory - (std::min)(margin, critical_memory / 2); // Log the first values and real changes (the budget moves by a few MiB all // the time). const u64 change = critical_memory > old_critical ? critical_memory - old_critical : old_critical - critical_memory; - ApplyThresholdOverrides(budget_check_frames == 1 || change >= 256_MiB); + const bool log = budget_check_frames == 1 || change >= 256_MiB; + ApplyThresholdOverrides(log); + if (log) { + LOG_INFO(HW_GPU, + "Memory margins: base {} MiB, out of memory {} MiB, block slack {} MiB", + margin >> 20, pressure_margin >> 20, slack_margin >> 20); + } } } } diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 1a6931d18a..24ebaf0689 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -139,6 +139,7 @@ class TextureCache : public VideoCommon::ChannelSetupCaches astc_scan_buffer; diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index 2ecbd2cb94..01e13cb384 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -1684,6 +1684,16 @@ u64 Device::GetDeviceMemoryUsage() const { return result; } +u64 Device::GetDeviceMemorySlack() const { + if (allocator == nullptr) { + return 0; + } + std::array heap_budgets{}; + vmaGetHeapBudgets(allocator, heap_budgets.data()); + const VmaStatistics& statistics = heap_budgets[main_heap].statistics; + return statistics.blockBytes - (std::min)(statistics.blockBytes, statistics.allocationBytes); +} + u64 Device::GetDeviceMemoryBudget() const { if (!extensions.memory_budget) { return 0; diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index 2d19321145..622d792210 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -1019,9 +1019,15 @@ FN_MAX_LIMIT_LIST /// The driver's current memory budget for this process (VK_EXT_memory_budget), in /// GetDeviceMemoryUsage() terms: the usage at which the main device-local heap reaches its - /// budget. 0 without the extension. + /// budget. 0 without the extension. The unused part of the allocator's blocks is not + /// subtracted: see GetDeviceMemorySlack(). u64 GetDeviceMemoryBudget() const; + /// Memory the allocator holds from the driver in the main device-local heap but has no + /// allocation in (the unused part of its blocks). GetDeviceMemoryUsage() does not count it, + /// yet the driver does: with the heap at its budget it is not available to new images. + u64 GetDeviceMemorySlack() const; + u32 GetSetsPerPool() const { return sets_per_pool; } -- cgit v1.2.3 From 92d49d8a0c3e60894765bc2c332fdc936c87fc45 Mon Sep 17 00:00:00 2001 From: Duck912 <350528965+Duck912@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:43:28 -0500 Subject: video_core: temporal upscale stand-in for Fortune's Weave's DLSS The one-frame blit left 3D edges aliased and shaking: the game jitters its camera every frame for DLSS to accumulate. Fire Emblem: Fortune's Weave's first DLSS pass (also rejected) has the motion vectors (cbuf0+0x290, R16G16) and the frame's jitter (+0x1b0/+0x1b4, in input pixels). Kepler compute keeps them, and the stand-in for the last network pass now draws temporal_upscale.frag: input samples placed where the jitter put them and weighted by distance, blended with the previous result reprojected along the motion vectors (Catmull-Rom) and clipped to the 3x3 neighbourhood in YCoCg. The history is copied from the result right after drawing it, so post-processing the game later applies to its output does not feed back. EDEN_NX2_TEMPORAL_UPSCALE=0 restores the blit. Assisted-by: Claude Opus 5.5 (debugging, implementation and drafting) Signed-off-by: Duck912 <350528965+Duck912@users.noreply.github.com> --- src/video_core/engines/kepler_compute.cpp | 47 ++++++- src/video_core/engines/kepler_compute.h | 8 ++ src/video_core/host_shaders/CMakeLists.txt | 1 + src/video_core/host_shaders/temporal_upscale.frag | 132 ++++++++++++++++++ src/video_core/rasterizer_interface.h | 11 ++ src/video_core/renderer_vulkan/blit_image.cpp | 147 ++++++++++++++++++++ src/video_core/renderer_vulkan/blit_image.h | 15 +++ src/video_core/renderer_vulkan/vk_rasterizer.cpp | 8 ++ src/video_core/renderer_vulkan/vk_rasterizer.h | 5 + .../renderer_vulkan/vk_texture_cache.cpp | 148 +++++++++++++++++++++ src/video_core/renderer_vulkan/vk_texture_cache.h | 15 +++ src/video_core/texture_cache/texture_cache.h | 67 ++++++++++ src/video_core/texture_cache/texture_cache_base.h | 7 + 13 files changed, 610 insertions(+), 1 deletion(-) create mode 100644 src/video_core/host_shaders/temporal_upscale.frag (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 233304d273..b8a6567907 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -557,6 +558,7 @@ bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& // The console runs this on tensor cores; stand in with a plain upscale of its input. return UpscaleInsteadOfNativeProgram(); } + RememberDlssMotionVectors(); return false; } return true; @@ -588,6 +590,30 @@ void KeplerCompute::DescribeNativeResources(const NativeQmdScheduler::Descriptor std::atomic g_nx2_trace_draws_after_upscale{}; +// Fire Emblem: Fortune's Weave's first DLSS pass (rejected, before the network) has the input colour +// at cbuf0+0x278, its motion vectors (R16G16 float, the input's size) at +0x290 and this frame's +// camera jitter (x, y; input pixels, Halton bases 2 and 3 less 0.5) at +0x1b0. +void KeplerCompute::RememberDlssMotionVectors() { + const auto& cbuf = launch_description.const_buffer_config[0]; + if (cbuf.size.Value() < 0x294) return; + const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low; + const auto image = [&](u32 offset) -> std::optional { + const u32 tic_index = memory_manager.Read(cbuf_address + offset) & 0xfffff; + if (tic_index > regs.tic.limit) return std::nullopt; + return GetTICEntry(tic_index); + }; + const auto color = image(0x278), motion = image(0x290); + if (!color || !motion || color->format != Texture::TextureFormat::B10G11R11 || + motion->format != Texture::TextureFormat::R16G16 || motion->Address() == 0 || + motion->Width() != color->Width() || motion->Height() != color->Height() || + !motion->IsBlockLinear()) + return; + dlss_motion_vectors = *motion; + dlss_motion_color = color->Address(); + dlss_jitter = {std::bit_cast(memory_manager.Read(cbuf_address + 0x1b0)), + std::bit_cast(memory_manager.Read(cbuf_address + 0x1b4))}; +} + // Title 4's upscaler reads the tonemapped frame through the texture handle at cbuf0+0x90 and // writes the output resolution image through the surface handle at cbuf0+0xa0 (from its // TLD/SUST operands). Fire Emblem: Fortune's Weave's DLSS network ends with a pass that reads the @@ -609,10 +635,12 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() { output->IsBlockLinear(); }; auto input = image(0x90), output = image(0xa0); + bool fortunes_weave = false; if (!usable(input, output)) { input = image(0x328); output = image(0x348); if (!usable(input, output)) return false; + fortunes_weave = true; } using RTFormat = Tegra::RenderTargetFormat; RTFormat format{}; @@ -709,7 +737,24 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() { source.format = static_cast(std::strtoul(value, nullptr, 16)); } } - const bool done = rasterizer->AccelerateSurfaceCopy(source, surface(*output), config); + // Fortune's Weave: a temporal upscale along its motion vectors that undoes this frame's camera + // jitter (both from the frame's first DLSS pass) and accumulates the frames as DLSS would. + // EDEN_NX2_TEMPORAL_UPSCALE=0 blits instead. + const auto upscale_temporally = [&] { + static const bool enabled = [] { + const char* value = std::getenv("EDEN_NX2_TEMPORAL_UPSCALE"); + return value == nullptr || std::string_view{value} != "0"; + }(); + if (!fortunes_weave || !enabled || !dlss_motion_vectors || + dlss_motion_color != input->Address()) + return false; + Fermi2D::Surface motion = surface(*dlss_motion_vectors); + motion.format = RTFormat::R16G16_FLOAT; + return rasterizer->AccelerateTemporalUpscale(source, surface(*output), motion, dlss_jitter, + {1.0f, 1.0f}); + }; + const bool done = upscale_temporally() || + rasterizer->AccelerateSurfaceCopy(source, surface(*output), config); if (static u32 checked{}; checked++ == sample_at) { const u64 size = u64{output->Width()} * output->Height() * 4; if (const auto device = memory_manager.GpuToCpuAddress(output->Address())) { diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index b525636dde..b5aee3706a 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -266,7 +266,15 @@ private: bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor); void ProcessNativeReportSemaphore(u32 argument); void DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd); + void RememberDlssMotionVectors(); bool UpscaleInsteadOfNativeProgram(); + + /// Motion vectors of Fire Emblem: Fortune's Weave's DLSS evaluation, from its first pass (run + /// earlier in the frame than the network pass the stand-in replaces), with the input colour + /// image and camera jitter they go with. + std::optional dlss_motion_vectors; + GPUVAddr dlss_motion_color{}; + std::array dlss_jitter{}; bool ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call); NativeQmdScheduler native_qmd; std::array* native_data_ram{}; diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index 35b350c2cd..ada5c779dc 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -71,6 +71,7 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/smaa_blending_weight_calculation.frag ${CMAKE_CURRENT_SOURCE_DIR}/smaa_neighborhood_blending.vert ${CMAKE_CURRENT_SOURCE_DIR}/smaa_neighborhood_blending.frag + ${CMAKE_CURRENT_SOURCE_DIR}/temporal_upscale.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_blit_depth_stencil.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_color_clear.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_color_clear.vert diff --git a/src/video_core/host_shaders/temporal_upscale.frag b/src/video_core/host_shaders/temporal_upscale.frag new file mode 100644 index 0000000000..0e76df2b43 --- /dev/null +++ b/src/video_core/host_shaders/temporal_upscale.frag @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// Stands in for a game's DLSS evaluation (the tensor-core network cannot run here): a temporal +// upscale. Each frame's input samples are placed where the camera jitter put them and weighted by +// their distance to the output pixel; they accumulate with the previous output, reprojected along +// the game's motion vectors (bicubic) and clipped to the current 3x3 neighbourhood. + +#version 450 + +layout(binding = 0) uniform sampler2D input_color; +layout(binding = 1) uniform sampler2D motion_vectors; +layout(binding = 2) uniform sampler2D history; + +layout(push_constant) uniform PushConstants { + // The vertex stage's texture scale and offset come first. + layout(offset = 16) vec2 jitter; // this frame's sample offset, in input pixels + vec2 motion_scale; // motion vector -> UV offset to the previous frame + float current_weight; // largest weight of this frame; 1 ignores the history +}; + +layout(location = 0) in vec2 texcoord; // 0 to 1 across the output +layout(location = 0) out vec4 color; + +// The blit helper's samplers take unnormalized (texel) coordinates. + +// Blending in a range-compressed space keeps bright pixels from dominating the average. +vec3 Compress(vec3 c) { + return c / (1.0 + max(max(c.r, c.g), c.b)); +} + +vec3 Expand(vec3 c) { + c = clamp(c, vec3(0.0), vec3(1.0 - 1.0 / 65504.0)); + return c / (1.0 - max(max(c.r, c.g), c.b)); +} + +vec3 RgbToYCoCg(vec3 c) { + return vec3(dot(c, vec3(0.25, 0.5, 0.25)), dot(c, vec3(0.5, 0.0, -0.5)), + dot(c, vec3(-0.25, 0.5, -0.25))); +} + +vec3 YCoCgToRgb(vec3 c) { + return vec3(c.x + c.y - c.z, c.x + c.z, c.x - c.y - c.z); +} + +// Moves value towards the box centre until it lies inside the box. +vec3 ClipToBox(vec3 value, vec3 low, vec3 high) { + const vec3 center = 0.5 * (low + high); + const vec3 extent = 0.5 * (high - low) + 1.0 / 65504.0; + const vec3 offset = value - center; + const vec3 units = abs(offset / extent); + const float largest = max(units.x, max(units.y, units.z)); + return largest > 1.0 ? center + offset / largest : value; +} + +// Catmull-Rom filtered history from five bilinear fetches: repeated bilinear reprojection would +// blur the accumulated image a little more every frame. +vec3 SampleHistory(vec2 position) { + const vec2 texel1 = floor(position - 0.5) + 0.5; + const vec2 f = position - texel1; + const vec2 w0 = f * (-0.5 + f * (1.0 - 0.5 * f)); + const vec2 w1 = 1.0 + f * f * (-2.5 + 1.5 * f); + const vec2 w2 = f * (0.5 + f * (2.0 - 1.5 * f)); + const vec2 w3 = f * f * (-0.5 + 0.5 * f); + const vec2 w12 = w1 + w2; + const vec2 texel0 = texel1 - 1.0; + const vec2 texel3 = texel1 + 2.0; + const vec2 texel12 = texel1 + w2 / w12; + vec3 sum = texture(history, vec2(texel12.x, texel0.y)).rgb * (w12.x * w0.y); + sum += texture(history, vec2(texel0.x, texel12.y)).rgb * (w0.x * w12.y); + sum += texture(history, texel12).rgb * (w12.x * w12.y); + sum += texture(history, vec2(texel3.x, texel12.y)).rgb * (w3.x * w12.y); + sum += texture(history, vec2(texel12.x, texel3.y)).rgb * (w12.x * w3.y); + const float weight = w12.x * w0.y + w0.x * w12.y + w12.x * w12.y + w3.x * w12.y + w12.x * w3.y; + return max(sum / weight, vec3(0.0)); +} + +void main() { + const ivec2 input_size = textureSize(input_color, 0); + const ivec2 motion_size = textureSize(motion_vectors, 0); + // The output pixel's centre in input pixels. Input texel k sampled the scene at k + 0.5 - jitter + // (the sign measured against consecutive frames). + const vec2 target = texcoord * vec2(input_size); + const ivec2 nearest = ivec2(floor(target + jitter)); + + vec3 filtered = vec3(0.0); + float total_weight = 0.0; + float nearest_weight = 0.0; + vec3 sum = vec3(0.0); + vec3 squares = vec3(0.0); + vec2 motion = vec2(0.0); + float longest = -1.0; + for (int y = -1; y <= 1; ++y) { + for (int x = -1; x <= 1; ++x) { + const ivec2 texel = clamp(nearest + ivec2(x, y), ivec2(0), input_size - 1); + const vec3 sample_color = RgbToYCoCg(Compress(texelFetch(input_color, texel, 0).rgb)); + // A Blackman-Harris-like window over the sample's distance to the output pixel. + const vec2 distance = vec2(texel) + 0.5 - jitter - target; + const float weight = exp(-2.29 * dot(distance, distance)); + filtered += sample_color * weight; + total_weight += weight; + nearest_weight = max(nearest_weight, weight); + sum += sample_color; + squares += sample_color * sample_color; + // The longest motion around the pixel, so that an edge follows the object in front. + const vec2 candidate = + texelFetch(motion_vectors, clamp(texel, ivec2(0), motion_size - 1), 0).rg; + const float length_squared = dot(candidate, candidate); + if (length_squared > longest) { + longest = length_squared; + motion = candidate; + } + } + } + const vec3 current = filtered / total_weight; + const vec3 mean = sum / 9.0; + const vec3 deviation = sqrt(max(squares / 9.0 - mean * mean, vec3(0.0))); + const vec3 low = mean - deviation; + const vec3 high = mean + deviation; + + const vec2 previous = texcoord + motion * motion_scale; + // A frame whose samples land close to this pixel counts more; one whose samples land between + // pixels mostly keeps the history (from other jitter positions) there. + float weight = current_weight * nearest_weight; + if (current_weight >= 1.0 || any(lessThan(previous, vec2(0.0))) || + any(greaterThan(previous, vec2(1.0)))) { + weight = 1.0; + } + const vec2 history_position = previous * vec2(textureSize(history, 0)); + const vec3 past = ClipToBox(RgbToYCoCg(Compress(SampleHistory(history_position))), low, high); + color = vec4(Expand(YCoCgToRgb(mix(past, current, weight))), 1.0); +} diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index b527c0e2a9..8486d44ac9 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -162,6 +163,16 @@ public: return false; } + /// Stand-in for a DLSS evaluation: temporal upscale of src into dst along motion_vectors + /// (jitter in src pixels; motion_scale turns a motion vector into a UV offset to the previous + /// frame). False when unsupported; the caller then falls back to AccelerateSurfaceCopy. + [[nodiscard]] virtual bool AccelerateTemporalUpscale( + const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, std::array jitter, + std::array motion_scale) { + return false; + } + [[nodiscard]] virtual Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() = 0; virtual void AccelerateInlineToMemory(GPUVAddr address, size_t copy_size, diff --git a/src/video_core/renderer_vulkan/blit_image.cpp b/src/video_core/renderer_vulkan/blit_image.cpp index ddd52e9426..edd151b563 100644 --- a/src/video_core/renderer_vulkan/blit_image.cpp +++ b/src/video_core/renderer_vulkan/blit_image.cpp @@ -33,6 +33,7 @@ #include "video_core/host_shaders/convert_non_msaa_to_msaa_depth_stencil_frag_spv.h" #include "video_core/host_shaders/convert_s8d24_to_abgr8_frag_spv.h" #include "video_core/host_shaders/full_screen_triangle_vert_spv.h" +#include "video_core/host_shaders/temporal_upscale_frag_spv.h" #include "video_core/host_shaders/vulkan_blit_depth_stencil_frag_spv.h" #include "video_core/host_shaders/vulkan_color_clear_frag_spv.h" #include "video_core/host_shaders/vulkan_color_clear_vert_spv.h" @@ -91,6 +92,15 @@ struct PushConstants { std::array tex_offset; }; +// The vertex stage's PushConstants first, then those of temporal_upscale.frag. +struct TemporalUpscalePushConstants { + PushConstants vertex; + std::array jitter; + std::array motion_scale; + float current_weight; + float padding; +}; + struct MSAACopyPushConstants { std::array dst_offset; std::array src_offset; @@ -109,6 +119,11 @@ constexpr std::array TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{ TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>, TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>, }; +constexpr std::array THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{ + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>, + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>, + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<2>, +}; constexpr VkDescriptorSetLayoutCreateInfo ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{ .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .pNext = nullptr, @@ -133,6 +148,13 @@ constexpr VkDescriptorSetLayoutCreateInfo TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CRE .bindingCount = static_cast(TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()), .pBindings = TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(), }; +constexpr VkDescriptorSetLayoutCreateInfo THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{ + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .bindingCount = static_cast(THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()), + .pBindings = THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(), +}; template inline constexpr VkPushConstantRange PUSH_CONSTANT_RANGE{ .stageFlags = stageFlags, @@ -357,6 +379,32 @@ void UpdateOneTextureDescriptorSet(const Device& device, VkDescriptorSet descrip device.GetLogical().UpdateDescriptorSets(write_descriptor_set, nullptr); } +void UpdateThreeTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set, + VkSampler sampler, std::array image_views) { + std::array image_infos{}; + std::array write_descriptor_sets{}; + for (u32 binding = 0; binding < 3; ++binding) { + image_infos[binding] = VkDescriptorImageInfo{ + .sampler = sampler, + .imageView = image_views[binding], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + }; + write_descriptor_sets[binding] = VkWriteDescriptorSet{ + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = nullptr, + .dstSet = descriptor_set, + .dstBinding = binding, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImageInfo = &image_infos[binding], + .pBufferInfo = nullptr, + .pTexelBufferView = nullptr, + }; + } + device.GetLogical().UpdateDescriptorSets(write_descriptor_sets, nullptr); +} + void UpdateTwoTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set, VkSampler sampler, VkImageView image_view_0, VkImageView image_view_1) { @@ -590,10 +638,14 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), two_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout( TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), + three_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout( + THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), one_texture_descriptor_allocator{ descriptor_pool.Allocator(device_, scheduler_, *one_texture_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<1>)}, two_textures_descriptor_allocator{ descriptor_pool.Allocator(device_, scheduler_, *two_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<2>)}, + three_textures_descriptor_allocator{ + descriptor_pool.Allocator(device_, scheduler_, *three_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<3>)}, one_texture_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( one_texture_set_layout.address(), PUSH_CONSTANT_RANGE))), @@ -601,6 +653,11 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( two_textures_set_layout.address(), PUSH_CONSTANT_RANGE))), + temporal_upscale_pipeline_layout( + device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( + three_textures_set_layout.address(), + PUSH_CONSTANT_RANGE))), clear_color_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( nullptr, PUSH_CONSTANT_RANGE))), msaa_copy_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( @@ -612,6 +669,7 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, PUSH_CONSTANT_RANGE))), full_screen_vert(BuildShader(device, FULL_SCREEN_TRIANGLE_VERT_SPV)), blit_color_to_color_frag(BuildShader(device, BLIT_COLOR_FLOAT_FRAG_SPV)), + temporal_upscale_frag(BuildShader(device, TEMPORAL_UPSCALE_FRAG_SPV)), blit_color_msaa_frag(BuildShader(device, BLIT_COLOR_MSAA_FRAG_SPV)), blit_depth_stencil_frag(device.IsExtShaderStencilExportSupported() ? BuildShader(device, VULKAN_BLIT_DEPTH_STENCIL_FRAG_SPV) @@ -702,6 +760,42 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView }); } +void BlitImageHelper::TemporalUpscale(const Framebuffer* dst_framebuffer, + const ImageView& input_color, const ImageView& motion_vectors, + VkImageView history, const Region2D& dst_region, + std::array jitter, std::array motion_scale, + f32 current_weight) { + const VkPipelineLayout layout = *temporal_upscale_pipeline_layout; + const VkPipeline pipeline = FindOrEmplaceTemporalUpscalePipeline(dst_framebuffer->RenderPass()); + const VkSampler sampler = *linear_sampler; + const std::array views{input_color.Handle(Shader::TextureType::Color2D), + motion_vectors.Handle(Shader::TextureType::Color2D), + history}; + const TemporalUpscalePushConstants push_constants{ + .vertex = {.tex_scale = {1.0f, 1.0f}, .tex_offset = {0.0f, 0.0f}}, + .jitter = jitter, + .motion_scale = motion_scale, + .current_weight = current_weight, + .padding = 0.0f, + }; + RecordShaderReadBarrier(scheduler, input_color); + RecordShaderReadBarrier(scheduler, motion_vectors); + scheduler.RequestRenderpass(dst_framebuffer); + scheduler.Record([this, dst_region, pipeline, layout, sampler, views, + push_constants](vk::CommandBuffer cmdbuf) { + const VkDescriptorSet descriptor_set = three_textures_descriptor_allocator.Commit(); + UpdateThreeTexturesDescriptorSet(device, descriptor_set, sampler, views); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, descriptor_set, + nullptr); + BindBlitState(cmdbuf, dst_region); + cmdbuf.PushConstants(layout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + push_constants); + cmdbuf.Draw(3, 1, 0, 0); + }); + scheduler.InvalidateState(); +} + void BlitImageHelper::BlitImpl(const Framebuffer* dst_framebuffer, const ImageView& src_image_view, const Region2D& dst_region, const Region2D& src_region, VkPipeline pipeline, VkSampler sampler, @@ -1285,6 +1379,59 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe return *blit_color_pipelines.back(); } +VkPipeline BlitImageHelper::FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass) { + const auto it = std::ranges::find(temporal_upscale_keys, renderpass); + if (it != temporal_upscale_keys.end()) { + return *temporal_upscale_pipelines[std::distance(temporal_upscale_keys.begin(), it)]; + } + temporal_upscale_keys.push_back(renderpass); + const std::array stages = MakeStages(*full_screen_vert, *temporal_upscale_frag); + const VkPipelineColorBlendAttachmentState blend_attachment{ + .blendEnable = VK_FALSE, + .srcColorBlendFactor = VK_BLEND_FACTOR_ZERO, + .dstColorBlendFactor = VK_BLEND_FACTOR_ZERO, + .colorBlendOp = VK_BLEND_OP_ADD, + .srcAlphaBlendFactor = VK_BLEND_FACTOR_ZERO, + .dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO, + .alphaBlendOp = VK_BLEND_OP_ADD, + .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | + VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, + }; + const VkPipelineColorBlendStateCreateInfo color_blend_create_info{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .logicOpEnable = VK_FALSE, + .logicOp = VK_LOGIC_OP_CLEAR, + .attachmentCount = 1, + .pAttachments = &blend_attachment, + .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, + }; + const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + temporal_upscale_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ + .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .stageCount = static_cast(stages.size()), + .pStages = stages.data(), + .pVertexInputState = &PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, + .pInputAssemblyState = &input_assembly_ci, + .pTessellationState = nullptr, + .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, + .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, + .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, + .pDepthStencilState = nullptr, + .pColorBlendState = &color_blend_create_info, + .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, + .layout = *temporal_upscale_pipeline_layout, + .renderPass = renderpass, + .subpass = 0, + .basePipelineHandle = VK_NULL_HANDLE, + .basePipelineIndex = 0, + }, device.StaticPipelineCache())); + return *temporal_upscale_pipelines.back(); +} + VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key) { const auto it = std::ranges::find(blit_depth_stencil_keys, key); if (it != blit_depth_stencil_keys.end()) { diff --git a/src/video_core/renderer_vulkan/blit_image.h b/src/video_core/renderer_vulkan/blit_image.h index 62e6743a23..303beeea5c 100644 --- a/src/video_core/renderer_vulkan/blit_image.h +++ b/src/video_core/renderer_vulkan/blit_image.h @@ -82,6 +82,13 @@ public: VkImage src_image, VkSampler src_sampler, const Region2D& dst_region, const Region2D& src_region, const Extent3D& src_size); + /// Temporal upscale (temporal_upscale.frag) into dst_framebuffer: the jittered input colour, + /// accumulated with the history image reprojected along the motion vectors. + void TemporalUpscale(const Framebuffer* dst_framebuffer, const ImageView& input_color, + const ImageView& motion_vectors, VkImageView history, + const Region2D& dst_region, std::array jitter, + std::array motion_scale, f32 current_weight); + void BlitColorMSAA(const Framebuffer* dst_framebuffer, const ImageView& src_image_view, const Region2D& dst_region, const Region2D& src_region); @@ -170,6 +177,8 @@ private: [[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key); + [[nodiscard]] VkPipeline FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass); + [[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key); [[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key); @@ -207,15 +216,19 @@ private: vk::DescriptorSetLayout one_texture_set_layout; vk::DescriptorSetLayout two_textures_set_layout; + vk::DescriptorSetLayout three_textures_set_layout; DescriptorAllocator one_texture_descriptor_allocator; DescriptorAllocator two_textures_descriptor_allocator; + DescriptorAllocator three_textures_descriptor_allocator; vk::PipelineLayout one_texture_pipeline_layout; vk::PipelineLayout two_textures_pipeline_layout; + vk::PipelineLayout temporal_upscale_pipeline_layout; vk::PipelineLayout clear_color_pipeline_layout; vk::PipelineLayout msaa_copy_pipeline_layout; vk::PipelineLayout msaa_copy_depth_stencil_pipeline_layout; vk::ShaderModule full_screen_vert; vk::ShaderModule blit_color_to_color_frag; + vk::ShaderModule temporal_upscale_frag; vk::ShaderModule blit_color_msaa_frag; vk::ShaderModule blit_depth_stencil_frag; vk::ShaderModule blit_depth_frag; @@ -246,6 +259,8 @@ private: std::vector blit_color_keys; std::vector blit_color_pipelines; + std::vector temporal_upscale_keys; + std::vector temporal_upscale_pipelines; std::vector blit_depth_stencil_keys; std::vector blit_depth_stencil_pipelines; std::vector clear_color_keys; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index cb31a0e7e4..c0e8825bac 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1231,6 +1231,14 @@ bool RasterizerVulkan::AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surf return texture_cache.BlitImage(dst, src, copy_config); } +bool RasterizerVulkan::AccelerateTemporalUpscale( + const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, std::array jitter, + std::array motion_scale) { + std::scoped_lock lock{texture_cache.mutex}; + return texture_cache.TemporalUpscale(dst, src, motion_vectors, jitter, motion_scale); +} + Tegra::Engines::AccelerateDMAInterface& RasterizerVulkan::AccessAccelerateDMA() { return accelerate_dma; } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index ae9929ca0b..7aca376614 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -128,6 +128,11 @@ public: bool AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, const Tegra::Engines::Fermi2D::Config& copy_config) override; + bool AccelerateTemporalUpscale(const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array jitter, + std::array motion_scale) override; Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() override; void AccelerateInlineToMemory(GPUVAddr address, size_t copy_size, std::span memory) override; diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index ddd55de690..4e428229f6 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -1491,6 +1491,154 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src, }); } +bool TextureCacheRuntime::TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst, + ImageView& input_color, ImageView& motion_vectors, + std::array jitter, + std::array motion_scale) { + if (dst.Samples() != VK_SAMPLE_COUNT_1_BIT || ImageAspectMask(dst.format) != VK_IMAGE_ASPECT_COLOR_BIT) { + return false; + } + const VkFormat format = + MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, false, dst.format).format; + const VkExtent2D extent{dst.size.width, dst.size.height}; + const VkImage output = dst.ImageHandle(); + bool reset = false; + if (!temporal_history.image || temporal_history.format != format || + temporal_history.extent.width != extent.width || + temporal_history.extent.height != extent.height) { + if (temporal_history.image) { + scheduler.Finish(); // the old history may still be sampled + } + temporal_history.image = memory_allocator.CreateImage(VkImageCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .imageType = VK_IMAGE_TYPE_2D, + .format = format, + .extent = {extent.width, extent.height, 1}, + .mipLevels = 1, + .arrayLayers = 1, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + }); + temporal_history.view = device.GetLogical().CreateImageView(VkImageViewCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .image = *temporal_history.image, + .viewType = VK_IMAGE_VIEW_TYPE_2D, + .format = format, + .components{}, + .subresourceRange{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + }, + }); + temporal_history.format = format; + temporal_history.extent = extent; + reset = true; + LOG_INFO(Render_Vulkan, "Temporal upscale into {}x{} (format {})", extent.width, extent.height, + static_cast(format)); + } + + // The history holds this pass's previous result, copied right after drawing it: the game may + // draw into its DLSS output afterwards (post-processing), which must not feed back. The history + // stays in the GENERAL layout, which the blit helper's descriptors use. + const VkImage history = *temporal_history.image; + const VkImageSubresourceLayers output_layers{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = static_cast(dst.range.base.level), + .baseArrayLayer = static_cast(dst.range.base.layer), + .layerCount = 1, + }; + static constexpr VkImageSubresourceLayers history_layers{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = 0, + .baseArrayLayer = 0, + .layerCount = 1, + }; + const auto barrier = [](VkImage image, VkAccessFlags src_access, VkAccessFlags dst_access, + VkImageLayout old_layout, const VkImageSubresourceLayers& layers) { + return VkImageMemoryBarrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = src_access, + .dstAccessMask = dst_access, + .oldLayout = old_layout, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = image, + .subresourceRange{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = layers.mipLevel, + .levelCount = 1, + .baseArrayLayer = layers.baseArrayLayer, + .layerCount = 1, + }, + }; + }; + if (reset) { + // A new history image has no contents yet; the first frame does not read it. + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([history, barrier](vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + 0, barrier(history, 0, VK_ACCESS_SHADER_READ_BIT, + VK_IMAGE_LAYOUT_UNDEFINED, history_layers)); + }); + } + const Region2D dst_region{ + .start = {0, 0}, + .end = {static_cast(extent.width), static_cast(extent.height)}, + }; + // Each frame weighs at most 0.1 against the history (scaled by how close its samples land to + // the pixel; the neighbourhood clip limits what older samples contribute). + constexpr f32 current_weight = 0.1f; + blit_image_helper.TemporalUpscale(dst_framebuffer, input_color, motion_vectors, + *temporal_history.view, dst_region, jitter, motion_scale, + reset ? 1.0f : current_weight); + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([output, history, extent, output_layers, barrier](vk::CommandBuffer cmdbuf) { + constexpr VkAccessFlags written = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + const std::array before{ + barrier(output, written, VK_ACCESS_TRANSFER_READ_BIT, VK_IMAGE_LAYOUT_GENERAL, + output_layers), + barrier(history, VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, + VK_IMAGE_LAYOUT_GENERAL, history_layers), + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, {}, before); + const VkImageCopy copy{ + .srcSubresource = output_layers, + .srcOffset = {0, 0, 0}, + .dstSubresource = history_layers, + .dstOffset = {0, 0, 0}, + .extent = {extent.width, extent.height, 1}, + }; + cmdbuf.CopyImage(output, VK_IMAGE_LAYOUT_GENERAL, history, VK_IMAGE_LAYOUT_GENERAL, copy); + const std::array after{ + barrier(output, VK_ACCESS_TRANSFER_READ_BIT, + VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_TRANSFER_READ_BIT, + VK_IMAGE_LAYOUT_GENERAL, output_layers), + barrier(history, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT, + VK_IMAGE_LAYOUT_GENERAL, history_layers), + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, + {}, {}, after); + }); + return true; +} + void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& src, const Region2D& dst_region, const Region2D& src_region, Tegra::Engines::Fermi2D::Filter filter, diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 43218d6ad3..25a6ce7c1e 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -87,6 +87,12 @@ public: Tegra::Engines::Fermi2D::Filter filter, Tegra::Engines::Fermi2D::Operation operation); + /// Stands in for a DLSS evaluation: temporal upscale of input_color into dst, accumulated with + /// a copy of dst's previous contents (the last output). False when dst cannot take it. + bool TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& input_color, + ImageView& motion_vectors, std::array jitter, + std::array motion_scale); + void CopyImage(Image& dst, Image& src, std::span copies); void CopyImageMSAA(Image& dst, Image& src, std::span copies); @@ -169,6 +175,15 @@ public: StagingBufferPool& staging_buffer_pool; BlitImageHelper& blit_image_helper; RenderPassCache& render_pass_cache; + + /// TemporalUpscale's history: its previous result, copied right after drawing it. + struct TemporalHistory { + vk::Image image; + vk::ImageView view; + VkFormat format{}; + VkExtent2D extent{}; + } temporal_history; + std::optional astc_decoder_pass; std::optional bc_encoder_pass; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 33ca051482..32775b8719 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -956,6 +956,73 @@ bool TextureCache

::BlitImage(const Tegra::Engines::Fermi2D::Surface& dst, return true; } +template +bool TextureCache

::TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array jitter, std::array motion_scale) { + if constexpr (FRAMEBUFFER_BLITS) { + return false; + } else { + const Tegra::Engines::Fermi2D::Config copy{ + .operation = Tegra::Engines::Fermi2D::Operation::SrcCopy, + .filter = Tegra::Engines::Fermi2D::Filter::Bilinear, + .must_accelerate = true, + .dst_x0 = 0, + .dst_y0 = 0, + .dst_x1 = static_cast(dst.width), + .dst_y1 = static_cast(dst.height), + .src_x0 = 0, + .src_y0 = 0, + .src_x1 = static_cast(src.width), + .src_y1 = static_cast(src.height), + }; + const auto result = GetBlitImages(dst, src, copy); + if (!result) { + return false; + } + const ImageInfo motion_info(motion_vectors); + has_deleted_images = false; + const ImageId motion_id = + FindOrInsertImage(motion_info, motion_vectors.Address(), RelaxedOptions{}); + if (has_deleted_images || !motion_id) { + return false; // the colour and output ids may be stale; the caller blits instead + } + const ImageId dst_id = result->dst_id; + const ImageId src_id = result->src_id; + PrepareImage(src_id, false, false); + PrepareImage(motion_id, false, false); + PrepareImage(dst_id, true, false); + for (const ImageId id : {src_id, motion_id, dst_id}) { + if (True(slot_images[id].flags & ImageFlagBits::Rescaled) || + slot_images[id].info.num_samples != 1) { + return false; + } + } + const auto view = [&](ImageId id, const Tegra::Engines::Fermi2D::Surface& surface, + PixelFormat format) -> std::optional> { + const std::optional base = slot_images[id].TryFindBase(surface.Address()); + if (!base) { + return std::nullopt; + } + const SubresourceRange range{.base = *base, .extent = {1, 1}}; + return RenderTargetFromImage(id, ImageViewInfo(ImageViewType::e2D, format, range)); + }; + const auto src_view = view(src_id, src, result->src_format); + const auto motion_view = view(motion_id, motion_vectors, motion_info.format); + const auto dst_view = view(dst_id, dst, result->dst_format); + if (!src_view || !motion_view || !dst_view) { + return false; + } + // After the views: creating them may reallocate the framebuffer slots. + Framebuffer* const dst_framebuffer = &slot_framebuffers[dst_view->first]; + return runtime.TemporalUpscale(dst_framebuffer, slot_image_views[dst_view->second], + slot_image_views[src_view->second], + slot_image_views[motion_view->second], jitter, + motion_scale); + } +} + template std::pair TextureCache

::TryFindFramebufferImageView( const Tegra::FramebufferConfig& config, DAddr cpu_addr) { diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 141e5b76d5..d4b0f597ae 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -276,6 +276,13 @@ public: const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Config& copy); + /// Stand-in for a DLSS evaluation: temporal upscale of src into dst (see the runtime's + /// TemporalUpscale). False when the images cannot take it; the caller then blits. + bool TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array jitter, std::array motion_scale); + /// Try to find a cached image view in the given CPU address [[nodiscard]] std::pair TryFindFramebufferImageView( const Tegra::FramebufferConfig& config, DAddr cpu_addr); -- cgit v1.2.3 From a407703f6d29a5ab73328b4ededb12cf29b2758d Mon Sep 17 00:00:00 2001 From: Whovian64 <86082882+whovian64@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:45:39 -0500 Subject: Linux and AMD: BC3 flashing on RADV, ASTC decode stalls, lower RAM usage, a buffer cache crash (#260) * host_memory: punch holes in the backing on Linux instead of zero filling freed memory Freeing guest memory zero filled the memfd backing with memset, which commits every page it touches. fallocate(PUNCH_HOLE) hands the pages back to the kernel instead, and they read as zero afterwards. Only the partial pages at the ends are still cleared by hand. Mario Kart World on Linux (RX 9070 XT, 31 GB): peak RSS 23.1 -> 15.9 GB. EDEN_PUNCH_HOLE=0 restores the memset. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * fiber: don't zero fill the stacks, untouched pages stay unbacked The value initialised stack arrays were zero filled when each fiber was made, which commits the whole stack. Left uninitialised, the pages a fiber never touches stay unbacked. Together with the descriptor ring change, about 1.2 GB less RSS in Mario Kart World. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * vulkan: don't zero fill the descriptor update ring Same as the fiber stacks: make_unique value initialised the whole ring for every frame in flight. Entries are always written before they are read, so allocate it without initialising it. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * vulkan: wait for the GPU after an ASTC transcode decode like the other ASTC decodes TranscodeAstcToBcn (astc_recompression Bc1/Bc3 with GPU decode) was the only ASTC decode recorded with wait = false. On RADV (RX 9070 XT, Mesa) Mario Kart World with BC3 flashed heavily all over the screen, while the uncompressed GPU decode, which waits, did not. CPU decode with BC3 does not flash either. Waiting after the transcode removes the flashing; full pipeline barriers around its copy did not, so the wait hides a race somewhere else rather than fixing one in the transcode itself. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * vulkan: submit after an ASTC decode instead of waiting for the GPU by default Every ASTC decode ended with scheduler.Finish(). Over a 19 minute Mario Kart World Grand Prix that stalled the GPU thread 30.7 s in total, up to 1.2 s out of every 5 s while new textures stream in (track sections, race starts), which is where the hitches were. Flushing keeps the ordering that stops the BC3 flashing on RADV without the CPU waiting: no flashing in play, and frames over 100 ms went from dozens to 3 in about 7 minutes. EDEN_NX2_ASTC_FLUSH=0 restores the wait. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * texture_cache: over the expected use, evict costly images unused for 1800 frames first The collector never deletes CostlyLoad images outside aggressive mode, so once a cup moves on, the previous tracks' textures stay resident and the cache sits over its expected use deleting the cheap images still in use. When over the expected use, costly images unused for 1800 frames (half a minute) now go first. Mario Kart World Grand Prix on an RX 9070 XT: frames over 20 ms 8.8% -> 4.0%, 58.7 fps average. EDEN_NX2_TC_OLD_AGE= sets the age, 0 turns it off. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * buffer_cache: no buffer for device addresses past the page table FindBuffer indexed page_table, which covers 2^36 bytes, with any address. Mario Kart World crashed after about 4.5 minutes with a uniform buffer at 0x72253b2f2890 (FindBuffer <- UpdateUniformBuffers <- DrawEnd from a macro), reading far past the table. Treat such an address as no buffer. Where that address comes from is still unknown; this only stops the crash. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) --- src/common/fiber.cpp | 4 ++-- src/common/host_memory.cpp | 22 ++++++++++++++++++++++ src/video_core/buffer_cache/buffer_cache.h | 4 ++-- src/video_core/renderer_vulkan/vk_compute_pass.cpp | 6 +++--- .../renderer_vulkan/vk_texture_cache.cpp | 2 +- .../renderer_vulkan/vk_update_descriptor.cpp | 8 ++++---- .../renderer_vulkan/vk_update_descriptor.h | 3 ++- src/video_core/texture_cache/texture_cache.h | 16 ++++++++++++++-- 8 files changed, 50 insertions(+), 15 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/common/fiber.cpp b/src/common/fiber.cpp index 0e7f28c5b9..08558a0026 100644 --- a/src/common/fiber.cpp +++ b/src/common/fiber.cpp @@ -25,8 +25,8 @@ struct Fiber::FiberImpl { FiberImpl() {} u32 canary_1 = CANARY_VALUE; - std::array stack{}; - std::array rewind_stack{}; + std::array stack; + std::array rewind_stack; u32 canary_2 = CANARY_VALUE; boost::context::detail::fcontext_t context{}; diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 75a9da80e9..147eba8477 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -51,6 +51,7 @@ #endif // ^^^ POSIX ^^^ +#include #include #include @@ -517,6 +518,14 @@ public: , virtual_size{virtual_size_} {} +#ifdef __linux__ + bool PunchHole(size_t offset, size_t length) { + return fd >= 0 && + fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, static_cast(offset), + static_cast(length)) == 0; + } +#endif + bool Init() { long page_size = sysconf(_SC_PAGESIZE); ASSERT_MSG(page_size == 0x1000, "page size {:#x} is incompatible with 4K paging", page_size); @@ -791,6 +800,19 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, MemoryPermission } void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value) { +#ifdef __linux__ + const size_t first = AlignUp(physical_offset, PageAlignment); + const size_t last = AlignDown(physical_offset + length, PageAlignment); + static const bool punch = [] { + const char* value = std::getenv("EDEN_PUNCH_HOLE"); + return value == nullptr || value[0] != '0'; + }(); + if (punch && fill_value == 0 && impl && last > first && impl->PunchHole(first, last - first)) { + std::memset(backing_base + physical_offset, 0, first - physical_offset); + std::memset(backing_base + last, 0, physical_offset + length - last); + return; + } +#endif std::memset(backing_base + physical_offset, fill_value, length); } diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e09a435dcc..a3358048dc 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1651,10 +1651,10 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 template BufferId BufferCache

::FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible) { - if (device_addr == 0) { + const u64 page = device_addr >> CACHING_PAGEBITS; + if (device_addr == 0 || page >= page_table.size()) { return NULL_BUFFER_ID; } - const u64 page = device_addr >> CACHING_PAGEBITS; const BufferId buffer_id = page_table[page]; if (buffer_id) { Buffer& buffer = slot_buffers[buffer_id]; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index 691be728d2..cb8dcde42f 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -801,11 +801,11 @@ void ASTCDecoderPass::AssembleInto(const VideoCommon::ImageInfo& info, u32 skip, cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier); }); - // Each decode ends with the GPU thread waiting for the host GPU to go idle. Measured here - // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=1 submits without waiting instead. + // Each decode ends with a submit to the host GPU. Measured here + // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=0 waits for the host GPU to go idle instead. static const bool flush_only = [] { const char* value = std::getenv("EDEN_NX2_ASTC_FLUSH"); - return value != nullptr && value[0] == '1'; + return value == nullptr || value[0] != '0'; }(); if (!target.wait) { return; diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 4e428229f6..b0e5eacb32 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -4267,7 +4267,7 @@ void TextureCacheRuntime::TranscodeAstcToBcn( .aspect_mask = VK_IMAGE_ASPECT_COLOR_BIT, .initialized = false, .level_view = [&pending](s32 level) { return *pending.views[level]; }, - .wait = false, + .wait = true, }, map, swizzles); diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp index 742a8d9969..29f3ee71a1 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp @@ -20,10 +20,10 @@ UpdateDescriptorQueue::UpdateDescriptorQueue(const Device& device_, size_t frame bool supports_descriptor_buffer_) : device{device_}, frame_payload_size{frame_payload_size_}, supports_descriptor_buffer{supports_descriptor_buffer_}, - payload(frame_payload_size_ * FRAMES_IN_FLIGHT) + payload(std::make_unique_for_overwrite(frame_payload_size_ * FRAMES_IN_FLIGHT)) { - payload_start = payload.data(); - payload_cursor = payload.data(); + payload_start = payload.get(); + payload_cursor = payload.get(); } UpdateDescriptorQueue::~UpdateDescriptorQueue() = default; @@ -32,7 +32,7 @@ void UpdateDescriptorQueue::TickFrame() { if (++frame_index >= FRAMES_IN_FLIGHT) { frame_index = 0; } - payload_start = payload.data() + frame_index * frame_payload_size; + payload_start = payload.get() + frame_index * frame_payload_size; payload_cursor = payload_start; } diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.h b/src/video_core/renderer_vulkan/vk_update_descriptor.h index 9f30e65dbd..4d393376b4 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.h +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -122,7 +123,7 @@ private: DescriptorUpdateEntry* payload_cursor = nullptr; DescriptorUpdateEntry* payload_start = nullptr; const DescriptorUpdateEntry* upload_start = nullptr; - std::vector payload; + std::unique_ptr payload; }; // TODO: should these be separate classes instead? diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 32775b8719..55ec4c64df 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -125,6 +125,7 @@ template void TextureCache

::RunGarbageCollector() { bool high_priority_mode = false; bool aggressive_mode = false; + bool evict_old_costly = false; u64 ticks_to_destroy = 0; size_t num_iterations = 0; const auto Configure = [&](bool allow_aggressive) { @@ -139,7 +140,8 @@ void TextureCache

::RunGarbageCollector() { ticks_to_destroy = (std::max)(ticks_to_destroy, min_age); num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10); }; - const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) { + const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode, + &evict_old_costly](ImageId image_id) { if (num_iterations == 0) { return true; } @@ -149,7 +151,8 @@ void TextureCache

::RunGarbageCollector() { return false; } const bool must_download = IsDownloadable(image) && False(image.flags & ImageFlagBits::BadOverlap); - if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) { + if ((!aggressive_mode && !evict_old_costly && True(image.flags & ImageFlagBits::CostlyLoad)) || + (!high_priority_mode && must_download)) { return false; } if (must_download) { @@ -176,6 +179,15 @@ void TextureCache

::RunGarbageCollector() { return false; }; Configure(false); + static const u64 old_age = [] { + const char* value = std::getenv("EDEN_NX2_TC_OLD_AGE"); + return value != nullptr ? std::strtoull(value, nullptr, 10) : 1800ULL; + }(); + if (high_priority_mode && old_age != 0 && frame_tick > old_age) { + evict_old_costly = true; + lru_cache.ForEachItemBelow(frame_tick - old_age, Cleanup); + evict_old_costly = false; + } lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); if (total_used_memory >= critical_memory) { Configure(true); -- cgit v1.2.3