From 4d8a9629a6f49f57adfa420a7196e5967677f10f Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:30:46 -0400 Subject: texture_cache: collect from 5 GiB of device memory on every GPU The collector's thresholds follow the card's size and budget (critical at budget - 768 MiB), so the cache keeps textures of earlier scenes until the card is nearly full: Mario Kart World used 6.7 GB of an 8 GB card, and more on larger cards. Cap the critical threshold at 5 GiB (and expected/minimum below it) whatever the card. Same build, back to back: peak device memory 6.7 -> 5.6 GB, frames over 100 ms 114 -> 64 in six minutes, average frame time unchanged. The EDEN_NX2_TC_*_MIB overrides still apply. --- src/video_core/texture_cache/texture_cache.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 63abffed0f..bd5f7debfb 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -185,6 +185,14 @@ void TextureCache

::RunGarbageCollector() { template void TextureCache

::ApplyThresholdOverrides(bool log) { + // On every GPU: past what a frame needs, more device memory only keeps the textures of earlier + // scenes (title 1 filled 6.7 GB of an 8 GB card and draws its races in ~5 GB), so collect from + // MaxCriticalMemory on rather than near the card's budget. + critical_memory = (std::min)(critical_memory, MaxCriticalMemory); + expected_memory = + (std::min)(expected_memory, critical_memory - (std::min)(critical_memory, u64{768_MiB})); + minimum_memory = + (std::min)(minimum_memory, expected_memory - (std::min)(expected_memory, u64{1_GiB})); // EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB override the collector's thresholds. for (const auto& [name, target] : {std::pair{"EDEN_NX2_TC_EXPECTED_MIB", &expected_memory}, std::pair{"EDEN_NX2_TC_CRITICAL_MIB", &critical_memory}}) { -- cgit v1.2.3 From 049c9633edd9b3f7eaed6b4b3a18a4ead0cd18b7 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:31:41 -0400 Subject: nx2: read texture handles that hold a header byte offset Switch Sports' scenery shaders take their 64-bit texture handles from a uniform register, and the texture part of those handles is the header's byte offset in the pool (index * 32), not its index: 0x96e46000 names header 8960 (a 640x152 2D texture), while the plain reading, 286720, runs far past the 33067-entry pool. Eden skipped them at draw time (null view) and built the shaders for a garbage header, so the island's ground, cliffs and buildings drew black. When a handle's index falls past the pool but index / 32 lands inside it, take that header, both when building the shader and when binding the view. Handles that already resolve keep their meaning. --- src/video_core/shader_environment.cpp | 5 +++-- src/video_core/texture_cache/texture_cache.h | 1 + src/video_core/textures/texture.h | 11 +++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp index 9d189ad4f0..532b1f97f4 100644 --- a/src/video_core/shader_environment.cpp +++ b/src/video_core/shader_environment.cpp @@ -315,8 +315,9 @@ std::optional GenericEnvironment::TryFindSize() { Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw) { const auto handle{Tegra::Texture::TexturePair(raw, via_header_index)}; - ASSERT(handle.first <= tic_limit); - const GPUVAddr descriptor_addr{tic_addr + handle.first * sizeof(Tegra::Texture::TICEntry)}; + const u32 index{Tegra::Texture::TextureHeaderIndex(handle.first, tic_limit)}; + ASSERT(index <= tic_limit); + const GPUVAddr descriptor_addr{tic_addr + index * sizeof(Tegra::Texture::TICEntry)}; Tegra::Texture::TICEntry entry; gpu_memory->ReadBlock(descriptor_addr, &entry, sizeof(entry)); return entry; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 63abffed0f..b0fa8ac1ce 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -673,6 +673,7 @@ typename P::Framebuffer* TextureCache

::GetFramebuffer() { template ImageViewId TextureCache

::VisitImageView(u32 index, bool compute) { auto& table = compute ? channel_state->compute_image_table : channel_state->graphics_image_table; + index = Tegra::Texture::TextureHeaderIndex(index, table.current_limit); if (index > table.current_limit) { LOG_DEBUG(HW_GPU, "Invalid image view index={}", index); return NULL_IMAGE_VIEW_ID; diff --git a/src/video_core/textures/texture.h b/src/video_core/textures/texture.h index 9fc6a631bd..79396d8a68 100644 --- a/src/video_core/textures/texture.h +++ b/src/video_core/textures/texture.h @@ -165,6 +165,17 @@ static_assert(sizeof(TextureHandle) == 4, "TextureHandle has wrong size"); } } +/// Some Switch 2 bindless handles (Switch Sports' scenery, read from a uniform register) hold +/// the texture header's byte offset in the pool, index * 32, where the handle normally has the +/// index. Read it as an offset only when the plain index falls past the pool and the offset +/// lands inside it, so handles that already resolve are left alone. +[[nodiscard]] constexpr u32 TextureHeaderIndex(u32 index, u32 limit) { + if (index > limit && index % 32 == 0 && index / 32 <= limit) { + return index / 32; + } + return index; +} + struct TICEntry { union { struct { -- cgit v1.2.3 From 75385e994dc8cc8699a918180ee566e8723b8394 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:05:55 -0400 Subject: texture_cache: don't count deleted images still waiting to be destroyed The collector deletes images into a ring that destroys them TICKS_TO_DESTROY frames later, and until then the driver still counts their memory. Reading the driver's usage every frame, the collector saw no drop after deleting and kept deleting 40+ images a frame in aggressive mode long after it had freed enough: once a session reached the critical threshold, frames took over 100 ms for seconds at a time. Keep the bytes deleted into each slot of the ring and leave them out of the reported usage until the ring destroys them. --- src/video_core/texture_cache/texture_cache.h | 11 +++++++++++ src/video_core/texture_cache/texture_cache_base.h | 4 ++++ 2 files changed, 15 insertions(+) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index c8aaf8dff5..71f89db367 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -250,6 +250,12 @@ void TextureCache

::TickFrame() { // If we can obtain the memory info, use it instead of the estimate. if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); + // Images the collector deleted keep their memory for TICKS_TO_DESTROY frames: counting + // it kept the collector in aggressive mode, deleting 40+ images a frame for frames after + // it had freed enough (title 1: frames over 100 ms for seconds at a time). + for (const u64 bytes : sentenced_image_bytes) { + total_used_memory -= (std::min)(total_used_memory, bytes); + } } // Every 600 frames: device memory in use against the collector's thresholds, and what the // images themselves account for (VRAM pressure makes the collector thrash). @@ -316,6 +322,8 @@ void TextureCache

::TickFrame() { RunGarbageCollector(); } sentenced_images.Tick(); + sentenced_image_index = (sentenced_image_index + 1) % TICKS_TO_DESTROY; + sentenced_image_bytes[sentenced_image_index] = 0; sentenced_framebuffers.Tick(); sentenced_image_view.Tick(); TickAsyncDecode(); @@ -2832,6 +2840,8 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { } // TickFrame sets the total from the driver, which can be under the sum of these estimates. total_used_memory -= (std::min)(total_used_memory, ImageMemoryBytes(slot_images[image_id])); + const u64 freed_bytes = (image.HasScaled() ? GetScaledImageSizeBytes(image) : 0) + + ImageMemoryBytes(slot_images[image_id]); const GPUVAddr gpu_addr = image.gpu_addr; const auto alloc_it = image_allocs_table.find(gpu_addr); if (alloc_it == image_allocs_table.end()) { @@ -2897,6 +2907,7 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value()); } else if (!immediate_delete) { sentenced_images.Push(std::move(slot_images[image_id])); + sentenced_image_bytes[sentenced_image_index] += freed_bytes; } slot_images.erase(image_id); diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 94102cd261..d51eefeee8 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -552,6 +552,10 @@ private: static constexpr size_t TICKS_TO_DESTROY = 8; #endif DelayedDestructionRing sentenced_images; + /// Device memory of the images in each slot of sentenced_images (same index): deleted, but + /// counted in the driver's usage until the ring destroys them. + std::array sentenced_image_bytes{}; + size_t sentenced_image_index = 0; DelayedDestructionRing sentenced_image_view; DelayedDestructionRing sentenced_framebuffers; -- cgit v1.2.3 From dbd9572a5a03b1e0bb00946750cf866420a25596 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:04:22 -0400 Subject: texture_cache: delete images without walking every channel's tables Deleting an image stamped every entry of every channel's view-id map with CORRUPT_ID (validation, which is on in release builds) and cleared every channel's descriptor table bitmaps on the spot. On Mario Kart World's title screen (~5,400 entries in 5 channels) that took 362 us per deleted image, with dozens deleted a frame while the collector runs: ~10% of the time in its frames over 100 ms. The stamp is debug-only now. The tables are invalidated along with it, so every entry is written again before it is read. Invalidate() marks a table and its bitmap is cleared on the next read, once however many images were deleted in between. Same title screen: 3.7 us per deleted image. --- src/video_core/texture_cache/descriptor_table.h | 10 +++++++++- src/video_core/texture_cache/texture_cache.h | 10 ++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/descriptor_table.h b/src/video_core/texture_cache/descriptor_table.h index e40c128ab5..75383809c5 100644 --- a/src/video_core/texture_cache/descriptor_table.h +++ b/src/video_core/texture_cache/descriptor_table.h @@ -30,11 +30,17 @@ public: } void Invalidate() noexcept { - std::ranges::fill(read_descriptors, 0); + // Cleared on the next read: the texture cache invalidates every channel's tables for + // each image it deletes, dozens of times a frame while its collector runs. + invalidated = true; } [[nodiscard]] std::pair Read(Tegra::MemoryManager const& gpu_memory, u32 index) noexcept { DEBUG_ASSERT(index <= current_limit); + if (invalidated) [[unlikely]] { + std::ranges::fill(read_descriptors, 0); + invalidated = false; + } const GPUVAddr gpu_addr = current_gpu_addr + index * sizeof(T); std::pair result; gpu_memory.ReadBlockUnsafe(gpu_addr, std::addressof(result.first), sizeof(T)); @@ -60,6 +66,7 @@ public: read_descriptors.resize(Common::DivCeil(num_descriptors, 64U)); old_size = (std::min)(old_size, read_descriptors.size()); std::fill(read_descriptors.begin(), read_descriptors.begin() + old_size, 0); + invalidated = false; // descriptors.resize(num_descriptors); } @@ -68,6 +75,7 @@ public: std::vector descriptors; GPUVAddr current_gpu_addr{}; u32 current_limit{}; + bool invalidated{}; ///< read_descriptors is to be cleared before the next read }; } // namespace VideoCommon diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index c8aaf8dff5..559423dbee 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -1712,9 +1712,14 @@ void TextureCache

::InvalidateScale(Image& image) { for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; +#ifndef NDEBUG + // A walk over every entry (title 1: ~5,400 in 5 channels) for each deleted image: debug + // builds only. The tables are invalidated below, so each entry is written again before + // it is read. if constexpr (ENABLE_VALIDATION) for (auto& e : channel_info.image_view_ids) e.second = CORRUPT_ID; +#endif channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } @@ -2906,9 +2911,14 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { } for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; +#ifndef NDEBUG + // A walk over every entry (title 1: ~5,400 in 5 channels) for each deleted image: debug + // builds only. The tables are invalidated below, so each entry is written again before + // it is read. if constexpr (ENABLE_VALIDATION) for (auto& e : channel_info.image_view_ids) e.second = CORRUPT_ID; +#endif channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } -- cgit v1.2.3 From 446c34aa8b305deb9f0870721d2126da8c186368 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:24:25 -0400 Subject: texture_cache: free deleted images once the GPU is past them Deleted images kept their device memory for TICKS_TO_DESTROY (8) frames. While the collector deletes dozens a frame that held 1-1.5 GB of VRAM on top of its threshold, which the collector does not count (Mario Kart World on an 8 GB card: 6.6-7.0 GB in use against 5 GiB). On runtimes that report GPU progress (Vulkan) an image is destroyed as soon as the GPU is past it. Title screen, 10 minutes: VRAM in use 6.75 -> 6.34 GB on average, frames over 100 ms 22 -> 15. (cherry picked from commit 8c899541baf398deb892a857a932590f32d2aa4f) --- src/video_core/renderer_vulkan/vk_texture_cache.cpp | 8 ++++++++ src/video_core/renderer_vulkan/vk_texture_cache.h | 5 +++++ src/video_core/texture_cache/texture_cache.h | 18 ++++++++++++++++-- src/video_core/texture_cache/texture_cache_base.h | 11 +++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 5ba36feee9..b8d2e4edd4 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -1979,6 +1979,14 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 TextureCacheRuntime::CurrentTick() const noexcept { + return scheduler.CurrentTick(); +} + +bool TextureCacheRuntime::IsFree(u64 tick) const noexcept { + return scheduler.IsFree(tick); +} + u64 TextureCacheRuntime::GetDeviceMemoryBudget() const { return device.GetDeviceMemoryBudget(); } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 3944e183fd..49eba2d7d3 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -67,6 +67,11 @@ public: u64 GetDeviceMemoryUsage() const; + /// GPU progress: the texture cache frees a deleted image once the GPU is past its last use. + [[nodiscard]] u64 CurrentTick() const noexcept; + + [[nodiscard]] bool IsFree(u64 tick) const noexcept; + u64 GetDeviceMemoryBudget() const; bool CanReportMemoryUsage() const; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 71f89db367..46ebf6a25f 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -247,6 +247,13 @@ void TextureCache

::TickFrame() { } } } + if constexpr (requires { runtime.CurrentTick(); }) { + while (!gpu_sentenced_images.empty() && + runtime.IsFree(gpu_sentenced_images.front().tick)) { + gpu_sentenced_bytes -= gpu_sentenced_images.front().bytes; + gpu_sentenced_images.pop_front(); + } + } // If we can obtain the memory info, use it instead of the estimate. if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); @@ -256,6 +263,7 @@ void TextureCache

::TickFrame() { for (const u64 bytes : sentenced_image_bytes) { total_used_memory -= (std::min)(total_used_memory, bytes); } + total_used_memory -= (std::min)(total_used_memory, gpu_sentenced_bytes); } // Every 600 frames: device memory in use against the collector's thresholds, and what the // images themselves account for (VRAM pressure makes the collector thrash). @@ -2906,8 +2914,14 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { if (retained) { LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value()); } else if (!immediate_delete) { - sentenced_images.Push(std::move(slot_images[image_id])); - sentenced_image_bytes[sentenced_image_index] += freed_bytes; + if constexpr (requires { runtime.CurrentTick(); }) { + gpu_sentenced_images.push_back( + {runtime.CurrentTick(), freed_bytes, std::move(slot_images[image_id])}); + gpu_sentenced_bytes += freed_bytes; + } else { + sentenced_images.Push(std::move(slot_images[image_id])); + sentenced_image_bytes[sentenced_image_index] += freed_bytes; + } } slot_images.erase(image_id); diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index d51eefeee8..151dd9bfa1 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -556,6 +556,17 @@ private: /// counted in the driver's usage until the ring destroys them. std::array sentenced_image_bytes{}; size_t sentenced_image_index = 0; + /// Deleted images on runtimes that report GPU progress (CurrentTick, IsFree): destroyed, with + /// their device memory, as soon as the GPU is past them rather than TICKS_TO_DESTROY frames + /// later. While the collector deletes dozens a frame, eight frames of them held 1-1.5 GB of + /// VRAM above its threshold (title 1 on an 8 GB card: 6.6-7.0 GB in use against 5 GiB). + struct GpuSentencedImage { + u64 tick; + u64 bytes; + Image image; + }; + std::deque gpu_sentenced_images; + u64 gpu_sentenced_bytes{}; DelayedDestructionRing sentenced_image_view; DelayedDestructionRing sentenced_framebuffers; -- cgit v1.2.3 From 3871de9da994f9c0fa581ecd993db40081ce6b48 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:24:28 -0400 Subject: texture_cache: find a deleted image's view entries without a scan Deleting an image erased the channels' descriptor -> view entries pointing to its views by scanning every channel's map (Mario Kart World: ~5,000 entries, 50-130 us per deleted image, dozens a frame while the collector runs). The cache keeps, per view, the (channel, descriptor) entries that point to it and erases just those, skipping entries of channels that are gone or that point to another view since. (cherry picked from commit c3d22e8e09dacf39000c742d03dfc7517bd6f343) --- src/video_core/texture_cache/texture_cache.h | 31 +++++++++++++++-------- src/video_core/texture_cache/texture_cache_base.h | 6 +++++ 2 files changed, 27 insertions(+), 10 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index c8aaf8dff5..279668813f 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -689,8 +689,13 @@ ImageViewId TextureCache

::VisitImageView(u32 index, bool compute) { if (IsValidEntry(*gpu_memory, descriptor)) { // Is new (registered view) on the texture cache side? const auto [pair, is_new_tc] = channel_state->image_views.try_emplace(descriptor); - if (is_new_tc) + if (is_new_tc) { pair->second = CreateImageView(descriptor); + if (pair->second != NULL_IMAGE_VIEW_ID) { + image_view_descriptors[pair->second].emplace_back(current_channel_id, + descriptor); + } + } PrepareImageView(pair->second, false, false); channel_state->image_view_ids.insert_or_assign(map_index, pair->second); return pair->second; @@ -2917,17 +2922,23 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { template void TextureCache

::RemoveImageViewReferences(std::span removed_views) { - for (size_t c : active_channel_ids) { - auto& channel_info = channel_storage[c]; - auto it = channel_info.image_views.begin(); - while (it != channel_info.image_views.end()) { - const auto found = std::ranges::find(removed_views, it->second); - if (found != removed_views.end()) { - it = channel_info.image_views.erase(it); - } else { - ++it; + for (const ImageViewId removed : removed_views) { + const auto refs = image_view_descriptors.find(removed); + if (refs == image_view_descriptors.end()) { + continue; + } + for (const auto& [channel, descriptor] : refs->second) { + // The channel may be gone, or its entry point to another view since. + if (std::ranges::find(active_channel_ids, channel) == active_channel_ids.end()) { + continue; + } + auto& image_views = channel_storage[channel].image_views; + if (const auto it = image_views.find(descriptor); + it != image_views.end() && it->second == removed) { + image_views.erase(it); } } + image_view_descriptors.erase(refs); } } diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 94102cd261..628a5d8558 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -433,6 +433,12 @@ private: /// Remove image views references from the cache void RemoveImageViewReferences(std::span removed_views); + /// Per image view, the (channel, descriptor) entries of the channels' image_views maps that + /// point to it. Deleting an image erased them by scanning every map (title 1: ~5,000 entries, + /// 50-130 us for each deleted image, dozens a frame while the collector runs). + std::unordered_map, 2>> + image_view_descriptors; + /// Remove framebuffers using the given image views from the cache void RemoveFramebuffers(std::span removed_views); -- cgit v1.2.3 From e2c512916c6f5042d030fcf1708043233f8df08e Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:30:52 -0400 Subject: texture_cache: mark guest memory textures were uploaded from cold Once a texture is uploaded the host GPU has its own copy, and the guest rarely reads that memory again, but it stayed in the emulator's working set. Every 600 frames the guest ranges of images uploaded since (64 KiB or more, not GPU-written or sparse) are marked cold through HostMemory::HintCold, backing and CPU view, from a low-priority worker, so the OS pages them out before anything else when it needs RAM. Their contents stay; using them brings them back. Mario Kart World's title screen at ~10 minutes: game memory in RAM 4.8-5.5 -> 3.6 GB, the emulator's real RAM 10.6-11.9 -> 9.4 GB, frames over 100 ms not worse. (cherry picked from commit ea36b2c5ca352f8c8cdcc829a2d3bde9bdc06ece) --- src/common/host_memory.cpp | 2 +- src/common/host_memory.h | 3 +- src/core/device_memory_manager.h | 7 ++++ src/core/device_memory_manager.inc | 44 +++++++++++++++++++++++ src/video_core/texture_cache/texture_cache.h | 14 ++++++++ src/video_core/texture_cache/texture_cache_base.h | 2 ++ 6 files changed, 70 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index c0e7160dd4..2c45dfd3a5 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -791,7 +791,7 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, MemoryPermission } void HostMemory::HintCold(size_t physical_offset, size_t length, - std::optional virtual_offset) { + std::optional virtual_offset) const { if (length == 0 || backing_base == nullptr) { return; } diff --git a/src/common/host_memory.h b/src/common/host_memory.h index fe0938103c..4d8d15cf58 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -57,7 +57,8 @@ public: /// Memory written once and read again rarely: its pages leave the working set (backing and, /// when given, its view at virtual_offset), so the OS compresses or pages them out first when /// it needs RAM. The contents stay; using the pages brings them back. - void HintCold(size_t physical_offset, size_t length, std::optional virtual_offset); + void HintCold(size_t physical_offset, size_t length, + std::optional virtual_offset) const; [[nodiscard]] u8* BackingBasePointer() noexcept { return backing_base; diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 9c034f01a1..05a8136169 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -18,6 +18,7 @@ #include "common/common_types.h" #include "common/range_mutex.h" #include "common/scratch_buffer.h" +#include "common/thread_worker.h" #include "common/sparse_large_vector.h" namespace Core { @@ -53,6 +54,10 @@ public: void BindInterface(DeviceInterface* device_inter); + /// Marks the memory behind [address, address + size) cold (Common::HostMemory::HintCold), + /// from a low-priority thread: data the host GPU has its own copy of, e.g. uploaded textures. + void HintCold(DAddr address, size_t size); + DAddr Allocate(size_t size); void AllocateFixed(DAddr start, size_t size); void Free(DAddr start, size_t size); @@ -194,7 +199,9 @@ private: std::unique_ptr> impl; const uintptr_t physical_base; + const DeviceMemory* device_memory_ptr; DeviceInterface* device_inter; + std::unique_ptr cold_hint_worker; struct TrackedEntry { VAddr cpu_backing_address; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index c45bc71656..77ab95e07e 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -171,6 +171,7 @@ struct DeviceMemoryManagerAllocator { template DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memory_) : physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())} + , device_memory_ptr{&device_memory_} , device_inter{nullptr} , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS)) , tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS) @@ -182,6 +183,49 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo template DeviceMemoryManager::~DeviceMemoryManager() = default; +template +void DeviceMemoryManager::HintCold(DAddr address, size_t size) { + struct Run { + u64 physical_offset; + VAddr virtual_address; ///< 0: no CPU mapping known + u64 size; + }; + std::vector runs; + const size_t page_end = Common::DivCeil(address + size, Memory::YUZU_PAGESIZE); + for (size_t page = address >> Memory::YUZU_PAGEBITS; page < page_end; ++page) { + const u32 compressed = tracked_entries[page].compressed_physical_ptr; + if (compressed == 0) { + continue; + } + const u64 physical = u64{compressed - 1} << Memory::YUZU_PAGEBITS; + const VAddr vaddr = ExtractCPUBacking(page).second; + if (!runs.empty()) { + Run& last = runs.back(); + const bool contiguous_view = + last.virtual_address == 0 ? vaddr == 0 : last.virtual_address + last.size == vaddr; + if (last.physical_offset + last.size == physical && contiguous_view) { + last.size += Memory::YUZU_PAGESIZE; + continue; + } + } + runs.push_back({physical, vaddr, Memory::YUZU_PAGESIZE}); + } + if (runs.empty()) { + return; + } + if (!cold_hint_worker) { + cold_hint_worker = std::make_unique(1, "DeviceMemory:ColdHint"); + } + cold_hint_worker->QueueWork([this, runs = std::move(runs)] { + for (const Run& run : runs) { + device_memory_ptr->buffer.HintCold( + run.physical_offset, run.size, + run.virtual_address != 0 ? std::optional{run.virtual_address} + : std::nullopt); + } + }); +} + template void DeviceMemoryManager::BindInterface(DeviceInterface* device_inter_) { device_inter = device_inter_; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 63abffed0f..13637aa265 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -315,6 +315,12 @@ void TextureCache

::TickFrame() { sentenced_image_view.Tick(); TickAsyncDecode(); TickAsyncUnswizzle(); + if (frame_tick % 600 == 0 && !uploaded_ranges.empty()) { + for (const auto& [address, size] : uploaded_ranges) { + device_memory.HintCold(address, size); + } + uploaded_ranges.clear(); + } runtime.TickFrame(); ++frame_tick; @@ -1352,6 +1358,14 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image)); UploadImageContents(image, staging); runtime.InsertUploadMemoryBarrier(); + // The host GPU has its own copy now and the guest rarely reads this memory again: TickFrame + // marks it cold, so the OS pages it out before anything else when it needs RAM (title 1 + // streams several GB of textures through it). + constexpr u64 ColdHintMinBytes = 64_KiB; + if (image.guest_size_bytes >= ColdHintMinBytes && + False(image.flags & (ImageFlagBits::GpuModified | ImageFlagBits::Sparse))) { + uploaded_ranges.emplace_back(image.cpu_addr, image.guest_size_bytes); + } } template diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 11e95f98b3..26d989f2da 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -558,6 +558,8 @@ private: Common::ThreadWorker texture_decode_worker{1, "TextureDecoder", {}, Common::ThreadPlacement::Efficiency}; std::vector> async_decodes; + /// Guest memory images were uploaded from since the last cold hint (device address, size). + std::vector> uploaded_ranges; std::deque unswizzle_queue; u8 current_unswizzle_frame; -- cgit v1.2.3 From 1c794af56407f324eedad99478388ba0c9cf9ad6 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:36:38 -0400 Subject: texture_cache: back large sparse 2D images by sparse block Mario Kart World's nine world-sized masks (R8 16384x16384, RG16 and R16 8192x8192, 170-341 MB each) live in sparse regions the game maps its own memory into, a few MiB of each. Since a mapped tile counts as written, the texture cache backed and uploaded each mask in full: about 2.9 GB of device memory, and a whole-mask upload for every tile the game streamed in. Such images (single layer, one texel per block, colour, in a sparse region, 32 MiB or more) are now unbacked SparseTiles images. The texture cache records the guest ranges written to them; Image::BindTiles gives the host sparse blocks under those ranges memory, and only those rectangles are unswizzled, a row of whole GOB blocks at a time, and uploaded. A new image starts with the tiles the game maps there (nx2:gpu 216 now records them per 64 KiB), or the written blocks where it maps none. Unmapped tiles keep their blocks and are loaded as zeros. Checked against the full unswizzle on 400 ordinary game images (row padding, mip chains, odd sizes): no row differs. Not on AMD drivers, which cannot sample images with unbound memory. Mario Kart World title screen, 150 s, against backing the masks whole: dedicated VRAM 6.6 -> 6.2 GB, shared 1.0 -> 0.3 GB, peak 8.2 -> 6.9 GB. --- src/core/hle/service/nx2_gsv.h | 2 + src/video_core/nx2.h | 60 ++++++ .../renderer_vulkan/vk_texture_cache.cpp | 210 +++++++++++++++++++++ src/video_core/renderer_vulkan/vk_texture_cache.h | 30 +++ src/video_core/texture_cache/image_base.h | 7 + src/video_core/texture_cache/texture_cache.h | 65 +++++++ src/video_core/texture_cache/texture_cache_base.h | 9 + src/video_core/texture_cache/types.h | 9 + src/video_core/texture_cache/util.cpp | 204 ++++++++++++++++++++ src/video_core/texture_cache/util.h | 21 +++ 10 files changed, 617 insertions(+) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h index 44d2b16240..39ef402024 100644 --- a/src/core/hle/service/nx2_gsv.h +++ b/src/core/hle/service/nx2_gsv.h @@ -962,6 +962,7 @@ private: ++m_tile_unmaps; LOG_DEBUG(Service, "nx2:gpu 216 unmap [{:#x}, +{:#x}]", gpu_address, size); if (region != nullptr) { + Tegra::Nx2SparseTilesMapped(start, gpu_address - start, size, false); registry.MapSparseTile(gpu_address, 0, size); MarkTiles(*region, gpu_address - start, size, false); } else { @@ -998,6 +999,7 @@ private: } Tegra::Nx2SparseBlockWritten(start, block); } + Tegra::Nx2SparseTilesMapped(start, gpu_address - start, size, true); registry.MapSparseTile(gpu_address, cpu_address, size); MarkTiles(*region, gpu_address - start, size, true); } diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 824dc76044..318607384b 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "common/common_types.h" @@ -101,7 +102,13 @@ struct Nx2SparseRegionState { u64 size; u64 block_size; std::vector written; + /// Once the game maps its own memory into the region (nx2:gpu 216): which tiles it maps. + /// Only those hold data then (writes elsewhere are dropped, as on the console). + std::vector mapped; }; + +/// The granularity of the game's sparse tiles (nx2:gpu 216 maps 64 KiB pages). +inline constexpr u64 Nx2SparseTileSize = 64ULL << 10; inline std::mutex g_nx2_sparse_state_mutex; inline std::map g_nx2_sparse_state; @@ -129,6 +136,59 @@ inline void Nx2SparseBlockWritten(u64 region_address, u64 block) { } } +/// The game maps its own memory over [offset, +size) of the region at region_address (mapped), or +/// unmaps it there. +inline void Nx2SparseTilesMapped(u64 region_address, u64 offset, u64 size, bool mapped) { + std::scoped_lock lk{g_nx2_sparse_state_mutex}; + const auto it = g_nx2_sparse_state.find(region_address); + if (it == g_nx2_sparse_state.end() || size == 0) { + return; + } + auto& region = it->second; + if (region.mapped.empty()) { + region.mapped.resize((region.size + Nx2SparseTileSize - 1) / Nx2SparseTileSize); + } + const u64 last = (std::min)((offset + size - 1) / Nx2SparseTileSize, u64{region.mapped.size() - 1}); + for (u64 tile = offset / Nx2SparseTileSize; tile <= last; ++tile) { + region.mapped[tile] = mapped; + } +} + +/// The parts of [gpu_address, +size), which lies in one sparse region, that can hold data: the +/// tiles the game maps there if it maps any, otherwise the blocks written. As (offset from +/// gpu_address, size) runs. +inline std::vector> Nx2SparseDataRanges(u64 gpu_address, u64 size) { + std::vector> ranges; + std::scoped_lock lk{g_nx2_sparse_state_mutex}; + auto it = g_nx2_sparse_state.upper_bound(gpu_address); + if (it == g_nx2_sparse_state.begin()) { + return ranges; + } + --it; + const auto& [start, region] = *it; + if (size == 0 || gpu_address + size > start + region.size) { + return ranges; + } + const bool by_tile = !region.mapped.empty(); + const u64 unit = by_tile ? Nx2SparseTileSize : region.block_size; + const std::vector& present = by_tile ? region.mapped : region.written; + const u64 begin = gpu_address - start; + const u64 end = begin + size; + for (u64 index = begin / unit; index * unit < end && index < present.size(); ++index) { + if (!present[index]) { + continue; + } + const u64 run_begin = (std::max)(index * unit, begin) - begin; + const u64 run_end = (std::min)((index + 1) * unit, end) - begin; + if (!ranges.empty() && ranges.back().first + ranges.back().second == run_begin) { + ranges.back().second += run_end - run_begin; + } else { + ranges.emplace_back(run_begin, run_end - run_begin); + } + } + return ranges; +} + /// True when [gpu_address, +size) lies in one sparse region. inline bool Nx2InSparseRegion(u64 gpu_address, u64 size) { if (size == 0 || gpu_address + size <= g_nx2_sparse_low.load(std::memory_order_relaxed) || diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 3294af9c22..947bbebe88 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -201,6 +201,23 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { return layer_uploads && size >= (32U << 20) && Tegra::Nx2InSparseRegion(gpu_addr, size); } +// A large single 2D image in a Switch 2 sparse region (title 1's world-sized R8/RG16/R16 masks, +// 170-341 MB, of which the game maps a few MiB) gets memory by sparse block where the guest memory +// under it can hold data: unbacked at first, then Image::BindTiles. Formats of one texel per +// block that the host samples as they are. +[[nodiscard]] bool UsesSparseTiles(const Device& device, const ImageInfo& info, GPUVAddr gpu_addr) { + if (!device.CanUseUnbackedSparseImages() || !device.IsSparseResidencyImage2DSupported() || + info.type != ImageType::e2D || info.resources.layers != 1 || info.num_samples != 1 || + info.size.depth != 1 || info.block.width != 0 || + VideoCore::Surface::DefaultBlockWidth(info.format) != 1 || + VideoCore::Surface::DefaultBlockHeight(info.format) != 1 || + VideoCore::Surface::GetFormatType(info.format) != VideoCore::Surface::SurfaceType::ColorTexture) { + return false; + } + const u32 size = VideoCommon::CalculateGuestSizeInBytes(info); + return size >= (32U << 20) && Tegra::Nx2InSparseRegion(gpu_addr, size); +} + // The most device memory one SparseLayers image binds: 768 MiB, 384 MiB on GPUs with less than // 7 GiB (title 1's BC6H arrays are 85 MB a layer). [[nodiscard]] VkDeviceSize SparseLayersCap(const Device& device) { @@ -2093,6 +2110,7 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu : runtime->ViewFormats(info.format), AstcHostFormat(runtime_.device, info), UsesSparseLayers(runtime_.device, info, gpu_addr_) || + UsesSparseTiles(runtime_.device, info, gpu_addr_) || IsUnwrittenSparse(runtime_.device, info, gpu_addr_))), aspect_mask(ImageAspectMask(info.format)) { if (original_image.IsUnbacked()) { @@ -2102,6 +2120,8 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu host_info.size.depth * static_cast(host_info.resources.layers); if (UsesSparseLayers(runtime_.device, info, gpu_addr_) && InitSparseLayers()) { flags |= VideoCommon::ImageFlagBits::SparseLayers; + } else if (UsesSparseTiles(runtime_.device, info, gpu_addr_) && InitSparseTiles()) { + flags |= VideoCommon::ImageFlagBits::SparseTiles; } else if (texels <= (u64{1} << 28)) { // Not too big to back (see MemoryAllocator::CreateImage): unbacked because nothing // was written there yet, so the first write gets it replaced. @@ -2633,6 +2653,196 @@ bool Image::InitSparseLayers() { return true; } +bool Image::InitSparseTiles() { + const Device& device = runtime->device; + const VkImage handle = *original_image; + const auto requirements = device.GetLogical().GetImageSparseMemoryRequirements(handle); + const auto it = std::ranges::find_if(requirements, [](const auto& r) { + return (r.formatProperties.aspectMask & VK_IMAGE_ASPECT_COLOR_BIT) != 0; + }); + if (it == requirements.end()) { + return false; + } + SparseLayerMemory& s = sparse_layers; + s.requirements = device.GetLogical().GetImageMemoryRequirements(handle); + const VkExtent3D granularity = it->formatProperties.imageGranularity; + const u32 bytes = VideoCore::Surface::BytesPerBlock(info.format); + // Rectangles are uploaded by whole GOB columns (64 bytes across). + if (granularity.width == 0 || granularity.height == 0 || (granularity.width * bytes) % 64 != 0 || + u64{granularity.width} * granularity.height * bytes != s.requirements.alignment) { + return false; + } + s.allocator = &runtime->memory_allocator; + s.sparse = *it; + s.width = info.size.width; + s.height = info.size.height; + s.levels = static_cast(info.resources.levels); + s.tile_width = granularity.width; + s.tile_height = granularity.height; + s.tail_level = (std::min)(it->imageMipTailFirstLod, s.levels); + s.level_tiles.resize(s.tail_level); + for (u32 level = 0; level < s.tail_level; ++level) { + const u32 width = (std::max)(s.width >> level, 1U); + const u32 height = (std::max)(s.height >> level, 1U); + s.level_tiles[level].assign( + size_t{Common::DivCeil(width, s.tile_width)} * Common::DivCeil(height, s.tile_height), + VK_NULL_HANDLE); + } + LOG_INFO(Render_Vulkan, "Sparse tiled image {:#x}: {}x{} {} levels, block {}x{}, mip tail from {}", + gpu_addr, s.width, s.height, s.levels, s.tile_width, s.tile_height, s.tail_level); + return true; +} + +bool Image::BindTiles(std::span rects) { + SparseLayerMemory& s = sparse_layers; + if (s.allocator == nullptr) { + return false; + } + const Device& device = runtime->device; + const u64 tile = s.requirements.alignment; + const VkMemoryRequirements tile_requirements{ + .size = tile, + .alignment = tile, + .memoryTypeBits = s.requirements.memoryTypeBits, + }; + std::vector image_binds; + std::vector opaque_binds; + std::vector fresh; // Blocks given memory by this call + bool fresh_tail = false; + // A failed bind leaves the blocks as they were, so the next call tries them again. + const auto roll_back = [&] { + for (VmaAllocation* const allocation : fresh) { + s.allocator->FreeSparseMemory(*allocation); + *allocation = VK_NULL_HANDLE; + s.bound_bytes -= tile; + } + if (fresh_tail) { + s.allocator->FreeSparseMemory(s.tail_memory); + s.tail_memory = VK_NULL_HANDLE; + s.bound_bytes -= Common::AlignUp(s.sparse.imageMipTailSize, tile); + } + }; + bool needs_tail = false; + for (const VideoCommon::LevelRect& rect : rects) { + const u32 level = static_cast(rect.level); + if (level >= s.tail_level) { + needs_tail = true; + continue; + } + const u32 width = (std::max)(s.width >> level, 1U); + const u32 height = (std::max)(s.height >> level, 1U); + const u32 tiles_x = Common::DivCeil(width, s.tile_width); + for (u32 ty = rect.y / s.tile_height; ty <= (rect.y + rect.height - 1) / s.tile_height; + ++ty) { + for (u32 tx = rect.x / s.tile_width; tx <= (rect.x + rect.width - 1) / s.tile_width; + ++tx) { + VmaAllocation& allocation = s.level_tiles[level][size_t{ty} * tiles_x + tx]; + if (allocation != VK_NULL_HANDLE) { + continue; + } + VkDeviceMemory memory{}; + VkDeviceSize offset{}; + allocation = s.allocator->AllocateSparseMemory(tile_requirements, memory, offset); + if (allocation == VK_NULL_HANDLE) { + LOG_WARNING(Render_Vulkan, "Sparse tiled image {:#x}: out of device memory", + gpu_addr); + roll_back(); + return false; + } + fresh.push_back(&allocation); + s.bound_bytes += tile; + const u32 x = tx * s.tile_width; + const u32 y = ty * s.tile_height; + image_binds.push_back(VkSparseImageMemoryBind{ + .subresource{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = level, + .arrayLayer = 0, + }, + .offset{static_cast(x), static_cast(y), 0}, + .extent{(std::min)(s.tile_width, width - x), (std::min)(s.tile_height, height - y), + 1}, + .memory = memory, + .memoryOffset = offset, + .flags = 0, + }); + } + } + } + if (needs_tail && s.tail_level < s.levels && s.tail_memory == VK_NULL_HANDLE) { + const VkMemoryRequirements requirements{ + .size = Common::AlignUp(s.sparse.imageMipTailSize, tile), + .alignment = tile, + .memoryTypeBits = s.requirements.memoryTypeBits, + }; + VkDeviceMemory memory{}; + VkDeviceSize offset{}; + s.tail_memory = s.allocator->AllocateSparseMemory(requirements, memory, offset); + if (s.tail_memory == VK_NULL_HANDLE) { + LOG_WARNING(Render_Vulkan, "Sparse tiled image {:#x}: out of device memory for the mip " + "tail", + gpu_addr); + roll_back(); + return false; + } + fresh_tail = true; + s.bound_bytes += requirements.size; + opaque_binds.push_back(VkSparseMemoryBind{ + .resourceOffset = s.sparse.imageMipTailOffset, + .size = s.sparse.imageMipTailSize, + .memory = memory, + .memoryOffset = offset, + .flags = 0, + }); + } + if (image_binds.empty() && opaque_binds.empty()) { + return true; + } + const VkSparseImageMemoryBindInfo image_bind_info{ + .image = *original_image, + .bindCount = static_cast(image_binds.size()), + .pBinds = image_binds.data(), + }; + const VkSparseImageOpaqueMemoryBindInfo opaque_bind_info{ + .image = *original_image, + .bindCount = static_cast(opaque_binds.size()), + .pBinds = opaque_binds.data(), + }; + const VkBindSparseInfo bind_info{ + .sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO, + .pNext = nullptr, + .waitSemaphoreCount = 0, + .pWaitSemaphores = nullptr, + .bufferBindCount = 0, + .pBufferBinds = nullptr, + .imageOpaqueBindCount = opaque_binds.empty() ? 0U : 1U, + .pImageOpaqueBinds = opaque_binds.empty() ? nullptr : &opaque_bind_info, + .imageBindCount = image_binds.empty() ? 0U : 1U, + .pImageBinds = image_binds.empty() ? nullptr : &image_bind_info, + .signalSemaphoreCount = 0, + .pSignalSemaphores = nullptr, + }; + // Bound before the command buffer that uploads the blocks is submitted (as BindLayers). + vk::Fence fence = device.GetLogical().CreateFence(VkFenceCreateInfo{ + .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + }); + VkResult result; + { + std::scoped_lock lock{scheduler->submit_mutex}; + result = device.GetGraphicsQueue().BindSparse(bind_info, *fence); + } + if (result != VK_SUCCESS) { + LOG_ERROR(Render_Vulkan, "Sparse tiled image {:#x}: binding failed ({})", gpu_addr, + static_cast(result)); + roll_back(); + return false; + } + fence.Wait(); + return true; +} + std::vector Image::BindLayers(std::span layers) { SparseLayerMemory& s = sparse_layers; std::vector bound; diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 3944e183fd..89eb8424eb 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -389,6 +389,21 @@ public: /// layers that have memory afterwards (a cap per image can leave some out). std::vector BindLayers(std::span layers); + /// SparseTiles images: gives the host sparse blocks under `rects` (and the mip tail, for + /// levels in it) device memory where they have none yet. False if that failed. + bool BindTiles(std::span rects); + + /// SparseTiles images: the host sparse block, in texels, and the first level of the mip tail. + u32 SparseTileWidth() const noexcept { + return sparse_layers.tile_width; + } + u32 SparseTileHeight() const noexcept { + return sparse_layers.tile_height; + } + s32 SparseTailLevel() const noexcept { + return static_cast(sparse_layers.tail_level); + } + /// Device memory BindLayers has bound to the image's layers (an unbacked image has no other). u64 SparseBoundBytes() const noexcept { return sparse_layers.bound_bytes; @@ -434,6 +449,9 @@ private: layer_bytes = other.layer_bytes; layer_memory = std::move(other.layer_memory); other.layer_memory.clear(); + level_tiles = std::move(other.level_tiles); + other.level_tiles.clear(); + tail_level = other.tail_level; tail_memory = std::exchange(other.tail_memory, VK_NULL_HANDLE); bound_bytes = other.bound_bytes; warned = other.warned; @@ -449,8 +467,14 @@ private: for (const VmaAllocation allocation : layer_memory) { allocator->FreeSparseMemory(allocation); } + for (const auto& tiles : level_tiles) { + for (const VmaAllocation allocation : tiles) { + allocator->FreeSparseMemory(allocation); + } + } allocator->FreeSparseMemory(tail_memory); layer_memory.clear(); + level_tiles.clear(); tail_memory = VK_NULL_HANDLE; } @@ -464,6 +488,9 @@ private: u32 tile_height{}; VkDeviceSize layer_bytes{}; ///< One layer's levels before the mip tail, and its own tail std::vector layer_memory; ///< Per layer, null until bound + /// SparseTiles: per level before the mip tail, per sparse block (rows), null until bound + std::vector> level_tiles; + u32 tail_level{}; ///< SparseTiles: first level of the mip tail VmaAllocation tail_memory{}; ///< The mip tail all layers share, if one VkDeviceSize bound_bytes{}; bool warned{}; @@ -473,6 +500,9 @@ private: /// Reads the image's sparse requirements; false when it can't be bound by layer. bool InitSparseLayers(); + /// Reads the image's sparse requirements; false when it can't be bound by sparse block. + bool InitSparseTiles(); + vk::Image original_image; vk::Image scaled_image; diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h index 04254fb417..878c229953 100644 --- a/src/video_core/texture_cache/image_base.h +++ b/src/video_core/texture_cache/image_base.h @@ -56,6 +56,10 @@ enum class ImageFlagBits : u32 { /// A large array over Switch 2 sparse blocks: unbacked, each layer gets device memory when /// something is written to it (Image::BindLayers) and only those layers are uploaded. SparseLayers = 1 << 22, + /// A large single image over Switch 2 sparse blocks: unbacked, each host sparse block gets + /// device memory when the guest memory under it can hold data (Image::BindTiles), and only + /// the parts written are uploaded. + SparseTiles = 1 << 23, }; DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits) @@ -108,6 +112,9 @@ struct ImageBase { /// Large layered images uploaded by layer: one bit per layer written since the last upload /// while CpuModified. Empty while CpuModified means the whole image. std::vector dirty_layers; + /// SparseTiles images: the guest byte ranges (offset, size) written since the last upload while + /// CpuModified. Empty while CpuModified means all of it that can hold data. + std::vector> dirty_ranges; u64 scale_tick = 0; bool has_scaled = false; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 3b783fc6c6..4b93cd135b 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -729,6 +729,10 @@ void TextureCache

::WriteMemory(DAddr cpu_addr, size_t size) { image.flags |= ImageFlagBits::CpuModified | ImageFlagBits::Remapped; return; } + if (True(image.flags & ImageFlagBits::SparseTiles)) { + MarkRangeWritten(image, cpu_addr, size); + return; + } if (UploadsByLayer(image)) { MarkLayersWritten(image, cpu_addr, size); return; @@ -1316,6 +1320,10 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { RefreshSparseLayers(image); return; } + if (True(image.flags & ImageFlagBits::SparseTiles)) { + RefreshSparseTiles(image); + return; + } if (!image.dirty_layers.empty() && UploadWrittenLayers(image)) { runtime.InsertUploadMemoryBarrier(); return; @@ -1400,6 +1408,63 @@ void TextureCache

::MarkLayersWritten(Image& image, DAddr cpu_addr, size_t siz } } +template +void TextureCache

::MarkRangeWritten(Image& image, DAddr cpu_addr, size_t size) { + if (True(image.flags & ImageFlagBits::CpuModified) && image.dirty_ranges.empty()) { + return; // all of it is dirty already + } + const DAddr begin = (std::max)(cpu_addr, image.cpu_addr); + const DAddr end = (std::min)(cpu_addr + size, image.cpu_addr + image.guest_size_bytes); + if (end <= begin) { + return; + } + image.flags |= ImageFlagBits::CpuModified; + image.dirty_ranges.emplace_back(begin - image.cpu_addr, end - begin); +} + +template +void TextureCache

::RefreshSparseTiles(Image& image) { + if constexpr (requires { &Image::BindTiles; }) { + // The ranges written since the last upload, or, for a new image, all of it that can hold + // data (the tiles the game maps there, or the blocks written). + std::vector> ranges = std::move(image.dirty_ranges); + image.dirty_ranges.clear(); + if (ranges.empty()) { + ranges = Tegra::Nx2SparseDataRanges(image.gpu_addr, image.guest_size_bytes); + } + const std::vector rects = + SparseTileRects(image.info, ranges, image.SparseTileWidth(), image.SparseTileHeight(), + image.SparseTailLevel()); + if (rects.empty() || !BindSparseTiles(image, rects)) { + runtime.TransitionImageLayout(image); + return; + } + const size_t bytes = UnswizzledRectsBytes(image.info, rects); + auto staging = runtime.UploadStagingBuffer(bytes); + const std::vector copies = + UnswizzleImageRects(*gpu_memory, image.gpu_addr, image.info, rects, staging.mapped_span); + image.UploadMemory(staging, copies); + runtime.InsertUploadMemoryBarrier(); + ++images_uploaded; + uploaded_bytes += bytes; + } +} + +template +bool TextureCache

::BindSparseTiles(Image& image, std::span rects) { + if constexpr (requires { &Image::BindTiles; }) { + const u64 bound_bytes = ImageMemoryBytes(image); + const bool bound = image.BindTiles(rects); + // RegisterImage counts what an image has bound by then; later binds are added here. + if (True(image.flags & ImageFlagBits::Registered)) { + total_used_memory += ImageMemoryBytes(image) - bound_bytes; + } + return bound; + } else { + return false; + } +} + template bool TextureCache

::UploadWrittenLayers(Image& image) { if constexpr (requires { &Runtime::AccelerateImageUploadLayers; }) { diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 94102cd261..f27e237cfa 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -164,6 +164,15 @@ class TextureCache : public VideoCommon::ChannelSetupCaches rects); + /// Uploads the image's dirty layers. False when it needs a full upload instead. bool UploadWrittenLayers(Image& image); diff --git a/src/video_core/texture_cache/types.h b/src/video_core/texture_cache/types.h index ecacd8c6bd..9e541d6afc 100644 --- a/src/video_core/texture_cache/types.h +++ b/src/video_core/texture_cache/types.h @@ -132,6 +132,15 @@ struct ImageCopy { Extent3D extent; }; +/// A rectangle of one mip level, in texels. +struct LevelRect { + s32 level; + u32 x; + u32 y; + u32 width; + u32 height; +}; + struct BufferImageCopy { size_t buffer_offset; size_t buffer_size; diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp index ef34019a08..0e1acba813 100644 --- a/src/video_core/texture_cache/util.cpp +++ b/src/video_core/texture_cache/util.cpp @@ -929,6 +929,210 @@ boost::container::small_vector UnswizzleImage(Tegra::Memory return copies; } +namespace { +// One level of a single-layer 2D block-linear image of 1x1-texel blocks, laid out as +// UnswizzleImage reads it: rows of blocks, each block one GOB wide and 2^block_height GOBs tall. +struct BlockLinearLevel { + u32 offset; ///< Into the image + u32 width; ///< Texels + u32 height; + u32 block_height; ///< Log2 of the GOBs a block is tall + u32 block_texels; ///< Texels across a block + u32 block_rows; ///< Texel rows in a block + u32 block_bytes; + u32 blocks_per_row; +}; + +std::vector BlockLinearLevels(const ImageInfo& info) { + const LevelInfo level_info = MakeLevelInfo(info); + const u32 bpp_log2 = BytesPerBlockLog2(info.format); + const Extent2D gob = GobSize(bpp_log2, info.block.height, info.tile_width_spacing); + std::vector levels; + u32 offset = 0; + for (s32 level = 0; level < info.resources.levels; ++level) { + const Extent3D level_size = AdjustMipSize(info.size, level); + const Extent3D num_tiles = AdjustTileSize(level_size, level_info.tile_size); + const Extent3D block = + AdjustMipBlockSize(num_tiles, level_info.block, level, level_info.num_levels); + const u32 stride_alignment = StrideAlignment(num_tiles, info.block, gob, bpp_log2); + const u32 stride = Common::AlignUpLog2(num_tiles.width, stride_alignment) << bpp_log2; + levels.push_back({ + .offset = offset, + .width = level_size.width, + .height = level_size.height, + .block_height = block.height, + .block_texels = GOB_SIZE_X >> bpp_log2, + .block_rows = GOB_SIZE_Y << block.height, + .block_bytes = GOB_SIZE << block.height, + .blocks_per_row = Common::DivCeilLog2(stride, GOB_SIZE_X_SHIFT), + }); + offset += CalculateLevelSize(level_info, level); + } + return levels; +} + +// Rows of blocks a rectangle covers, and its blocks across. +struct RectBlocks { + u32 first_row; + u32 last_row; + u32 first_block; + u32 num_blocks; +}; + +RectBlocks BlocksOf(const BlockLinearLevel& level, const LevelRect& rect) { + const u32 first_block = rect.x / level.block_texels; + return { + .first_row = rect.y / level.block_rows, + .last_row = (rect.y + rect.height - 1) / level.block_rows, + .first_block = first_block, + .num_blocks = (std::min)(Common::DivCeil(rect.width, level.block_texels), + level.blocks_per_row - first_block), + }; +} + +u32 BlockRowTexelRows(const BlockLinearLevel& level, u32 row) { + return (std::min)(level.block_rows, level.height - row * level.block_rows); +} +} // Anonymous namespace + +std::vector SparseTileRects(const ImageInfo& info, + std::span> ranges, u32 tile_width, + u32 tile_height, s32 tail_level) { + std::vector rects; + const std::vector levels = BlockLinearLevels(info); + const u64 image_end = CalculateLevelBytes( + CalculateLevelSizes(MakeLevelInfo(info), info.resources.levels), info.resources.levels); + for (s32 index = 0; index < static_cast(levels.size()); ++index) { + const BlockLinearLevel& level = levels[index]; + const u64 level_end = + index + 1 < static_cast(levels.size()) ? levels[index + 1].offset : image_end; + const auto overlaps = [&](const std::pair& range) { + return range.first < level_end && range.first + range.second > level.offset; + }; + if (index >= tail_level) { + if (std::ranges::any_of(ranges, overlaps)) { + rects.push_back({index, 0, 0, level.width, level.height}); + } + continue; + } + const u32 tiles_x = Common::DivCeil(level.width, tile_width); + const u32 tiles_y = Common::DivCeil(level.height, tile_height); + std::vector touched(size_t{tiles_x} * tiles_y); + for (const auto& range : ranges) { + if (!overlaps(range)) { + continue; + } + const u64 begin = (std::max)(range.first, level.offset) - level.offset; + const u64 end = (std::min)(range.first + range.second, level_end) - level.offset; + const u64 first = begin / level.block_bytes; + const u64 last = (end - 1) / level.block_bytes; + for (u64 row = first / level.blocks_per_row; row <= last / level.blocks_per_row; ++row) { + const u64 row_start = row * level.blocks_per_row; + const u64 low = (std::max)(first, row_start) - row_start; + const u64 high = (std::min)(last, row_start + level.blocks_per_row - 1) - row_start; + const u64 x0 = low * level.block_texels; + const u64 y0 = row * level.block_rows; + if (x0 >= level.width || y0 >= level.height) { + continue; + } + const u64 x1 = (std::min)((high + 1) * level.block_texels, level.width); + const u64 y1 = (std::min)(y0 + level.block_rows, level.height); + for (u64 ty = y0 / tile_height; ty <= (y1 - 1) / tile_height; ++ty) { + for (u64 tx = x0 / tile_width; tx <= (x1 - 1) / tile_width; ++tx) { + touched[ty * tiles_x + tx] = true; + } + } + } + } + for (u32 ty = 0; ty < tiles_y; ++ty) { + for (u32 tx = 0; tx < tiles_x;) { + if (!touched[size_t{ty} * tiles_x + tx]) { + ++tx; + continue; + } + const u32 first_tile = tx; + while (tx < tiles_x && touched[size_t{ty} * tiles_x + tx]) { + ++tx; + } + const u32 x = first_tile * tile_width; + const u32 y = ty * tile_height; + rects.push_back({ + .level = index, + .x = x, + .y = y, + .width = (std::min)(tx * tile_width, level.width) - x, + .height = (std::min)(y + tile_height, level.height) - y, + }); + } + } + } + return rects; +} + +size_t UnswizzledRectsBytes(const ImageInfo& info, std::span rects) { + const std::vector levels = BlockLinearLevels(info); + const u32 bpp_log2 = BytesPerBlockLog2(info.format); + size_t bytes = 0; + for (const LevelRect& rect : rects) { + const BlockLinearLevel& level = levels[rect.level]; + const RectBlocks blocks = BlocksOf(level, rect); + for (u32 row = blocks.first_row; row <= blocks.last_row; ++row) { + bytes += (size_t{blocks.num_blocks * level.block_texels} << bpp_log2) * + BlockRowTexelRows(level, row); + } + } + return bytes; +} + +std::vector UnswizzleImageRects(Tegra::MemoryManager& gpu_memory, + GPUVAddr gpu_addr, const ImageInfo& info, + std::span rects, + std::span output) { + const std::vector levels = BlockLinearLevels(info); + const u32 bpp_log2 = BytesPerBlockLog2(info.format); + std::vector copies; + std::vector swizzled; + size_t output_offset = 0; + for (const LevelRect& rect : rects) { + const BlockLinearLevel& level = levels[rect.level]; + const RectBlocks blocks = BlocksOf(level, rect); + const u32 width = blocks.num_blocks * level.block_texels; + const size_t row_bytes = size_t{blocks.num_blocks} * level.block_bytes; + for (u32 row = blocks.first_row; row <= blocks.last_row; ++row) { + // A row of whole blocks is laid out like an image of that width on its own. + const u32 texel_rows = BlockRowTexelRows(level, row); + const size_t bytes = (size_t{width} << bpp_log2) * texel_rows; + const GPUVAddr row_addr = gpu_addr + level.offset + + u64{row} * level.blocks_per_row * level.block_bytes + + u64{blocks.first_block} * level.block_bytes; + swizzled.assign(row_bytes, 0); + gpu_memory.ReadBlockUnsafe(row_addr, swizzled.data(), row_bytes); + UnswizzleTexture(output.subspan(output_offset, bytes), swizzled, 1U << bpp_log2, width, + texel_rows, 1, level.block_height, 0, 0, + gpu_memory.UsesNativeTiling()); + const u32 row_y = row * level.block_rows; + const u32 y0 = (std::max)(rect.y, row_y); + const u32 y1 = (std::min)(rect.y + rect.height, row_y + texel_rows); + copies.push_back(BufferImageCopy{ + .buffer_offset = output_offset + ((size_t{y0 - row_y} * width) << bpp_log2), + .buffer_size = bytes, + .buffer_row_length = width, + .buffer_image_height = texel_rows, + .image_subresource = + { + .base_level = rect.level, + .base_layer = 0, + .num_layers = 1, + }, + .image_offset = {static_cast(rect.x), static_cast(y0), 0}, + .image_extent = {rect.width, y1 - y0, 1}, + }); + output_offset += bytes; + } + } + return copies; +} + void ConvertedLayout(const ImageInfo& info, std::span copies) { // Same offsets and sizes as ConvertImage below. const auto recompression = Settings::GetAstcRecompression(); diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h index 0381d2d24b..74da0f90f9 100644 --- a/src/video_core/texture_cache/util.h +++ b/src/video_core/texture_cache/util.h @@ -8,6 +8,8 @@ #include #include +#include +#include #include #include "common/common_types.h" @@ -72,6 +74,25 @@ struct OverlapResult { Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info, std::span input, std::span output); +/// For a single-layer 2D block-linear image of 1x1-texel blocks: the host sparse blocks +/// (tile_width x tile_height texels) that the guest byte ranges (offset, size) touch, merged into +/// runs along x. Levels from tail_level on (the host's mip tail) come whole. +[[nodiscard]] std::vector SparseTileRects(const ImageInfo& info, + std::span> ranges, + u32 tile_width, u32 tile_height, + s32 tail_level); + +/// Bytes UnswizzleImageRects writes for `rects`. +[[nodiscard]] size_t UnswizzledRectsBytes(const ImageInfo& info, std::span rects); + +/// Unswizzles `rects` (from SparseTileRects: on GOB columns) of a single-layer 2D block-linear +/// image into output, reading only the guest blocks under them. +[[nodiscard]] std::vector UnswizzleImageRects(Tegra::MemoryManager& gpu_memory, + GPUVAddr gpu_addr, + const ImageInfo& info, + std::span rects, + std::span output); + /// Rewrites `copies` (from UnswizzleImage) to the layout ConvertImage produces, without converting. void ConvertedLayout(const ImageInfo& info, std::span copies); -- cgit v1.2.3 From 130c255ed91e2bf8f726b38101cc5fc2abcf3935 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:21:38 -0400 Subject: vulkan: share one zeroed block between empty sparse blocks on AMD AMD's drivers can't sample images with unbound memory, so large sparse images are backed whole there, Mario Kart World's nine sparse masks included (about 2.9 GB, re-uploaded whole for every tile the game maps). Where the driver supports sparseResidencyAliased, SparseTiles images are now created with VK_IMAGE_CREATE_SPARSE_ALIASED_BIT and every block is bound to one zeroed 64 KiB block when the image is created (the image is cleared once); a block gets memory of its own when data arrives, as on other drivers. Nothing is ever left unbound. If the GPU renders into such an image, every block gets memory of its own first, cleared, and the data is loaded again, so the shared block stays zero. EDEN_NX2_SPARSE_ALIASED=1 uses the shared block wherever it is supported, =0 never. Radeon 780M (proprietary driver 32.0.31041), Mario Kart World title screen, 200 s: no hang or device loss, world lit, shared memory 12.7 GB at the end growing 1.8 GB/min, against 18.7 GB and 5.1 GB/min with the masks backed whole. With every sparse image unbound instead (no AMD fallback at all) the static world renders dark there, so the other large sparse images stay backed whole on AMD. --- .../renderer_vulkan/vk_texture_cache.cpp | 227 ++++++++++++++++++++- src/video_core/renderer_vulkan/vk_texture_cache.h | 21 ++ src/video_core/texture_cache/texture_cache.h | 26 +++ src/video_core/texture_cache/texture_cache_base.h | 3 + src/video_core/vulkan_common/vulkan_device.h | 5 + .../vulkan_common/vulkan_memory_allocator.cpp | 5 +- .../vulkan_common/vulkan_memory_allocator.h | 3 +- 7 files changed, 281 insertions(+), 9 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 947bbebe88..ddd55de690 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -201,13 +201,39 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { return layer_uploads && size >= (32U << 20) && Tegra::Nx2InSparseRegion(gpu_addr, size); } +// How SparseTiles images leave the blocks without data: unbound (reads give zero), or all bound to +// one zeroed block where the driver can't sample unbound memory (AMD) but lets blocks share +// memory. EDEN_NX2_SPARSE_ALIASED=1 takes the shared block wherever it is supported, =0 never. +enum class SparseTilesMode { + None, + Unbound, + Aliased, +}; + +[[nodiscard]] SparseTilesMode SparseTilesModeOf(const Device& device) { + static const char force_aliased = [] { + const char* value = std::getenv("EDEN_NX2_SPARSE_ALIASED"); + return value == nullptr ? '\0' : value[0]; + }(); + if (!device.IsSparseResidencyImage2DSupported()) { + return SparseTilesMode::None; + } + const bool aliased = device.IsSparseResidencyAliasedSupported() && force_aliased != '0'; + if (aliased && force_aliased == '1') { + return SparseTilesMode::Aliased; + } + if (device.CanUseUnbackedSparseImages()) { + return SparseTilesMode::Unbound; + } + return aliased ? SparseTilesMode::Aliased : SparseTilesMode::None; +} + // A large single 2D image in a Switch 2 sparse region (title 1's world-sized R8/RG16/R16 masks, // 170-341 MB, of which the game maps a few MiB) gets memory by sparse block where the guest memory // under it can hold data: unbacked at first, then Image::BindTiles. Formats of one texel per // block that the host samples as they are. [[nodiscard]] bool UsesSparseTiles(const Device& device, const ImageInfo& info, GPUVAddr gpu_addr) { - if (!device.CanUseUnbackedSparseImages() || !device.IsSparseResidencyImage2DSupported() || - info.type != ImageType::e2D || info.resources.layers != 1 || info.num_samples != 1 || + if (SparseTilesModeOf(device) == SparseTilesMode::None || info.type != ImageType::e2D || info.resources.layers != 1 || info.num_samples != 1 || info.size.depth != 1 || info.block.width != 0 || VideoCore::Surface::DefaultBlockWidth(info.format) != 1 || VideoCore::Surface::DefaultBlockHeight(info.format) != 1 || @@ -370,7 +396,7 @@ constexpr std::array OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM [[nodiscard]] vk::Image MakeImage(const Device& device, const MemoryAllocator& allocator, const ImageInfo& info, std::span view_formats, std::optional format_override = {}, - bool unbacked = false) { + bool unbacked = false, bool aliased = false) { if (info.type == ImageType::Buffer) { return vk::Image{}; } @@ -399,7 +425,7 @@ constexpr std::array OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM } } if (unbacked) { - if (vk::Image image = allocator.CreateUnbackedImage(image_ci)) { + if (vk::Image image = allocator.CreateUnbackedImage(image_ci, aliased)) { return image; } } @@ -2111,7 +2137,9 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu AstcHostFormat(runtime_.device, info), UsesSparseLayers(runtime_.device, info, gpu_addr_) || UsesSparseTiles(runtime_.device, info, gpu_addr_) || - IsUnwrittenSparse(runtime_.device, info, gpu_addr_))), + IsUnwrittenSparse(runtime_.device, info, gpu_addr_), + UsesSparseTiles(runtime_.device, info, gpu_addr_) && + SparseTilesModeOf(runtime_.device) == SparseTilesMode::Aliased)), aspect_mask(ImageAspectMask(info.format)) { if (original_image.IsUnbacked()) { flags |= VideoCommon::ImageFlagBits::Unbacked; @@ -2688,8 +2716,193 @@ bool Image::InitSparseTiles() { size_t{Common::DivCeil(width, s.tile_width)} * Common::DivCeil(height, s.tile_height), VK_NULL_HANDLE); } - LOG_INFO(Render_Vulkan, "Sparse tiled image {:#x}: {}x{} {} levels, block {}x{}, mip tail from {}", - gpu_addr, s.width, s.height, s.levels, s.tile_width, s.tile_height, s.tail_level); + if (SparseTilesModeOf(device) == SparseTilesMode::Aliased && !BindZeroBlock()) { + LOG_WARNING(Render_Vulkan, "Sparse tiled image {:#x}: no zeroed block to share", gpu_addr); + s.Release(); + s.allocator = nullptr; + s.bound_bytes = 0; + return false; + } + LOG_INFO(Render_Vulkan, "Sparse tiled image {:#x}: {}x{} {} levels, block {}x{}, mip tail from {}{}", + gpu_addr, s.width, s.height, s.levels, s.tile_width, s.tile_height, s.tail_level, + s.zero_block != VK_NULL_HANDLE ? ", empty blocks share a zeroed one" : ""); + return true; +} + +bool Image::BindZeroBlock() { + SparseLayerMemory& s = sparse_layers; + const Device& device = runtime->device; + const u64 tile = s.requirements.alignment; + VkDeviceMemory memory{}; + VkDeviceSize offset{}; + s.zero_block = s.allocator->AllocateSparseMemory( + VkMemoryRequirements{ + .size = tile, + .alignment = tile, + .memoryTypeBits = s.requirements.memoryTypeBits, + }, + memory, offset); + if (s.zero_block == VK_NULL_HANDLE) { + return false; + } + s.bound_bytes += tile; + std::vector image_binds; + for (u32 level = 0; level < s.tail_level; ++level) { + const u32 width = (std::max)(s.width >> level, 1U); + const u32 height = (std::max)(s.height >> level, 1U); + for (u32 y = 0; y < height; y += s.tile_height) { + for (u32 x = 0; x < width; x += s.tile_width) { + image_binds.push_back(VkSparseImageMemoryBind{ + .subresource{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = level, + .arrayLayer = 0, + }, + .offset{static_cast(x), static_cast(y), 0}, + .extent{(std::min)(s.tile_width, width - x), (std::min)(s.tile_height, height - y), + 1}, + .memory = memory, + .memoryOffset = offset, + .flags = 0, + }); + } + } + } + // The mip tail gets memory of its own: it is small, and the clear below zeroes it too. + std::vector opaque_binds; + if (s.tail_level < s.levels) { + const VkMemoryRequirements requirements{ + .size = Common::AlignUp(s.sparse.imageMipTailSize, tile), + .alignment = tile, + .memoryTypeBits = s.requirements.memoryTypeBits, + }; + VkDeviceMemory tail_memory{}; + VkDeviceSize tail_offset{}; + s.tail_memory = s.allocator->AllocateSparseMemory(requirements, tail_memory, tail_offset); + if (s.tail_memory == VK_NULL_HANDLE) { + return false; + } + s.bound_bytes += requirements.size; + opaque_binds.push_back(VkSparseMemoryBind{ + .resourceOffset = s.sparse.imageMipTailOffset, + .size = s.sparse.imageMipTailSize, + .memory = tail_memory, + .memoryOffset = tail_offset, + .flags = 0, + }); + } + const VkSparseImageMemoryBindInfo image_bind_info{ + .image = *original_image, + .bindCount = static_cast(image_binds.size()), + .pBinds = image_binds.data(), + }; + const VkSparseImageOpaqueMemoryBindInfo opaque_bind_info{ + .image = *original_image, + .bindCount = static_cast(opaque_binds.size()), + .pBinds = opaque_binds.data(), + }; + const VkBindSparseInfo bind_info{ + .sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO, + .pNext = nullptr, + .waitSemaphoreCount = 0, + .pWaitSemaphores = nullptr, + .bufferBindCount = 0, + .pBufferBinds = nullptr, + .imageOpaqueBindCount = opaque_binds.empty() ? 0U : 1U, + .pImageOpaqueBinds = opaque_binds.empty() ? nullptr : &opaque_bind_info, + .imageBindCount = image_binds.empty() ? 0U : 1U, + .pImageBinds = image_binds.empty() ? nullptr : &image_bind_info, + .signalSemaphoreCount = 0, + .pSignalSemaphores = nullptr, + }; + vk::Fence fence = device.GetLogical().CreateFence(VkFenceCreateInfo{ + .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + }); + VkResult result; + { + std::scoped_lock lock{scheduler->submit_mutex}; + result = device.GetGraphicsQueue().BindSparse(bind_info, *fence); + } + if (result != VK_SUCCESS) { + LOG_ERROR(Render_Vulkan, "Sparse tiled image {:#x}: binding the zeroed block failed ({})", + gpu_addr, static_cast(result)); + return false; + } + fence.Wait(); + // Every block now reads the shared block (and the tail its own memory): zero them all. + ClearTiles(); + return true; +} + +void Image::ClearTiles() { + scheduler->RequestOutsideRenderPassOperationContext(); + const VkImage vk_image = *original_image; + const bool was_initialized = std::exchange(initialized, true); + scheduler->Record([vk_image, was_initialized](vk::CommandBuffer cmdbuf) { + static constexpr VkImageSubresourceRange range{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }; + const VkImageMemoryBarrier to_transfer{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .oldLayout = was_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange = range, + }; + const VkImageMemoryBarrier to_general{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | + VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | + VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | + VK_ACCESS_TRANSFER_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange = range, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, + to_transfer); + cmdbuf.ClearColorImage(vk_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VkClearColorValue{}, + range); + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, + to_general); + }); +} + +bool Image::BackTilesWhole() { + SparseLayerMemory& s = sparse_layers; + if (s.zero_block == VK_NULL_HANDLE) { + return true; + } + std::vector rects; + for (u32 level = 0; level < s.levels; ++level) { + rects.push_back({static_cast(level), 0, 0, (std::max)(s.width >> level, 1U), + (std::max)(s.height >> level, 1U)}); + } + if (!BindTiles(rects)) { + return false; + } + // No block reads the shared one any more. + s.allocator->FreeSparseMemory(s.zero_block); + s.zero_block = VK_NULL_HANDLE; + s.bound_bytes -= s.requirements.alignment; + ClearTiles(); return true; } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 89eb8424eb..43218d6ad3 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -404,6 +404,15 @@ public: return static_cast(sparse_layers.tail_level); } + /// SparseTiles images whose blocks without memory of their own share one zeroed block. + bool SparseTilesShareZeroBlock() const noexcept { + return sparse_layers.zero_block != VK_NULL_HANDLE; + } + + /// SparseTiles images sharing a zeroed block, before the GPU renders into them: every block + /// gets memory of its own, all cleared (the caller loads the data again). False on failure. + bool BackTilesWhole(); + /// Device memory BindLayers has bound to the image's layers (an unbacked image has no other). u64 SparseBoundBytes() const noexcept { return sparse_layers.bound_bytes; @@ -452,6 +461,7 @@ private: level_tiles = std::move(other.level_tiles); other.level_tiles.clear(); tail_level = other.tail_level; + zero_block = std::exchange(other.zero_block, VK_NULL_HANDLE); tail_memory = std::exchange(other.tail_memory, VK_NULL_HANDLE); bound_bytes = other.bound_bytes; warned = other.warned; @@ -473,6 +483,8 @@ private: } } allocator->FreeSparseMemory(tail_memory); + allocator->FreeSparseMemory(zero_block); + zero_block = VK_NULL_HANDLE; layer_memory.clear(); level_tiles.clear(); tail_memory = VK_NULL_HANDLE; @@ -491,6 +503,9 @@ private: /// SparseTiles: per level before the mip tail, per sparse block (rows), null until bound std::vector> level_tiles; u32 tail_level{}; ///< SparseTiles: first level of the mip tail + /// SparseTiles: the zeroed block every block without memory of its own is bound to, where + /// the driver can't sample unbound memory + VmaAllocation zero_block{}; VmaAllocation tail_memory{}; ///< The mip tail all layers share, if one VkDeviceSize bound_bytes{}; bool warned{}; @@ -503,6 +518,12 @@ private: /// Reads the image's sparse requirements; false when it can't be bound by sparse block. bool InitSparseTiles(); + /// Binds one zeroed block to every block of a SparseTiles image (and memory to its mip tail). + bool BindZeroBlock(); + + /// Records a clear of the whole image to zero. + void ClearTiles(); + vk::Image original_image; vk::Image scaled_image; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 4b93cd135b..5f655e5b60 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -1465,6 +1465,20 @@ bool TextureCache

::BindSparseTiles(Image& image, std::span r } } +template +bool TextureCache

::BackSparseTilesWhole(Image& image) { + if constexpr (requires { &Image::BackTilesWhole; }) { + const u64 bound_bytes = ImageMemoryBytes(image); + const bool backed = image.BackTilesWhole(); + if (True(image.flags & ImageFlagBits::Registered)) { + total_used_memory += ImageMemoryBytes(image) - bound_bytes; + } + return backed; + } else { + return false; + } +} + template bool TextureCache

::UploadWrittenLayers(Image& image) { if constexpr (requires { &Runtime::AccelerateImageUploadLayers; }) { @@ -3090,6 +3104,18 @@ void TextureCache

::PrepareImage(ImageId image_id, bool is_modification, bool } } if (is_modification) { + if constexpr (requires { &Image::BackTilesWhole; }) { + if (True(image.flags & ImageFlagBits::SparseTiles) && image.SparseTilesShareZeroBlock()) { + // The render would write the zeroed block all empty blocks share: each gets memory + // of its own first, and the parts that hold data are loaded again. + LOG_INFO(HW_GPU, "GPU write into sparse tiled image {:#x}; backing it whole", + image.gpu_addr); + if (BackSparseTilesWhole(image)) { + image.dirty_ranges.clear(); + RefreshSparseTiles(image); + } + } + } if constexpr (requires { &Image::BindLayers; }) { if (True(image.flags & ImageFlagBits::SparseLayers)) { std::vector layers(static_cast(image.info.resources.layers)); diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index f27e237cfa..141e5b76d5 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -173,6 +173,9 @@ class TextureCache : public VideoCommon::ChannelSetupCaches rects); + /// Image::BackTilesWhole, counting what it binds in total_used_memory. + bool BackSparseTilesWhole(Image& image); + /// Uploads the image's dirty layers. False when it needs a full upload instead. bool UploadWrittenLayers(Image& image); diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index abd31ad904..3d2408fc35 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -326,6 +326,11 @@ public: return IsSparseBindingSupported() && features.features.sparseResidencyImage2D; } + /// Whether a sparse image may bind the same memory to several of its blocks. + bool IsSparseResidencyAliasedSupported() const { + return IsSparseResidencyImage2DSupported() && features.features.sparseResidencyAliased; + } + /// Whether images with no memory bound may be sampled and used as attachments. AMD's drivers /// fault on it (the GPU hangs shortly after the first such draw), so large images there are /// backed normally. diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index a0c96d1b45..1798611e26 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -280,12 +280,15 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat device.GetDispatchLoader()); } -vk::Image MemoryAllocator::CreateUnbackedImage(const VkImageCreateInfo &ci) const { +vk::Image MemoryAllocator::CreateUnbackedImage(const VkImageCreateInfo &ci, bool aliased) const { if (ci.imageType == VK_IMAGE_TYPE_1D) { return vk::Image{}; } VkImageCreateInfo sparse_ci = ci; sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT; + if (aliased) { + sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_ALIASED_BIT; + } VkImage sparse{}; if (device.GetDispatchLoader().vkCreateImage(*device.GetLogical(), &sparse_ci, nullptr, &sparse) != VK_SUCCESS) { diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index f3c6c177a2..e1c267ab79 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -109,7 +109,8 @@ namespace Vulkan { /// A sparse resident image with nothing bound (reads return zero, writes are dropped), or /// a null image when the device can't make one. - vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci) const; + /// aliased: its sparse blocks may share memory (VK_IMAGE_CREATE_SPARSE_ALIASED_BIT). + vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci, bool aliased = false) const; /// Device memory to bind to part of a sparse resource. Null when there is none left. VmaAllocation AllocateSparseMemory(const VkMemoryRequirements &requirements, -- cgit v1.2.3 From 3d6156434a96c75e30fc4cfa65409fb2800d737a Mon Sep 17 00:00:00 2001 From: kraibse <33869366+kraibse@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:17:54 +0100 Subject: nx2: guard empty and overflowing 216 map requests, log failed whole-backing A zero-size request at a region start wrapped last_block and wrote past the region's blocks; the offset + size memory bounds check could overflow. Both are guest-controlled. Also log an error when a sparse tiled image sharing the zeroed block can't be backed whole before a GPU write. --- src/core/hle/service/nx2_gsv.h | 7 +++++-- src/video_core/texture_cache/texture_cache.h | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h index 39ef402024..628bdeda5b 100644 --- a/src/core/hle/service/nx2_gsv.h +++ b/src/core/hle/service/nx2_gsv.h @@ -941,6 +941,9 @@ private: void ApplySparseMapRequest(const SparseMapRequest& request) { ++m_tile_requests; + if (request.size == 0) { + return; + } const u64 gpu_address = request.gpu_address; const u64 size = request.size; auto& registry = Nvidia::Devices::Nx2GpuMemoryRegistry::Get(); @@ -972,7 +975,7 @@ private: } ++m_tile_maps; const auto memory = m_memory.find(request.memory_handle); - if (memory == m_memory.end() || request.offset + size > memory->second.size) { + if (memory == m_memory.end() || request.offset > memory->second.size || size > memory->second.size - request.offset) { LOG_WARNING(Service, "nx2:gpu 216 map [{:#x}, +{:#x}] of unknown memory {} offset {:#x}", gpu_address, size, request.memory_handle, request.offset); return; @@ -1050,7 +1053,7 @@ private: return; } const auto memory = m_memory.find(request.memory_handle); - if (memory == m_memory.end() || request.offset + size > memory->second.size) { + if (memory == m_memory.end() || request.offset > memory->second.size || size > memory->second.size - request.offset) { LOG_WARNING(Service, "nx2:gpu 216 map [{:#x}, +{:#x}] of unknown memory {} offset {:#x}", gpu_address, size, request.memory_handle, request.offset); return; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 5f655e5b60..33ca051482 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -3113,6 +3113,11 @@ void TextureCache

::PrepareImage(ImageId image_id, bool is_modification, bool if (BackSparseTilesWhole(image)) { image.dirty_ranges.clear(); RefreshSparseTiles(image); + } else { + LOG_ERROR(HW_GPU, + "Sparse tiled image {:#x}: out of memory backing it whole, the GPU " + "write lands in the shared zeroed block", + image.gpu_addr); } } } -- cgit v1.2.3 From f398ff2e17f4e86701b698e68886e09d387889af Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:18:26 -0500 Subject: texture_cache: free the slots of deleted allocations and framebuffers Deleting the last image of an allocation erased its lookup but kept its slot, and removing a framebuffer moved it to the deletion queue but kept its slot, so both slot vectors grew with every allocation and framebuffer ever made (title 1: 8.3k allocations and 1k framebuffers made in its first 4 minutes, 5.2k and 600 alive). --- src/video_core/texture_cache/texture_cache.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 33ca051482..e0cc2fc517 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -2982,6 +2982,7 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { alloc_images.erase(alloc_image_it); if (alloc_images.empty()) { image_allocs_table.erase(alloc_it); + slot_image_allocs.erase(alloc_id); } for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; @@ -3023,6 +3024,7 @@ void TextureCache

::RemoveFramebuffers(std::span removed_vi last_framebuffer_serial = 0; } sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id])); + slot_framebuffers.erase(framebuffer_id); it = framebuffers.erase(it); } else { ++it; -- cgit v1.2.3 From ca1fac93fb4397a89424c6621be3515949b94729 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 01:11:11 -0400 Subject: texture_cache: cold-hint the guest memory of sparse images too Images whose guest memory is not contiguous were left out of the cold hint. With sparse tile remaps Mario Kart World maps its own memory into such images, and its game memory in RAM rose by ~1.9 GB in one run. Each mapped piece is now hinted. (cherry picked from commit 97f034615c174178301a4999c334fb39cd03e568) --- src/video_core/texture_cache/texture_cache.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 8ff3e012e1..249559e7be 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -1406,8 +1406,15 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { // streams several GB of textures through it). constexpr u64 ColdHintMinBytes = 64_KiB; if (image.guest_size_bytes >= ColdHintMinBytes && - False(image.flags & (ImageFlagBits::GpuModified | ImageFlagBits::Sparse))) { - uploaded_ranges.emplace_back(image.cpu_addr, image.guest_size_bytes); + False(image.flags & ImageFlagBits::GpuModified)) { + if (True(image.flags & ImageFlagBits::Sparse)) { + // Non-contiguous: each mapped piece (title 1 maps its own memory into these). + ForEachSparseSegment(image, [&](GPUVAddr, DAddr address, size_t size) { + uploaded_ranges.emplace_back(address, size); + }); + } else { + uploaded_ranges.emplace_back(image.cpu_addr, image.guest_size_bytes); + } } } -- cgit v1.2.3 From e37b4b9fa60f02fa77a4795cdd1bed0b8c71b933 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:02:00 -0500 Subject: texture_cache: collect images in the emptiest device memory blocks first Deleting by age alone left every block partly used: the collector deletes thousands of images a minute and new ones land in the holes, so blocks seldom empty (title 1: ~0.8 GB of holes in ~190 32 MiB blocks after ten minutes, counted by the driver as used). Among the oldest candidates, 8 per image a pass may delete, those in the emptiest blocks go first, so blocks drain and VMA returns them; placement already prefers the fullest blocks. (cherry picked from commit 2f8f0716fafd874cfbb3cc9086141aea61aff2df) --- src/video_core/renderer_vulkan/vk_texture_cache.h | 5 +++ src/video_core/texture_cache/texture_cache.h | 55 ++++++++++++++++++++++- src/video_core/texture_cache/texture_cache_base.h | 4 ++ src/video_core/vulkan_common/vulkan_wrapper.cpp | 13 ++++++ src/video_core/vulkan_common/vulkan_wrapper.h | 9 ++++ 5 files changed, 84 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index efdb705e93..e7ae9752a8 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -387,6 +387,11 @@ public: return (this->*current_image).UsageFlags(); } + /// Where the image's memory lies, for the collector (TextureCache::RunGarbageCollector). + [[nodiscard]] vk::Image::MemoryBlock MemoryBlock() const noexcept { + return original_image.GetMemoryBlock(); + } + /// Returns true when the image is already initialized and mark it as initialized [[nodiscard]] bool ExchangeInitialization() noexcept { return std::exchange(initialized, true); diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 249559e7be..c623c7d585 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -175,11 +175,62 @@ void TextureCache

::RunGarbageCollector() { } return false; }; + // Deleting by age alone left every device memory block partly used: the collector deletes + // thousands of images a minute and new ones land in the holes, so blocks seldom empty (title + // 1: ~0.8 GB of holes in ~190 32 MiB blocks after ten minutes, which the driver counts as + // used). Among the oldest candidates (8 per image this pass may delete), those in the + // emptiest blocks go first, so blocks drain and go back to the driver; placement already + // prefers the fullest blocks (best fit). Images with memory of their own sit in the middle. + const auto Collect = [&] { + if constexpr (requires(const Image& image) { image.MemoryBlock(); }) { + std::vector candidates; + const size_t limit = num_iterations * 8; + lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, [&](ImageId image_id) { + candidates.push_back(image_id); + return candidates.size() >= limit; + }); + if (candidates.empty()) { + return; + } + if (block_image_bytes_frame == ~0ULL || frame_tick - block_image_bytes_frame >= 60) { + block_image_bytes_frame = frame_tick; + block_image_bytes.clear(); + for (const auto [image_id, image] : slot_images) { + const auto block = image->MemoryBlock(); + if (block.block != 0) { + block_image_bytes[block.block] += block.size; + } + } + } + std::vector> ordered; + ordered.reserve(candidates.size()); + for (const ImageId image_id : candidates) { + const auto block = slot_images[image_id].MemoryBlock(); + double fill = 0.5; + if (block.block != 0 && block.block_size != 0) { + const auto it = block_image_bytes.find(block.block); + fill = it != block_image_bytes.end() + ? static_cast(it->second) / static_cast(block.block_size) + : 0.0; + } + ordered.emplace_back(fill, image_id); + } + std::stable_sort(ordered.begin(), ordered.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + for (const auto& [fill, image_id] : ordered) { + if (Cleanup(image_id)) { + break; + } + } + } else { + lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + } + }; Configure(false); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + Collect(); if (total_used_memory >= critical_memory) { Configure(true); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + Collect(); } } diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index af9d4f05a5..36702a9bff 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -588,6 +588,10 @@ private: }; std::deque gpu_sentenced_images; u64 gpu_sentenced_bytes{}; + + /// Bytes of images per device memory block, for the collector (refreshed every 60 frames). + std::unordered_map block_image_bytes; + u64 block_image_bytes_frame{~0ULL}; DelayedDestructionRing sentenced_image_view; DelayedDestructionRing sentenced_framebuffers; diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index ac67b95707..d0bca88125 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -526,6 +526,19 @@ void Image::Release() const noexcept { } } +Image::MemoryBlock Image::GetMemoryBlock() const noexcept { + if (allocation == nullptr) { + return {}; + } + VmaAllocationInfo2 info{}; + vmaGetAllocationInfo2(allocator, allocation, &info); + if (info.dedicatedMemory) { + return {0, info.blockSize, info.allocationInfo.size}; + } + return {reinterpret_cast(info.allocationInfo.deviceMemory), info.blockSize, + info.allocationInfo.size}; +} + void Buffer::Flush() const { if (!is_coherent) { vmaFlushAllocation(allocator, allocation, 0, VK_WHOLE_SIZE); diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index f9b3cc15df..f49e9518bb 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -737,6 +737,15 @@ public: return usage; } + /// Where the image's memory lies: the block (0 when the image has no memory or a block of + /// its own), the block's size and the image's part of it. + struct MemoryBlock { + u64 block; + u64 block_size; + u64 size; + }; + [[nodiscard]] MemoryBlock GetMemoryBlock() const noexcept; + private: void Release() const noexcept; -- cgit v1.2.3 From 16967eb30e97937f4ac3b200e681adde70089161 Mon Sep 17 00:00:00 2001 From: kraibse <33869366+kraibse@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:19:37 +0100 Subject: vulkan: don't make small images unbacked when device memory runs out When VRAM ran out, any image that failed to allocate became a sparse image with nothing bound, which AMD's drivers don't survive being sampled (an RX 6600 died mid-flood, an RX 7800 XT made 211 of them, 8x8 ones included). Try host memory first, and count the images that didn't fit: the texture cache then keeps more room under the budget (512 MiB a step, up to 3 GiB, decaying) so its collector runs earlier. The sparse fallback stays for images too large to back. --- src/video_core/renderer_vulkan/vk_texture_cache.h | 5 +++++ src/video_core/texture_cache/texture_cache.h | 23 ++++++++++++++++++++-- src/video_core/texture_cache/texture_cache_base.h | 3 +++ .../vulkan_common/vulkan_memory_allocator.cpp | 21 ++++++++++++++++++++ .../vulkan_common/vulkan_memory_allocator.h | 8 ++++++++ 5 files changed, 58 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index e7ae9752a8..5ac833e787 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -76,6 +76,11 @@ public: bool CanReportMemoryUsage() const; + /// Images that did not fit the device memory budget so far (see MemoryAllocator). + u32 MemoryPressureEvents() const noexcept { + return memory_allocator.MemoryPressureEvents(); + } + void LogMemoryStatistics() const; /// True when ASTC is re-compressed to BC3 and opaque images may use BC1 instead. diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index c623c7d585..cf042c1488 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -289,13 +289,32 @@ void TextureCache

::TickFrame() { // budget as the card fills (16 GB card: 13.5 GB left mid-race), hence the re-read every 30 // frames. EDEN_NX2_TC_*_MIB overrides still win. if constexpr (requires { runtime.GetDeviceMemoryBudget(); }) { + if constexpr (requires { runtime.MemoryPressureEvents(); }) { + // Images that did not fit (made in host memory or with nothing bound) mean the + // thresholds sit too high for this card and driver: leave more room until the + // pressure has been gone for a while. + if (native_thresholds_applied) { + const u32 events = runtime.MemoryPressureEvents(); + if (events != seen_pressure_events) { + seen_pressure_events = events; + pressure_margin = (std::min)(pressure_margin + 512_MiB, 3_GiB); + pressure_quiet_frames = 0; + budget_check_frames = 0; // thresholds are recomputed below + LOG_WARNING(HW_GPU, "Out of device memory: keeping {} MiB more free", + pressure_margin >> 20); + } else if (pressure_margin != 0 && ++pressure_quiet_frames % 600 == 0) { + pressure_margin -= (std::min)(pressure_margin, 128_MiB); + budget_check_frames = 0; + } + } + } if (native_thresholds_applied && budget_check_frames++ % 30 == 0) { const u64 budget = runtime.GetDeviceMemoryBudget(); if (budget > 2_GiB) { const u64 old_critical = critical_memory; const u64 margin = (std::min)(budget / 8, 1_GiB); - critical_memory = budget - margin; - expected_memory = budget - 2 * margin; + critical_memory = budget - (std::min)(margin + pressure_margin, budget / 2); + expected_memory = critical_memory - (std::min)(margin, critical_memory / 2); // Log the first values and real changes (the budget moves by a few MiB all // the time). const u64 change = critical_memory > old_critical diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 36702a9bff..1a6931d18a 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -136,6 +136,9 @@ class TextureCache : public VideoCommon::ChannelSetupCaches astc_scan_buffer; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 79fb331de0..a0a39abe39 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -245,11 +245,32 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, ci.arrayLayers, static_cast(ci.format)); } + pressure_events.fetch_add(1, std::memory_order_relaxed); VmaAllocationCreateInfo relaxed = alloc_ci; relaxed.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; result = vmaCreateImage(allocator, &ci, &relaxed, &handle, &allocation, &alloc_info); } + if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && !huge) { + // Still out of device memory: a real image in host memory (slow, but valid) beats one + // with nothing bound, which AMD's drivers do not survive being sampled. + VmaAllocationCreateInfo host = alloc_ci; + host.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + host.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST; + host.preferredFlags = 0; + result = vmaCreateImage(allocator, &ci, &host, &handle, &allocation, &alloc_info); + static u32 host_logged{}; + if (result == VK_SUCCESS && host_logged++ < 16) { + LOG_WARNING(Render_Vulkan, + "Image {}x{}x{} ({} levels, {} layers, format {}) is out of device memory; " + "placed in host memory", + ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, + ci.arrayLayers, static_cast(ci.format)); + } + } if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && ci.imageType != VK_IMAGE_TYPE_1D) { + if (!huge) { + pressure_events.fetch_add(1, std::memory_order_relaxed); + } // Too big to back at all, e.g. title 1's sparse virtual texture arrays (8192x8192, // 68 layers, ~23 GB): on the console only resident tiles have memory. Use an image with // nothing bound, so views and copies still work, instead of aborting. diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index ffca933ee7..8b9e72b3e3 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -126,6 +127,12 @@ namespace Vulkan { void FreeSparseMemory(VmaAllocation allocation) const; + /// Counts the images that did not fit the device memory budget (made anyway, in host + /// memory or with nothing bound): the texture cache collects harder when it grows. + u32 MemoryPressureEvents() const noexcept { + return pressure_events.load(std::memory_order_relaxed); + } + vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const; vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage, @@ -161,6 +168,7 @@ namespace Vulkan { const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties. VkDeviceSize buffer_image_granularity; ///< Adjacent buffer/image granularity u32 valid_memory_types{~0u}; + mutable std::atomic pressure_events{0}; mutable std::mutex buffer_type_mutex; /// Memory type of device-local buffers by usage and flags (CreateBufferInSeparatePool). mutable std::unordered_map buffer_memory_types; -- cgit v1.2.3 From 51d798ff0373a40098aa77ef13d54307f39e44ac Mon Sep 17 00:00:00 2001 From: kraibse <33869366+kraibse@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:38:05 +0100 Subject: texture_cache: keep the allocator's block slack out of the usable VRAM The collector's usage leaves out the unused part of VMA's blocks, but the driver counts it and a heap at its budget cannot give it to a new image. On an RX 7800 XT that slack was 1.06 GiB, the whole 1 GiB margin: the heap sat at its budget for a minute while the collector saw room, then images stopped fitting and the process died. Reserve the slack (capped at 1.5 GiB) on top of the margin: at once when it grows, slowly when it shrinks, so eviction leaving partly used blocks doesn't chase the threshold down. Assisted-by: Claude Sonnet 5.5 (diagnosis and code) (cherry picked from commit 73fd5a489ea28226fa16a55f5008e622d631b54c) --- .../renderer_vulkan/vk_texture_cache.cpp | 4 ++++ src/video_core/renderer_vulkan/vk_texture_cache.h | 2 ++ src/video_core/texture_cache/texture_cache.h | 23 ++++++++++++++++++++-- src/video_core/texture_cache/texture_cache_base.h | 1 + src/video_core/vulkan_common/vulkan_device.cpp | 10 ++++++++++ src/video_core/vulkan_common/vulkan_device.h | 8 +++++++- 6 files changed, 45 insertions(+), 3 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 09e7961ff9..cb0453c178 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -2082,6 +2082,10 @@ u64 TextureCacheRuntime::GetDeviceMemoryBudget() const { return device.GetDeviceMemoryBudget(); } +u64 TextureCacheRuntime::GetDeviceMemorySlack() const { + return device.GetDeviceMemorySlack(); +} + bool TextureCacheRuntime::CanDownloadMsaa(const VideoCommon::ImageInfo& info) const { const VkImageAspectFlags aspect_mask = ImageAspectMask(info.format); if ((aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0) { diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 5ac833e787..d5f8756486 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -74,6 +74,8 @@ public: u64 GetDeviceMemoryBudget() const; + u64 GetDeviceMemorySlack() const; + bool CanReportMemoryUsage() const; /// Images that did not fit the device memory budget so far (see MemoryAllocator). diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index cf042c1488..45bc35dee6 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -313,14 +313,33 @@ void TextureCache

::TickFrame() { if (budget > 2_GiB) { const u64 old_critical = critical_memory; const u64 margin = (std::min)(budget / 8, 1_GiB); - critical_memory = budget - (std::min)(margin + pressure_margin, budget / 2); + if constexpr (requires { runtime.GetDeviceMemorySlack(); }) { + // The usage counted here leaves out the unused part of the allocator's + // blocks, which the driver counts and a full heap cannot hand to a new + // image (RX 7800 XT: 1.06 GiB of it, the whole margin, with the heap at + // its budget for a minute before images stopped fitting). Reserve it + // too: at once when it grows, slowly when it shrinks, so evicting images + // (which leaves partly used blocks) does not chase the threshold down. + const u64 slack = (std::min)(runtime.GetDeviceMemorySlack(), 1536_MiB); + slack_margin = slack >= slack_margin + ? slack + : slack_margin - (std::min)(slack_margin - slack, 16_MiB); + } + critical_memory = + budget - (std::min)(margin + pressure_margin + slack_margin, budget / 2); expected_memory = critical_memory - (std::min)(margin, critical_memory / 2); // Log the first values and real changes (the budget moves by a few MiB all // the time). const u64 change = critical_memory > old_critical ? critical_memory - old_critical : old_critical - critical_memory; - ApplyThresholdOverrides(budget_check_frames == 1 || change >= 256_MiB); + const bool log = budget_check_frames == 1 || change >= 256_MiB; + ApplyThresholdOverrides(log); + if (log) { + LOG_INFO(HW_GPU, + "Memory margins: base {} MiB, out of memory {} MiB, block slack {} MiB", + margin >> 20, pressure_margin >> 20, slack_margin >> 20); + } } } } diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 1a6931d18a..24ebaf0689 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -139,6 +139,7 @@ class TextureCache : public VideoCommon::ChannelSetupCaches astc_scan_buffer; diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index 2ecbd2cb94..01e13cb384 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -1684,6 +1684,16 @@ u64 Device::GetDeviceMemoryUsage() const { return result; } +u64 Device::GetDeviceMemorySlack() const { + if (allocator == nullptr) { + return 0; + } + std::array heap_budgets{}; + vmaGetHeapBudgets(allocator, heap_budgets.data()); + const VmaStatistics& statistics = heap_budgets[main_heap].statistics; + return statistics.blockBytes - (std::min)(statistics.blockBytes, statistics.allocationBytes); +} + u64 Device::GetDeviceMemoryBudget() const { if (!extensions.memory_budget) { return 0; diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index 2d19321145..622d792210 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -1019,9 +1019,15 @@ FN_MAX_LIMIT_LIST /// The driver's current memory budget for this process (VK_EXT_memory_budget), in /// GetDeviceMemoryUsage() terms: the usage at which the main device-local heap reaches its - /// budget. 0 without the extension. + /// budget. 0 without the extension. The unused part of the allocator's blocks is not + /// subtracted: see GetDeviceMemorySlack(). u64 GetDeviceMemoryBudget() const; + /// Memory the allocator holds from the driver in the main device-local heap but has no + /// allocation in (the unused part of its blocks). GetDeviceMemoryUsage() does not count it, + /// yet the driver does: with the heap at its budget it is not available to new images. + u64 GetDeviceMemorySlack() const; + u32 GetSetsPerPool() const { return sets_per_pool; } -- cgit v1.2.3 From 92d49d8a0c3e60894765bc2c332fdc936c87fc45 Mon Sep 17 00:00:00 2001 From: Duck912 <350528965+Duck912@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:43:28 -0500 Subject: video_core: temporal upscale stand-in for Fortune's Weave's DLSS The one-frame blit left 3D edges aliased and shaking: the game jitters its camera every frame for DLSS to accumulate. Fire Emblem: Fortune's Weave's first DLSS pass (also rejected) has the motion vectors (cbuf0+0x290, R16G16) and the frame's jitter (+0x1b0/+0x1b4, in input pixels). Kepler compute keeps them, and the stand-in for the last network pass now draws temporal_upscale.frag: input samples placed where the jitter put them and weighted by distance, blended with the previous result reprojected along the motion vectors (Catmull-Rom) and clipped to the 3x3 neighbourhood in YCoCg. The history is copied from the result right after drawing it, so post-processing the game later applies to its output does not feed back. EDEN_NX2_TEMPORAL_UPSCALE=0 restores the blit. Assisted-by: Claude Opus 5.5 (debugging, implementation and drafting) Signed-off-by: Duck912 <350528965+Duck912@users.noreply.github.com> --- src/video_core/engines/kepler_compute.cpp | 47 ++++++- src/video_core/engines/kepler_compute.h | 8 ++ src/video_core/host_shaders/CMakeLists.txt | 1 + src/video_core/host_shaders/temporal_upscale.frag | 132 ++++++++++++++++++ src/video_core/rasterizer_interface.h | 11 ++ src/video_core/renderer_vulkan/blit_image.cpp | 147 ++++++++++++++++++++ src/video_core/renderer_vulkan/blit_image.h | 15 +++ src/video_core/renderer_vulkan/vk_rasterizer.cpp | 8 ++ src/video_core/renderer_vulkan/vk_rasterizer.h | 5 + .../renderer_vulkan/vk_texture_cache.cpp | 148 +++++++++++++++++++++ src/video_core/renderer_vulkan/vk_texture_cache.h | 15 +++ src/video_core/texture_cache/texture_cache.h | 67 ++++++++++ src/video_core/texture_cache/texture_cache_base.h | 7 + 13 files changed, 610 insertions(+), 1 deletion(-) create mode 100644 src/video_core/host_shaders/temporal_upscale.frag (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 233304d273..b8a6567907 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -557,6 +558,7 @@ bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& // The console runs this on tensor cores; stand in with a plain upscale of its input. return UpscaleInsteadOfNativeProgram(); } + RememberDlssMotionVectors(); return false; } return true; @@ -588,6 +590,30 @@ void KeplerCompute::DescribeNativeResources(const NativeQmdScheduler::Descriptor std::atomic g_nx2_trace_draws_after_upscale{}; +// Fire Emblem: Fortune's Weave's first DLSS pass (rejected, before the network) has the input colour +// at cbuf0+0x278, its motion vectors (R16G16 float, the input's size) at +0x290 and this frame's +// camera jitter (x, y; input pixels, Halton bases 2 and 3 less 0.5) at +0x1b0. +void KeplerCompute::RememberDlssMotionVectors() { + const auto& cbuf = launch_description.const_buffer_config[0]; + if (cbuf.size.Value() < 0x294) return; + const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low; + const auto image = [&](u32 offset) -> std::optional { + const u32 tic_index = memory_manager.Read(cbuf_address + offset) & 0xfffff; + if (tic_index > regs.tic.limit) return std::nullopt; + return GetTICEntry(tic_index); + }; + const auto color = image(0x278), motion = image(0x290); + if (!color || !motion || color->format != Texture::TextureFormat::B10G11R11 || + motion->format != Texture::TextureFormat::R16G16 || motion->Address() == 0 || + motion->Width() != color->Width() || motion->Height() != color->Height() || + !motion->IsBlockLinear()) + return; + dlss_motion_vectors = *motion; + dlss_motion_color = color->Address(); + dlss_jitter = {std::bit_cast(memory_manager.Read(cbuf_address + 0x1b0)), + std::bit_cast(memory_manager.Read(cbuf_address + 0x1b4))}; +} + // Title 4's upscaler reads the tonemapped frame through the texture handle at cbuf0+0x90 and // writes the output resolution image through the surface handle at cbuf0+0xa0 (from its // TLD/SUST operands). Fire Emblem: Fortune's Weave's DLSS network ends with a pass that reads the @@ -609,10 +635,12 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() { output->IsBlockLinear(); }; auto input = image(0x90), output = image(0xa0); + bool fortunes_weave = false; if (!usable(input, output)) { input = image(0x328); output = image(0x348); if (!usable(input, output)) return false; + fortunes_weave = true; } using RTFormat = Tegra::RenderTargetFormat; RTFormat format{}; @@ -709,7 +737,24 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() { source.format = static_cast(std::strtoul(value, nullptr, 16)); } } - const bool done = rasterizer->AccelerateSurfaceCopy(source, surface(*output), config); + // Fortune's Weave: a temporal upscale along its motion vectors that undoes this frame's camera + // jitter (both from the frame's first DLSS pass) and accumulates the frames as DLSS would. + // EDEN_NX2_TEMPORAL_UPSCALE=0 blits instead. + const auto upscale_temporally = [&] { + static const bool enabled = [] { + const char* value = std::getenv("EDEN_NX2_TEMPORAL_UPSCALE"); + return value == nullptr || std::string_view{value} != "0"; + }(); + if (!fortunes_weave || !enabled || !dlss_motion_vectors || + dlss_motion_color != input->Address()) + return false; + Fermi2D::Surface motion = surface(*dlss_motion_vectors); + motion.format = RTFormat::R16G16_FLOAT; + return rasterizer->AccelerateTemporalUpscale(source, surface(*output), motion, dlss_jitter, + {1.0f, 1.0f}); + }; + const bool done = upscale_temporally() || + rasterizer->AccelerateSurfaceCopy(source, surface(*output), config); if (static u32 checked{}; checked++ == sample_at) { const u64 size = u64{output->Width()} * output->Height() * 4; if (const auto device = memory_manager.GpuToCpuAddress(output->Address())) { diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index b525636dde..b5aee3706a 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -266,7 +266,15 @@ private: bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor); void ProcessNativeReportSemaphore(u32 argument); void DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd); + void RememberDlssMotionVectors(); bool UpscaleInsteadOfNativeProgram(); + + /// Motion vectors of Fire Emblem: Fortune's Weave's DLSS evaluation, from its first pass (run + /// earlier in the frame than the network pass the stand-in replaces), with the input colour + /// image and camera jitter they go with. + std::optional dlss_motion_vectors; + GPUVAddr dlss_motion_color{}; + std::array dlss_jitter{}; bool ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call); NativeQmdScheduler native_qmd; std::array* native_data_ram{}; diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index 35b350c2cd..ada5c779dc 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -71,6 +71,7 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/smaa_blending_weight_calculation.frag ${CMAKE_CURRENT_SOURCE_DIR}/smaa_neighborhood_blending.vert ${CMAKE_CURRENT_SOURCE_DIR}/smaa_neighborhood_blending.frag + ${CMAKE_CURRENT_SOURCE_DIR}/temporal_upscale.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_blit_depth_stencil.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_color_clear.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_color_clear.vert diff --git a/src/video_core/host_shaders/temporal_upscale.frag b/src/video_core/host_shaders/temporal_upscale.frag new file mode 100644 index 0000000000..0e76df2b43 --- /dev/null +++ b/src/video_core/host_shaders/temporal_upscale.frag @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// Stands in for a game's DLSS evaluation (the tensor-core network cannot run here): a temporal +// upscale. Each frame's input samples are placed where the camera jitter put them and weighted by +// their distance to the output pixel; they accumulate with the previous output, reprojected along +// the game's motion vectors (bicubic) and clipped to the current 3x3 neighbourhood. + +#version 450 + +layout(binding = 0) uniform sampler2D input_color; +layout(binding = 1) uniform sampler2D motion_vectors; +layout(binding = 2) uniform sampler2D history; + +layout(push_constant) uniform PushConstants { + // The vertex stage's texture scale and offset come first. + layout(offset = 16) vec2 jitter; // this frame's sample offset, in input pixels + vec2 motion_scale; // motion vector -> UV offset to the previous frame + float current_weight; // largest weight of this frame; 1 ignores the history +}; + +layout(location = 0) in vec2 texcoord; // 0 to 1 across the output +layout(location = 0) out vec4 color; + +// The blit helper's samplers take unnormalized (texel) coordinates. + +// Blending in a range-compressed space keeps bright pixels from dominating the average. +vec3 Compress(vec3 c) { + return c / (1.0 + max(max(c.r, c.g), c.b)); +} + +vec3 Expand(vec3 c) { + c = clamp(c, vec3(0.0), vec3(1.0 - 1.0 / 65504.0)); + return c / (1.0 - max(max(c.r, c.g), c.b)); +} + +vec3 RgbToYCoCg(vec3 c) { + return vec3(dot(c, vec3(0.25, 0.5, 0.25)), dot(c, vec3(0.5, 0.0, -0.5)), + dot(c, vec3(-0.25, 0.5, -0.25))); +} + +vec3 YCoCgToRgb(vec3 c) { + return vec3(c.x + c.y - c.z, c.x + c.z, c.x - c.y - c.z); +} + +// Moves value towards the box centre until it lies inside the box. +vec3 ClipToBox(vec3 value, vec3 low, vec3 high) { + const vec3 center = 0.5 * (low + high); + const vec3 extent = 0.5 * (high - low) + 1.0 / 65504.0; + const vec3 offset = value - center; + const vec3 units = abs(offset / extent); + const float largest = max(units.x, max(units.y, units.z)); + return largest > 1.0 ? center + offset / largest : value; +} + +// Catmull-Rom filtered history from five bilinear fetches: repeated bilinear reprojection would +// blur the accumulated image a little more every frame. +vec3 SampleHistory(vec2 position) { + const vec2 texel1 = floor(position - 0.5) + 0.5; + const vec2 f = position - texel1; + const vec2 w0 = f * (-0.5 + f * (1.0 - 0.5 * f)); + const vec2 w1 = 1.0 + f * f * (-2.5 + 1.5 * f); + const vec2 w2 = f * (0.5 + f * (2.0 - 1.5 * f)); + const vec2 w3 = f * f * (-0.5 + 0.5 * f); + const vec2 w12 = w1 + w2; + const vec2 texel0 = texel1 - 1.0; + const vec2 texel3 = texel1 + 2.0; + const vec2 texel12 = texel1 + w2 / w12; + vec3 sum = texture(history, vec2(texel12.x, texel0.y)).rgb * (w12.x * w0.y); + sum += texture(history, vec2(texel0.x, texel12.y)).rgb * (w0.x * w12.y); + sum += texture(history, texel12).rgb * (w12.x * w12.y); + sum += texture(history, vec2(texel3.x, texel12.y)).rgb * (w3.x * w12.y); + sum += texture(history, vec2(texel12.x, texel3.y)).rgb * (w12.x * w3.y); + const float weight = w12.x * w0.y + w0.x * w12.y + w12.x * w12.y + w3.x * w12.y + w12.x * w3.y; + return max(sum / weight, vec3(0.0)); +} + +void main() { + const ivec2 input_size = textureSize(input_color, 0); + const ivec2 motion_size = textureSize(motion_vectors, 0); + // The output pixel's centre in input pixels. Input texel k sampled the scene at k + 0.5 - jitter + // (the sign measured against consecutive frames). + const vec2 target = texcoord * vec2(input_size); + const ivec2 nearest = ivec2(floor(target + jitter)); + + vec3 filtered = vec3(0.0); + float total_weight = 0.0; + float nearest_weight = 0.0; + vec3 sum = vec3(0.0); + vec3 squares = vec3(0.0); + vec2 motion = vec2(0.0); + float longest = -1.0; + for (int y = -1; y <= 1; ++y) { + for (int x = -1; x <= 1; ++x) { + const ivec2 texel = clamp(nearest + ivec2(x, y), ivec2(0), input_size - 1); + const vec3 sample_color = RgbToYCoCg(Compress(texelFetch(input_color, texel, 0).rgb)); + // A Blackman-Harris-like window over the sample's distance to the output pixel. + const vec2 distance = vec2(texel) + 0.5 - jitter - target; + const float weight = exp(-2.29 * dot(distance, distance)); + filtered += sample_color * weight; + total_weight += weight; + nearest_weight = max(nearest_weight, weight); + sum += sample_color; + squares += sample_color * sample_color; + // The longest motion around the pixel, so that an edge follows the object in front. + const vec2 candidate = + texelFetch(motion_vectors, clamp(texel, ivec2(0), motion_size - 1), 0).rg; + const float length_squared = dot(candidate, candidate); + if (length_squared > longest) { + longest = length_squared; + motion = candidate; + } + } + } + const vec3 current = filtered / total_weight; + const vec3 mean = sum / 9.0; + const vec3 deviation = sqrt(max(squares / 9.0 - mean * mean, vec3(0.0))); + const vec3 low = mean - deviation; + const vec3 high = mean + deviation; + + const vec2 previous = texcoord + motion * motion_scale; + // A frame whose samples land close to this pixel counts more; one whose samples land between + // pixels mostly keeps the history (from other jitter positions) there. + float weight = current_weight * nearest_weight; + if (current_weight >= 1.0 || any(lessThan(previous, vec2(0.0))) || + any(greaterThan(previous, vec2(1.0)))) { + weight = 1.0; + } + const vec2 history_position = previous * vec2(textureSize(history, 0)); + const vec3 past = ClipToBox(RgbToYCoCg(Compress(SampleHistory(history_position))), low, high); + color = vec4(Expand(YCoCgToRgb(mix(past, current, weight))), 1.0); +} diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index b527c0e2a9..8486d44ac9 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -162,6 +163,16 @@ public: return false; } + /// Stand-in for a DLSS evaluation: temporal upscale of src into dst along motion_vectors + /// (jitter in src pixels; motion_scale turns a motion vector into a UV offset to the previous + /// frame). False when unsupported; the caller then falls back to AccelerateSurfaceCopy. + [[nodiscard]] virtual bool AccelerateTemporalUpscale( + const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, std::array jitter, + std::array motion_scale) { + return false; + } + [[nodiscard]] virtual Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() = 0; virtual void AccelerateInlineToMemory(GPUVAddr address, size_t copy_size, diff --git a/src/video_core/renderer_vulkan/blit_image.cpp b/src/video_core/renderer_vulkan/blit_image.cpp index ddd52e9426..edd151b563 100644 --- a/src/video_core/renderer_vulkan/blit_image.cpp +++ b/src/video_core/renderer_vulkan/blit_image.cpp @@ -33,6 +33,7 @@ #include "video_core/host_shaders/convert_non_msaa_to_msaa_depth_stencil_frag_spv.h" #include "video_core/host_shaders/convert_s8d24_to_abgr8_frag_spv.h" #include "video_core/host_shaders/full_screen_triangle_vert_spv.h" +#include "video_core/host_shaders/temporal_upscale_frag_spv.h" #include "video_core/host_shaders/vulkan_blit_depth_stencil_frag_spv.h" #include "video_core/host_shaders/vulkan_color_clear_frag_spv.h" #include "video_core/host_shaders/vulkan_color_clear_vert_spv.h" @@ -91,6 +92,15 @@ struct PushConstants { std::array tex_offset; }; +// The vertex stage's PushConstants first, then those of temporal_upscale.frag. +struct TemporalUpscalePushConstants { + PushConstants vertex; + std::array jitter; + std::array motion_scale; + float current_weight; + float padding; +}; + struct MSAACopyPushConstants { std::array dst_offset; std::array src_offset; @@ -109,6 +119,11 @@ constexpr std::array TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{ TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>, TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>, }; +constexpr std::array THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{ + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>, + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>, + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<2>, +}; constexpr VkDescriptorSetLayoutCreateInfo ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{ .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .pNext = nullptr, @@ -133,6 +148,13 @@ constexpr VkDescriptorSetLayoutCreateInfo TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CRE .bindingCount = static_cast(TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()), .pBindings = TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(), }; +constexpr VkDescriptorSetLayoutCreateInfo THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{ + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .bindingCount = static_cast(THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()), + .pBindings = THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(), +}; template inline constexpr VkPushConstantRange PUSH_CONSTANT_RANGE{ .stageFlags = stageFlags, @@ -357,6 +379,32 @@ void UpdateOneTextureDescriptorSet(const Device& device, VkDescriptorSet descrip device.GetLogical().UpdateDescriptorSets(write_descriptor_set, nullptr); } +void UpdateThreeTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set, + VkSampler sampler, std::array image_views) { + std::array image_infos{}; + std::array write_descriptor_sets{}; + for (u32 binding = 0; binding < 3; ++binding) { + image_infos[binding] = VkDescriptorImageInfo{ + .sampler = sampler, + .imageView = image_views[binding], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + }; + write_descriptor_sets[binding] = VkWriteDescriptorSet{ + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = nullptr, + .dstSet = descriptor_set, + .dstBinding = binding, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImageInfo = &image_infos[binding], + .pBufferInfo = nullptr, + .pTexelBufferView = nullptr, + }; + } + device.GetLogical().UpdateDescriptorSets(write_descriptor_sets, nullptr); +} + void UpdateTwoTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set, VkSampler sampler, VkImageView image_view_0, VkImageView image_view_1) { @@ -590,10 +638,14 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), two_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout( TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), + three_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout( + THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), one_texture_descriptor_allocator{ descriptor_pool.Allocator(device_, scheduler_, *one_texture_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<1>)}, two_textures_descriptor_allocator{ descriptor_pool.Allocator(device_, scheduler_, *two_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<2>)}, + three_textures_descriptor_allocator{ + descriptor_pool.Allocator(device_, scheduler_, *three_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<3>)}, one_texture_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( one_texture_set_layout.address(), PUSH_CONSTANT_RANGE))), @@ -601,6 +653,11 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( two_textures_set_layout.address(), PUSH_CONSTANT_RANGE))), + temporal_upscale_pipeline_layout( + device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( + three_textures_set_layout.address(), + PUSH_CONSTANT_RANGE))), clear_color_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( nullptr, PUSH_CONSTANT_RANGE))), msaa_copy_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( @@ -612,6 +669,7 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, PUSH_CONSTANT_RANGE))), full_screen_vert(BuildShader(device, FULL_SCREEN_TRIANGLE_VERT_SPV)), blit_color_to_color_frag(BuildShader(device, BLIT_COLOR_FLOAT_FRAG_SPV)), + temporal_upscale_frag(BuildShader(device, TEMPORAL_UPSCALE_FRAG_SPV)), blit_color_msaa_frag(BuildShader(device, BLIT_COLOR_MSAA_FRAG_SPV)), blit_depth_stencil_frag(device.IsExtShaderStencilExportSupported() ? BuildShader(device, VULKAN_BLIT_DEPTH_STENCIL_FRAG_SPV) @@ -702,6 +760,42 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView }); } +void BlitImageHelper::TemporalUpscale(const Framebuffer* dst_framebuffer, + const ImageView& input_color, const ImageView& motion_vectors, + VkImageView history, const Region2D& dst_region, + std::array jitter, std::array motion_scale, + f32 current_weight) { + const VkPipelineLayout layout = *temporal_upscale_pipeline_layout; + const VkPipeline pipeline = FindOrEmplaceTemporalUpscalePipeline(dst_framebuffer->RenderPass()); + const VkSampler sampler = *linear_sampler; + const std::array views{input_color.Handle(Shader::TextureType::Color2D), + motion_vectors.Handle(Shader::TextureType::Color2D), + history}; + const TemporalUpscalePushConstants push_constants{ + .vertex = {.tex_scale = {1.0f, 1.0f}, .tex_offset = {0.0f, 0.0f}}, + .jitter = jitter, + .motion_scale = motion_scale, + .current_weight = current_weight, + .padding = 0.0f, + }; + RecordShaderReadBarrier(scheduler, input_color); + RecordShaderReadBarrier(scheduler, motion_vectors); + scheduler.RequestRenderpass(dst_framebuffer); + scheduler.Record([this, dst_region, pipeline, layout, sampler, views, + push_constants](vk::CommandBuffer cmdbuf) { + const VkDescriptorSet descriptor_set = three_textures_descriptor_allocator.Commit(); + UpdateThreeTexturesDescriptorSet(device, descriptor_set, sampler, views); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, descriptor_set, + nullptr); + BindBlitState(cmdbuf, dst_region); + cmdbuf.PushConstants(layout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + push_constants); + cmdbuf.Draw(3, 1, 0, 0); + }); + scheduler.InvalidateState(); +} + void BlitImageHelper::BlitImpl(const Framebuffer* dst_framebuffer, const ImageView& src_image_view, const Region2D& dst_region, const Region2D& src_region, VkPipeline pipeline, VkSampler sampler, @@ -1285,6 +1379,59 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe return *blit_color_pipelines.back(); } +VkPipeline BlitImageHelper::FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass) { + const auto it = std::ranges::find(temporal_upscale_keys, renderpass); + if (it != temporal_upscale_keys.end()) { + return *temporal_upscale_pipelines[std::distance(temporal_upscale_keys.begin(), it)]; + } + temporal_upscale_keys.push_back(renderpass); + const std::array stages = MakeStages(*full_screen_vert, *temporal_upscale_frag); + const VkPipelineColorBlendAttachmentState blend_attachment{ + .blendEnable = VK_FALSE, + .srcColorBlendFactor = VK_BLEND_FACTOR_ZERO, + .dstColorBlendFactor = VK_BLEND_FACTOR_ZERO, + .colorBlendOp = VK_BLEND_OP_ADD, + .srcAlphaBlendFactor = VK_BLEND_FACTOR_ZERO, + .dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO, + .alphaBlendOp = VK_BLEND_OP_ADD, + .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | + VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, + }; + const VkPipelineColorBlendStateCreateInfo color_blend_create_info{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .logicOpEnable = VK_FALSE, + .logicOp = VK_LOGIC_OP_CLEAR, + .attachmentCount = 1, + .pAttachments = &blend_attachment, + .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, + }; + const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + temporal_upscale_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ + .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .stageCount = static_cast(stages.size()), + .pStages = stages.data(), + .pVertexInputState = &PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, + .pInputAssemblyState = &input_assembly_ci, + .pTessellationState = nullptr, + .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, + .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, + .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, + .pDepthStencilState = nullptr, + .pColorBlendState = &color_blend_create_info, + .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, + .layout = *temporal_upscale_pipeline_layout, + .renderPass = renderpass, + .subpass = 0, + .basePipelineHandle = VK_NULL_HANDLE, + .basePipelineIndex = 0, + }, device.StaticPipelineCache())); + return *temporal_upscale_pipelines.back(); +} + VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key) { const auto it = std::ranges::find(blit_depth_stencil_keys, key); if (it != blit_depth_stencil_keys.end()) { diff --git a/src/video_core/renderer_vulkan/blit_image.h b/src/video_core/renderer_vulkan/blit_image.h index 62e6743a23..303beeea5c 100644 --- a/src/video_core/renderer_vulkan/blit_image.h +++ b/src/video_core/renderer_vulkan/blit_image.h @@ -82,6 +82,13 @@ public: VkImage src_image, VkSampler src_sampler, const Region2D& dst_region, const Region2D& src_region, const Extent3D& src_size); + /// Temporal upscale (temporal_upscale.frag) into dst_framebuffer: the jittered input colour, + /// accumulated with the history image reprojected along the motion vectors. + void TemporalUpscale(const Framebuffer* dst_framebuffer, const ImageView& input_color, + const ImageView& motion_vectors, VkImageView history, + const Region2D& dst_region, std::array jitter, + std::array motion_scale, f32 current_weight); + void BlitColorMSAA(const Framebuffer* dst_framebuffer, const ImageView& src_image_view, const Region2D& dst_region, const Region2D& src_region); @@ -170,6 +177,8 @@ private: [[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key); + [[nodiscard]] VkPipeline FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass); + [[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key); [[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key); @@ -207,15 +216,19 @@ private: vk::DescriptorSetLayout one_texture_set_layout; vk::DescriptorSetLayout two_textures_set_layout; + vk::DescriptorSetLayout three_textures_set_layout; DescriptorAllocator one_texture_descriptor_allocator; DescriptorAllocator two_textures_descriptor_allocator; + DescriptorAllocator three_textures_descriptor_allocator; vk::PipelineLayout one_texture_pipeline_layout; vk::PipelineLayout two_textures_pipeline_layout; + vk::PipelineLayout temporal_upscale_pipeline_layout; vk::PipelineLayout clear_color_pipeline_layout; vk::PipelineLayout msaa_copy_pipeline_layout; vk::PipelineLayout msaa_copy_depth_stencil_pipeline_layout; vk::ShaderModule full_screen_vert; vk::ShaderModule blit_color_to_color_frag; + vk::ShaderModule temporal_upscale_frag; vk::ShaderModule blit_color_msaa_frag; vk::ShaderModule blit_depth_stencil_frag; vk::ShaderModule blit_depth_frag; @@ -246,6 +259,8 @@ private: std::vector blit_color_keys; std::vector blit_color_pipelines; + std::vector temporal_upscale_keys; + std::vector temporal_upscale_pipelines; std::vector blit_depth_stencil_keys; std::vector blit_depth_stencil_pipelines; std::vector clear_color_keys; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index cb31a0e7e4..c0e8825bac 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1231,6 +1231,14 @@ bool RasterizerVulkan::AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surf return texture_cache.BlitImage(dst, src, copy_config); } +bool RasterizerVulkan::AccelerateTemporalUpscale( + const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, std::array jitter, + std::array motion_scale) { + std::scoped_lock lock{texture_cache.mutex}; + return texture_cache.TemporalUpscale(dst, src, motion_vectors, jitter, motion_scale); +} + Tegra::Engines::AccelerateDMAInterface& RasterizerVulkan::AccessAccelerateDMA() { return accelerate_dma; } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index ae9929ca0b..7aca376614 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -128,6 +128,11 @@ public: bool AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, const Tegra::Engines::Fermi2D::Config& copy_config) override; + bool AccelerateTemporalUpscale(const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array jitter, + std::array motion_scale) override; Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() override; void AccelerateInlineToMemory(GPUVAddr address, size_t copy_size, std::span memory) override; diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index ddd55de690..4e428229f6 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -1491,6 +1491,154 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src, }); } +bool TextureCacheRuntime::TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst, + ImageView& input_color, ImageView& motion_vectors, + std::array jitter, + std::array motion_scale) { + if (dst.Samples() != VK_SAMPLE_COUNT_1_BIT || ImageAspectMask(dst.format) != VK_IMAGE_ASPECT_COLOR_BIT) { + return false; + } + const VkFormat format = + MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, false, dst.format).format; + const VkExtent2D extent{dst.size.width, dst.size.height}; + const VkImage output = dst.ImageHandle(); + bool reset = false; + if (!temporal_history.image || temporal_history.format != format || + temporal_history.extent.width != extent.width || + temporal_history.extent.height != extent.height) { + if (temporal_history.image) { + scheduler.Finish(); // the old history may still be sampled + } + temporal_history.image = memory_allocator.CreateImage(VkImageCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .imageType = VK_IMAGE_TYPE_2D, + .format = format, + .extent = {extent.width, extent.height, 1}, + .mipLevels = 1, + .arrayLayers = 1, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + }); + temporal_history.view = device.GetLogical().CreateImageView(VkImageViewCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .image = *temporal_history.image, + .viewType = VK_IMAGE_VIEW_TYPE_2D, + .format = format, + .components{}, + .subresourceRange{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + }, + }); + temporal_history.format = format; + temporal_history.extent = extent; + reset = true; + LOG_INFO(Render_Vulkan, "Temporal upscale into {}x{} (format {})", extent.width, extent.height, + static_cast(format)); + } + + // The history holds this pass's previous result, copied right after drawing it: the game may + // draw into its DLSS output afterwards (post-processing), which must not feed back. The history + // stays in the GENERAL layout, which the blit helper's descriptors use. + const VkImage history = *temporal_history.image; + const VkImageSubresourceLayers output_layers{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = static_cast(dst.range.base.level), + .baseArrayLayer = static_cast(dst.range.base.layer), + .layerCount = 1, + }; + static constexpr VkImageSubresourceLayers history_layers{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = 0, + .baseArrayLayer = 0, + .layerCount = 1, + }; + const auto barrier = [](VkImage image, VkAccessFlags src_access, VkAccessFlags dst_access, + VkImageLayout old_layout, const VkImageSubresourceLayers& layers) { + return VkImageMemoryBarrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = src_access, + .dstAccessMask = dst_access, + .oldLayout = old_layout, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = image, + .subresourceRange{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = layers.mipLevel, + .levelCount = 1, + .baseArrayLayer = layers.baseArrayLayer, + .layerCount = 1, + }, + }; + }; + if (reset) { + // A new history image has no contents yet; the first frame does not read it. + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([history, barrier](vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + 0, barrier(history, 0, VK_ACCESS_SHADER_READ_BIT, + VK_IMAGE_LAYOUT_UNDEFINED, history_layers)); + }); + } + const Region2D dst_region{ + .start = {0, 0}, + .end = {static_cast(extent.width), static_cast(extent.height)}, + }; + // Each frame weighs at most 0.1 against the history (scaled by how close its samples land to + // the pixel; the neighbourhood clip limits what older samples contribute). + constexpr f32 current_weight = 0.1f; + blit_image_helper.TemporalUpscale(dst_framebuffer, input_color, motion_vectors, + *temporal_history.view, dst_region, jitter, motion_scale, + reset ? 1.0f : current_weight); + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([output, history, extent, output_layers, barrier](vk::CommandBuffer cmdbuf) { + constexpr VkAccessFlags written = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + const std::array before{ + barrier(output, written, VK_ACCESS_TRANSFER_READ_BIT, VK_IMAGE_LAYOUT_GENERAL, + output_layers), + barrier(history, VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, + VK_IMAGE_LAYOUT_GENERAL, history_layers), + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, {}, before); + const VkImageCopy copy{ + .srcSubresource = output_layers, + .srcOffset = {0, 0, 0}, + .dstSubresource = history_layers, + .dstOffset = {0, 0, 0}, + .extent = {extent.width, extent.height, 1}, + }; + cmdbuf.CopyImage(output, VK_IMAGE_LAYOUT_GENERAL, history, VK_IMAGE_LAYOUT_GENERAL, copy); + const std::array after{ + barrier(output, VK_ACCESS_TRANSFER_READ_BIT, + VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_TRANSFER_READ_BIT, + VK_IMAGE_LAYOUT_GENERAL, output_layers), + barrier(history, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT, + VK_IMAGE_LAYOUT_GENERAL, history_layers), + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, + {}, {}, after); + }); + return true; +} + void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& src, const Region2D& dst_region, const Region2D& src_region, Tegra::Engines::Fermi2D::Filter filter, diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 43218d6ad3..25a6ce7c1e 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -87,6 +87,12 @@ public: Tegra::Engines::Fermi2D::Filter filter, Tegra::Engines::Fermi2D::Operation operation); + /// Stands in for a DLSS evaluation: temporal upscale of input_color into dst, accumulated with + /// a copy of dst's previous contents (the last output). False when dst cannot take it. + bool TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& input_color, + ImageView& motion_vectors, std::array jitter, + std::array motion_scale); + void CopyImage(Image& dst, Image& src, std::span copies); void CopyImageMSAA(Image& dst, Image& src, std::span copies); @@ -169,6 +175,15 @@ public: StagingBufferPool& staging_buffer_pool; BlitImageHelper& blit_image_helper; RenderPassCache& render_pass_cache; + + /// TemporalUpscale's history: its previous result, copied right after drawing it. + struct TemporalHistory { + vk::Image image; + vk::ImageView view; + VkFormat format{}; + VkExtent2D extent{}; + } temporal_history; + std::optional astc_decoder_pass; std::optional bc_encoder_pass; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 33ca051482..32775b8719 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -956,6 +956,73 @@ bool TextureCache

::BlitImage(const Tegra::Engines::Fermi2D::Surface& dst, return true; } +template +bool TextureCache

::TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array jitter, std::array motion_scale) { + if constexpr (FRAMEBUFFER_BLITS) { + return false; + } else { + const Tegra::Engines::Fermi2D::Config copy{ + .operation = Tegra::Engines::Fermi2D::Operation::SrcCopy, + .filter = Tegra::Engines::Fermi2D::Filter::Bilinear, + .must_accelerate = true, + .dst_x0 = 0, + .dst_y0 = 0, + .dst_x1 = static_cast(dst.width), + .dst_y1 = static_cast(dst.height), + .src_x0 = 0, + .src_y0 = 0, + .src_x1 = static_cast(src.width), + .src_y1 = static_cast(src.height), + }; + const auto result = GetBlitImages(dst, src, copy); + if (!result) { + return false; + } + const ImageInfo motion_info(motion_vectors); + has_deleted_images = false; + const ImageId motion_id = + FindOrInsertImage(motion_info, motion_vectors.Address(), RelaxedOptions{}); + if (has_deleted_images || !motion_id) { + return false; // the colour and output ids may be stale; the caller blits instead + } + const ImageId dst_id = result->dst_id; + const ImageId src_id = result->src_id; + PrepareImage(src_id, false, false); + PrepareImage(motion_id, false, false); + PrepareImage(dst_id, true, false); + for (const ImageId id : {src_id, motion_id, dst_id}) { + if (True(slot_images[id].flags & ImageFlagBits::Rescaled) || + slot_images[id].info.num_samples != 1) { + return false; + } + } + const auto view = [&](ImageId id, const Tegra::Engines::Fermi2D::Surface& surface, + PixelFormat format) -> std::optional> { + const std::optional base = slot_images[id].TryFindBase(surface.Address()); + if (!base) { + return std::nullopt; + } + const SubresourceRange range{.base = *base, .extent = {1, 1}}; + return RenderTargetFromImage(id, ImageViewInfo(ImageViewType::e2D, format, range)); + }; + const auto src_view = view(src_id, src, result->src_format); + const auto motion_view = view(motion_id, motion_vectors, motion_info.format); + const auto dst_view = view(dst_id, dst, result->dst_format); + if (!src_view || !motion_view || !dst_view) { + return false; + } + // After the views: creating them may reallocate the framebuffer slots. + Framebuffer* const dst_framebuffer = &slot_framebuffers[dst_view->first]; + return runtime.TemporalUpscale(dst_framebuffer, slot_image_views[dst_view->second], + slot_image_views[src_view->second], + slot_image_views[motion_view->second], jitter, + motion_scale); + } +} + template std::pair TextureCache

::TryFindFramebufferImageView( const Tegra::FramebufferConfig& config, DAddr cpu_addr) { diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 141e5b76d5..d4b0f597ae 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -276,6 +276,13 @@ public: const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Config& copy); + /// Stand-in for a DLSS evaluation: temporal upscale of src into dst (see the runtime's + /// TemporalUpscale). False when the images cannot take it; the caller then blits. + bool TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array jitter, std::array motion_scale); + /// Try to find a cached image view in the given CPU address [[nodiscard]] std::pair TryFindFramebufferImageView( const Tegra::FramebufferConfig& config, DAddr cpu_addr); -- cgit v1.2.3 From a407703f6d29a5ab73328b4ededb12cf29b2758d Mon Sep 17 00:00:00 2001 From: Whovian64 <86082882+whovian64@users.noreply.github.com> Date: Mon, 2 Nov 2026 02:45:39 -0500 Subject: Linux and AMD: BC3 flashing on RADV, ASTC decode stalls, lower RAM usage, a buffer cache crash (#260) * host_memory: punch holes in the backing on Linux instead of zero filling freed memory Freeing guest memory zero filled the memfd backing with memset, which commits every page it touches. fallocate(PUNCH_HOLE) hands the pages back to the kernel instead, and they read as zero afterwards. Only the partial pages at the ends are still cleared by hand. Mario Kart World on Linux (RX 9070 XT, 31 GB): peak RSS 23.1 -> 15.9 GB. EDEN_PUNCH_HOLE=0 restores the memset. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * fiber: don't zero fill the stacks, untouched pages stay unbacked The value initialised stack arrays were zero filled when each fiber was made, which commits the whole stack. Left uninitialised, the pages a fiber never touches stay unbacked. Together with the descriptor ring change, about 1.2 GB less RSS in Mario Kart World. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * vulkan: don't zero fill the descriptor update ring Same as the fiber stacks: make_unique value initialised the whole ring for every frame in flight. Entries are always written before they are read, so allocate it without initialising it. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * vulkan: wait for the GPU after an ASTC transcode decode like the other ASTC decodes TranscodeAstcToBcn (astc_recompression Bc1/Bc3 with GPU decode) was the only ASTC decode recorded with wait = false. On RADV (RX 9070 XT, Mesa) Mario Kart World with BC3 flashed heavily all over the screen, while the uncompressed GPU decode, which waits, did not. CPU decode with BC3 does not flash either. Waiting after the transcode removes the flashing; full pipeline barriers around its copy did not, so the wait hides a race somewhere else rather than fixing one in the transcode itself. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * vulkan: submit after an ASTC decode instead of waiting for the GPU by default Every ASTC decode ended with scheduler.Finish(). Over a 19 minute Mario Kart World Grand Prix that stalled the GPU thread 30.7 s in total, up to 1.2 s out of every 5 s while new textures stream in (track sections, race starts), which is where the hitches were. Flushing keeps the ordering that stops the BC3 flashing on RADV without the CPU waiting: no flashing in play, and frames over 100 ms went from dozens to 3 in about 7 minutes. EDEN_NX2_ASTC_FLUSH=0 restores the wait. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * texture_cache: over the expected use, evict costly images unused for 1800 frames first The collector never deletes CostlyLoad images outside aggressive mode, so once a cup moves on, the previous tracks' textures stay resident and the cache sits over its expected use deleting the cheap images still in use. When over the expected use, costly images unused for 1800 frames (half a minute) now go first. Mario Kart World Grand Prix on an RX 9070 XT: frames over 20 ms 8.8% -> 4.0%, 58.7 fps average. EDEN_NX2_TC_OLD_AGE= sets the age, 0 turns it off. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) * buffer_cache: no buffer for device addresses past the page table FindBuffer indexed page_table, which covers 2^36 bytes, with any address. Mario Kart World crashed after about 4.5 minutes with a uniform buffer at 0x72253b2f2890 (FindBuffer <- UpdateUniformBuffers <- DrawEnd from a macro), reading far past the table. Treat such an address as no buffer. Where that address comes from is still unknown; this only stops the crash. Assisted-by: Claude Opus 5.5 (debugging, profiling and code drafts) --- src/common/fiber.cpp | 4 ++-- src/common/host_memory.cpp | 22 ++++++++++++++++++++++ src/video_core/buffer_cache/buffer_cache.h | 4 ++-- src/video_core/renderer_vulkan/vk_compute_pass.cpp | 6 +++--- .../renderer_vulkan/vk_texture_cache.cpp | 2 +- .../renderer_vulkan/vk_update_descriptor.cpp | 8 ++++---- .../renderer_vulkan/vk_update_descriptor.h | 3 ++- src/video_core/texture_cache/texture_cache.h | 16 ++++++++++++++-- 8 files changed, 50 insertions(+), 15 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/common/fiber.cpp b/src/common/fiber.cpp index 0e7f28c5b9..08558a0026 100644 --- a/src/common/fiber.cpp +++ b/src/common/fiber.cpp @@ -25,8 +25,8 @@ struct Fiber::FiberImpl { FiberImpl() {} u32 canary_1 = CANARY_VALUE; - std::array stack{}; - std::array rewind_stack{}; + std::array stack; + std::array rewind_stack; u32 canary_2 = CANARY_VALUE; boost::context::detail::fcontext_t context{}; diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 75a9da80e9..147eba8477 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -51,6 +51,7 @@ #endif // ^^^ POSIX ^^^ +#include #include #include @@ -517,6 +518,14 @@ public: , virtual_size{virtual_size_} {} +#ifdef __linux__ + bool PunchHole(size_t offset, size_t length) { + return fd >= 0 && + fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, static_cast(offset), + static_cast(length)) == 0; + } +#endif + bool Init() { long page_size = sysconf(_SC_PAGESIZE); ASSERT_MSG(page_size == 0x1000, "page size {:#x} is incompatible with 4K paging", page_size); @@ -791,6 +800,19 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, MemoryPermission } void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value) { +#ifdef __linux__ + const size_t first = AlignUp(physical_offset, PageAlignment); + const size_t last = AlignDown(physical_offset + length, PageAlignment); + static const bool punch = [] { + const char* value = std::getenv("EDEN_PUNCH_HOLE"); + return value == nullptr || value[0] != '0'; + }(); + if (punch && fill_value == 0 && impl && last > first && impl->PunchHole(first, last - first)) { + std::memset(backing_base + physical_offset, 0, first - physical_offset); + std::memset(backing_base + last, 0, physical_offset + length - last); + return; + } +#endif std::memset(backing_base + physical_offset, fill_value, length); } diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e09a435dcc..a3358048dc 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1651,10 +1651,10 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 template BufferId BufferCache

::FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible) { - if (device_addr == 0) { + const u64 page = device_addr >> CACHING_PAGEBITS; + if (device_addr == 0 || page >= page_table.size()) { return NULL_BUFFER_ID; } - const u64 page = device_addr >> CACHING_PAGEBITS; const BufferId buffer_id = page_table[page]; if (buffer_id) { Buffer& buffer = slot_buffers[buffer_id]; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index 691be728d2..cb8dcde42f 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -801,11 +801,11 @@ void ASTCDecoderPass::AssembleInto(const VideoCommon::ImageInfo& info, u32 skip, cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier); }); - // Each decode ends with the GPU thread waiting for the host GPU to go idle. Measured here - // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=1 submits without waiting instead. + // Each decode ends with a submit to the host GPU. Measured here + // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=0 waits for the host GPU to go idle instead. static const bool flush_only = [] { const char* value = std::getenv("EDEN_NX2_ASTC_FLUSH"); - return value != nullptr && value[0] == '1'; + return value == nullptr || value[0] != '0'; }(); if (!target.wait) { return; diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 4e428229f6..b0e5eacb32 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -4267,7 +4267,7 @@ void TextureCacheRuntime::TranscodeAstcToBcn( .aspect_mask = VK_IMAGE_ASPECT_COLOR_BIT, .initialized = false, .level_view = [&pending](s32 level) { return *pending.views[level]; }, - .wait = false, + .wait = true, }, map, swizzles); diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp index 742a8d9969..29f3ee71a1 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp @@ -20,10 +20,10 @@ UpdateDescriptorQueue::UpdateDescriptorQueue(const Device& device_, size_t frame bool supports_descriptor_buffer_) : device{device_}, frame_payload_size{frame_payload_size_}, supports_descriptor_buffer{supports_descriptor_buffer_}, - payload(frame_payload_size_ * FRAMES_IN_FLIGHT) + payload(std::make_unique_for_overwrite(frame_payload_size_ * FRAMES_IN_FLIGHT)) { - payload_start = payload.data(); - payload_cursor = payload.data(); + payload_start = payload.get(); + payload_cursor = payload.get(); } UpdateDescriptorQueue::~UpdateDescriptorQueue() = default; @@ -32,7 +32,7 @@ void UpdateDescriptorQueue::TickFrame() { if (++frame_index >= FRAMES_IN_FLIGHT) { frame_index = 0; } - payload_start = payload.data() + frame_index * frame_payload_size; + payload_start = payload.get() + frame_index * frame_payload_size; payload_cursor = payload_start; } diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.h b/src/video_core/renderer_vulkan/vk_update_descriptor.h index 9f30e65dbd..4d393376b4 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.h +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -122,7 +123,7 @@ private: DescriptorUpdateEntry* payload_cursor = nullptr; DescriptorUpdateEntry* payload_start = nullptr; const DescriptorUpdateEntry* upload_start = nullptr; - std::vector payload; + std::unique_ptr payload; }; // TODO: should these be separate classes instead? diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 32775b8719..55ec4c64df 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -125,6 +125,7 @@ template void TextureCache

::RunGarbageCollector() { bool high_priority_mode = false; bool aggressive_mode = false; + bool evict_old_costly = false; u64 ticks_to_destroy = 0; size_t num_iterations = 0; const auto Configure = [&](bool allow_aggressive) { @@ -139,7 +140,8 @@ void TextureCache

::RunGarbageCollector() { ticks_to_destroy = (std::max)(ticks_to_destroy, min_age); num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10); }; - const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) { + const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode, + &evict_old_costly](ImageId image_id) { if (num_iterations == 0) { return true; } @@ -149,7 +151,8 @@ void TextureCache

::RunGarbageCollector() { return false; } const bool must_download = IsDownloadable(image) && False(image.flags & ImageFlagBits::BadOverlap); - if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) { + if ((!aggressive_mode && !evict_old_costly && True(image.flags & ImageFlagBits::CostlyLoad)) || + (!high_priority_mode && must_download)) { return false; } if (must_download) { @@ -176,6 +179,15 @@ void TextureCache

::RunGarbageCollector() { return false; }; Configure(false); + static const u64 old_age = [] { + const char* value = std::getenv("EDEN_NX2_TC_OLD_AGE"); + return value != nullptr ? std::strtoull(value, nullptr, 10) : 1800ULL; + }(); + if (high_priority_mode && old_age != 0 && frame_tick > old_age) { + evict_old_costly = true; + lru_cache.ForEachItemBelow(frame_tick - old_age, Cleanup); + evict_old_costly = false; + } lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); if (total_used_memory >= critical_memory) { Configure(true); -- cgit v1.2.3