texture_cache: give images over unwritten sparse blocks no device memory

Mario Kart World creates nine terrain layer pools in sparse regions (four
R8 16384x16384, four RG16 8192x8192, one R16 8192x8192: 2.9 GB of device
memory). In 180 s of title and attract demo not one of their blocks is
written and nothing renders into them, so they hold zeros; on the console
only the few tiles the game maps have memory.

nx2:gpu now publishes which 4 MiB blocks of each sparse region have been
written (nx2.h, a leaf lock the texture cache may take). An image of 32
MiB or more lying in one sparse region with no written block is created
as a sparse resident image with nothing bound: reads return zero, as an
unmapped tile does. The first write into its range, or the first time the
GPU renders into it (that range then counts as written), marks it
remapped, so a backed image replaces it at its next use.

Mario Kart World, 180 s: the collector deletes 0-80 images per 600 frames
instead of thousands, frames over 100 ms per 600 presents drop from 1-9 to
0-1, peak working set 14.7 GB. One render target in a small sparse region
gets replaced once (logged).

6 files changed, 144 insertions, 12 deletions

diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h
index 58c9b17bcf..1b565e73d4 100644
--- a/src/video_core/nx2.h
+++ b/src/video_core/nx2.h
@@ -7,6 +7,7 @@
 #include <array>
 #include <atomic>
 #include <functional>
+#include <map>
 #include <mutex>
 #include <vector>
 
@@ -93,6 +94,81 @@ inline void Nx2BackSparseWrite(u64 gpu_address, u64 size) {
     }
 }
 
+// Which blocks of each sparse region have been written (got memory), for the texture cache: an
+// image over blocks nobody wrote is all zeros and needs no device memory. Its own lock, taken last:
+// the texture cache asks with its lock held, and nx2:gpu updates it holding none of the GPU's.
+struct Nx2SparseRegionState {
+    u64 size;
+    u64 block_size;
+    std::vector<bool> written;
+};
+inline std::mutex g_nx2_sparse_state_mutex;
+inline std::map<u64, Nx2SparseRegionState> g_nx2_sparse_state;
+
+inline void Nx2SparseRegionCreated(u64 gpu_address, u64 size, u64 block_size) {
+    std::scoped_lock lk{g_nx2_sparse_state_mutex};
+    g_nx2_sparse_state[gpu_address] = {size, block_size,
+                                       std::vector<bool>((size + block_size - 1) / block_size)};
+}
+
+inline void Nx2SparseRegionDestroyed(u64 gpu_address) {
+    std::scoped_lock lk{g_nx2_sparse_state_mutex};
+    g_nx2_sparse_state.erase(gpu_address);
+}
+
+inline void Nx2SparseBlockWritten(u64 region_address, u64 block) {
+    std::scoped_lock lk{g_nx2_sparse_state_mutex};
+    if (const auto it = g_nx2_sparse_state.find(region_address); it != g_nx2_sparse_state.end()) {
+        it->second.written[block] = true;
+    }
+}
+
+/// The GPU renders into [gpu_address, +size): count its blocks as written, so images made there
+/// later get memory (their contents live in the host image, not in guest memory).
+inline void Nx2SparseMarkWritten(u64 gpu_address, u64 size) {
+    if (size == 0 || gpu_address + size <= g_nx2_sparse_low.load(std::memory_order_relaxed) ||
+        gpu_address >= g_nx2_sparse_high.load(std::memory_order_relaxed)) [[likely]] {
+        return;
+    }
+    std::scoped_lock lk{g_nx2_sparse_state_mutex};
+    for (auto& [start, region] : g_nx2_sparse_state) {
+        const u64 end = start + region.size;
+        if (gpu_address >= end || gpu_address + size <= start) {
+            continue;
+        }
+        const u64 first = ((std::max)(gpu_address, start) - start) / region.block_size;
+        const u64 last = ((std::min)(gpu_address + size, end) - 1 - start) / region.block_size;
+        for (u64 block = first; block <= last; ++block) {
+            region.written[block] = true;
+        }
+    }
+}
+
+/// True when [gpu_address, +size) lies in one sparse region and none of it has been written.
+inline bool Nx2SparseUnwritten(u64 gpu_address, u64 size) {
+    if (size == 0 || gpu_address + size <= g_nx2_sparse_low.load(std::memory_order_relaxed) ||
+        gpu_address >= g_nx2_sparse_high.load(std::memory_order_relaxed)) [[likely]] {
+        return false;
+    }
+    std::scoped_lock lk{g_nx2_sparse_state_mutex};
+    auto it = g_nx2_sparse_state.upper_bound(gpu_address);
+    if (it == g_nx2_sparse_state.begin()) {
+        return false;
+    }
+    --it;
+    const auto& [start, region] = *it;
+    if (gpu_address + size > start + region.size) {
+        return false;
+    }
+    const u64 last = (gpu_address + size - 1 - start) / region.block_size;
+    for (u64 block = (gpu_address - start) / region.block_size; block <= last; ++block) {
+        if (region.written[block]) {
+            return false;
+        }
+    }
+    return true;
+}
+
 /// GPU VA in the first page that mirrors a release at `address`, or 0.
 inline u64 Nx2SemaphoreMirror(u64 address) {
     for (const auto& region : g_nx2_semaphore_regions) {
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index 27f0e58321..08a471cfd6 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -20,6 +20,7 @@
 #include "common/settings.h"
 
 #include "video_core/renderer_vulkan/vk_texture_cache.h"
+#include "video_core/nx2.h"
 #include "video_core/gpu_logging/gpu_logging.h"
 
 #include "video_core/engines/fermi_2d.h"
@@ -165,6 +166,16 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
     return bytes * info.resources.layers > (u64{1} << 30);
 }
 
+// A large image over Switch 2 sparse blocks nobody has written (title 1's terrain layer pools, 2.9
+// GB of R8/RG16/R16 that stay empty in the attract demo) is all zeros: it gets no device memory
+// until a write makes the texture cache replace it.
+[[nodiscard]] bool IsUnwrittenSparse(const ImageInfo& info, GPUVAddr gpu_addr) {
+    constexpr u32 MinSize = 32U << 20;
+    const u32 size = VideoCommon::CalculateGuestSizeInBytes(info);
+    return size >= MinSize && info.type != ImageType::e1D && info.type != ImageType::Buffer &&
+           info.type != ImageType::Linear && Tegra::Nx2SparseUnwritten(gpu_addr, size);
+}
+
 // The host format of a huge ASTC array: the GPU decoder's RGBA8 (sRGB kept for sampling).
 [[nodiscard]] std::optional<VkFormat> HugeAstcHostFormat(const Device& device, const ImageInfo& info) {
     if (!IsHugeAstcArray(device, info)) {
@@ -310,7 +321,8 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM
 
 [[nodiscard]] vk::Image MakeImage(const Device& device, const MemoryAllocator& allocator,
                                   const ImageInfo& info, std::span<const VkFormat> view_formats,
-                                  std::optional<VkFormat> format_override = {}) {
+                                  std::optional<VkFormat> format_override = {},
+                                  bool unbacked = false) {
     if (info.type == ImageType::Buffer) {
         return vk::Image{};
     }
@@ -338,6 +350,11 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM
             image_ci.pNext = &image_format_list;
         }
     }
+    if (unbacked) {
+        if (vk::Image image = allocator.CreateUnbackedImage(image_ci)) {
+            return image;
+        }
+    }
     return allocator.CreateImage(image_ci);
 }
 
@@ -2019,10 +2036,19 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu
                                : IsOpaqueBc1Astc(runtime_.device, info)
                                    ? std::span<const VkFormat>{OpaqueAstcViewFormats}
                                    : runtime->ViewFormats(info.format),
-                               AstcHostFormat(runtime_.device, info))),
+                               AstcHostFormat(runtime_.device, info),
+                               IsUnwrittenSparse(info, gpu_addr_))),
       aspect_mask(ImageAspectMask(info.format)) {
     if (original_image.IsUnbacked()) {
         flags |= VideoCommon::ImageFlagBits::Unbacked;
+        // Not too big to back (see MemoryAllocator::CreateImage): unbacked because nothing was
+        // written there yet, so the first write gets it replaced.
+        const ImageInfo host_info = TrimmedHostInfo(runtime_.device, info, host_level_offset);
+        const u64 texels = u64{host_info.size.width} * host_info.size.height *
+                           host_info.size.depth * static_cast<u64>(host_info.resources.layers);
+        if (texels <= (u64{1} << 28)) {
+            flags |= VideoCommon::ImageFlagBits::UnwrittenSparse;
+        }
     }
     if (host_level_offset != 0) {
         flags |= VideoCommon::ImageFlagBits::TrimmedLevels;
diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index 20d3883130..cfb839d17e 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -50,6 +50,9 @@ enum class ImageFlagBits : u32 {
     /// in VRAM); uploads skip the rest, downloads are never done.
     TrimmedLevels = 1 << 19,
     ZeroFilled = 1 << 20, ///< Cleared to zero blocks before its first asynchronous decode.
+    /// Over Switch 2 sparse blocks nobody wrote: all zeros, so the host image has no memory
+    /// (Unbacked) until the first write, which makes a backed image replace it.
+    UnwrittenSparse = 1 << 21,
 };
 DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits)
 
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index d582f520f7..168df50c8b 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -28,6 +28,7 @@
 #include "video_core/texture_cache/samples_helper.h"
 #include "video_core/texture_cache/texture_cache_base.h"
 #include "video_core/texture_cache/util.h"
+#include "video_core/nx2.h"
 #include "video_core/nx2_profile.h"
 #include "video_core/textures/decoders.h"
 
@@ -717,6 +718,11 @@ FramebufferId TextureCache<P>::GetFramebufferId(const RenderTargets& key) {
 template <class P>
 void TextureCache<P>::WriteMemory(DAddr cpu_addr, size_t size) {
     ForEachImageInRegion(cpu_addr, size, [this, cpu_addr, size](ImageId image_id, Image& image) {
+        if (True(image.flags & ImageFlagBits::UnwrittenSparse)) {
+            // Written at last: a backed image replaces it where it is used next.
+            image.flags |= ImageFlagBits::CpuModified | ImageFlagBits::Remapped;
+            return;
+        }
         if (UploadsByLayer(image)) {
             MarkLayersWritten(image, cpu_addr, size);
             return;
@@ -2918,6 +2924,14 @@ void TextureCache<P>::PrepareImage(ImageId image_id, bool is_modification, bool
         }
     }
     if (is_modification) {
+        if (True(image.flags & ImageFlagBits::UnwrittenSparse) &&
+            False(image.flags & ImageFlagBits::Remapped)) {
+            // The GPU renders into it: these writes are lost, later uses get a backed image.
+            LOG_WARNING(HW_GPU, "GPU write into unwritten sparse image {:#x}; replacing it",
+                        image.gpu_addr);
+            Tegra::Nx2SparseMarkWritten(image.gpu_addr, image.guest_size_bytes);
+            image.flags |= ImageFlagBits::CpuModified | ImageFlagBits::Remapped;
+        }
         MarkModification(image);
     }
     lru_cache.Touch(image.lru_index, frame_tick);
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
index 8bf692b00e..92f537aa3c 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
@@ -248,21 +248,15 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat
     }
     if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && ci.imageType != VK_IMAGE_TYPE_1D) {
         // Too big to back at all, e.g. title 1's sparse virtual texture arrays (8192x8192,
-        // 68 layers, ~23 GB): on the console only resident tiles have memory. Create a sparse
-        // resident image with nothing bound, so views and copies still work; reads of it return
-        // zero and writes are dropped, instead of aborting.
-        VkImageCreateInfo sparse_ci = ci;
-        sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
-        VkImage sparse{};
-        if (device.GetDispatchLoader().vkCreateImage(*device.GetLogical(), &sparse_ci, nullptr,
-                                                     &sparse) == VK_SUCCESS) {
+        // 68 layers, ~23 GB): on the console only resident tiles have memory. Use an image with
+        // nothing bound, so views and copies still work, instead of aborting.
+        if (vk::Image sparse = CreateUnbackedImage(ci)) {
             LOG_WARNING(Render_Vulkan,
                         "Image {}x{}x{} ({} levels, {} layers, format {}) does not fit; using an "
                         "unbacked sparse image",
                         ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels,
                         ci.arrayLayers, static_cast<u32>(ci.format));
-            return vk::Image(sparse, ci.usage, *device.GetLogical(), allocator, VK_NULL_HANDLE,
-                             device.GetDispatchLoader());
+            return sparse;
         }
     }
     if (result != VK_SUCCESS) {
@@ -286,6 +280,21 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat
                         device.GetDispatchLoader());
 }
 
+vk::Image MemoryAllocator::CreateUnbackedImage(const VkImageCreateInfo &ci) const {
+    if (ci.imageType == VK_IMAGE_TYPE_1D) {
+        return vk::Image{};
+    }
+    VkImageCreateInfo sparse_ci = ci;
+    sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
+    VkImage sparse{};
+    if (device.GetDispatchLoader().vkCreateImage(*device.GetLogical(), &sparse_ci, nullptr,
+                                                 &sparse) != VK_SUCCESS) {
+        return vk::Image{};
+    }
+    return vk::Image(sparse, ci.usage, *device.GetLogical(), allocator, VK_NULL_HANDLE,
+                     device.GetDispatchLoader());
+}
+
 vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const {
     // MESA will do memcpy() if not marked as host cached, so just force mark it for most buffers
     auto const anv_flags = (usage == MemoryUsage::Stream
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h
index fa7d4b41c3..99649b9aef 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.h
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h
@@ -107,6 +107,10 @@ namespace Vulkan {
         /// images would otherwise leave holes in the shared blocks).
         vk::Image CreateImage(const VkImageCreateInfo &ci, bool dedicated = false) const;
 
+        /// A sparse resident image with nothing bound (reads return zero, writes are dropped), or
+        /// a null image when the device can't make one.
+        vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci) const;
+
         vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const;
 
         vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage,