texture_cache: keep Switch 2 VRAM use under the driver's budget

Mario Kart World filled the GPU after a few minutes on 8 GB cards, and on
16 GB cards with uncompressed ASTC: textures were created without memory
(garbage), then the emulator crashed. The Switch 2 collector thresholds
came from the sum of all heap budgets, but on AMD the host-visible
device-local heap is the same VRAM, so the critical threshold ended above
the real budget and the collector never kicked in. Unbacked images were
also counted at their full size.

Thresholds now follow the main device-local heap's budget, re-read every
30 frames, and images count the memory they really have. Switch 2 titles
on GPUs under 12 GiB re-compress ASTC to BC3 for the session when it is
set to Uncompressed (EDEN_NX2_KEEP_ASTC=1 opts out), and
EDEN_NX2_VRAM_LIMIT_MIB simulates a smaller GPU for testing.

Assisted-by: Claude Opus 5.5 (diagnosis and code)

1 files changed, 52 insertions, 22 deletions

diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 63abffed0f..c8aaf8dff5 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -225,19 +225,24 @@ void TextureCache<P>::TickFrame() {
         // VRAM (another game: 7.0 -> 6.4 GB), and collecting only past it means the driver pages
         // textures out. It is also the room to use: Eden's device memory figure is capped near
         // 7 GB, which on a 12 GB card (11.2 GB budget) kept title 1 evicting textures with 3 GB
-        // unused. EDEN_NX2_TC_*_MIB overrides still win.
+        // unused. Both thresholds stay under it with a margin (1/8 of it, at most 1 GiB): past it
+        // allocations fail and images end up unbacked (garbage textures), and Windows shrinks the
+        // budget as the card fills (16 GB card: 13.5 GB left mid-race), hence the re-read every 30
+        // frames. EDEN_NX2_TC_*_MIB overrides still win.
         if constexpr (requires { runtime.GetDeviceMemoryBudget(); }) {
-            if (native_thresholds_applied && budget_check_frames++ % 60 == 0) {
+            if (native_thresholds_applied && budget_check_frames++ % 30 == 0) {
                 const u64 budget = runtime.GetDeviceMemoryBudget();
                 if (budget > 2_GiB) {
                     const u64 old_critical = critical_memory;
-                    critical_memory = budget - 768_MiB;
-                    expected_memory = budget - 1536_MiB;
-                    // Log only real changes (the budget moves by a few MiB all the time).
+                    const u64 margin = (std::min)(budget / 8, 1_GiB);
+                    critical_memory = budget - margin;
+                    expected_memory = budget - 2 * margin;
+                    // Log the first values and real changes (the budget moves by a few MiB all
+                    // the time).
                     const u64 change = critical_memory > old_critical
                                            ? critical_memory - old_critical
                                            : old_critical - critical_memory;
-                    ApplyThresholdOverrides(change >= 256_MiB);
+                    ApplyThresholdOverrides(budget_check_frames == 1 || change >= 256_MiB);
                 }
             }
         }
@@ -1434,7 +1439,8 @@ void TextureCache<P>::RefreshSparseLayers(Image& image) {
             }
         }
         image.dirty_layers.clear();
-        const std::vector<u32> bound = layers.empty() ? std::vector<u32>{} : image.BindLayers(layers);
+        const std::vector<u32> bound =
+            layers.empty() ? std::vector<u32>{} : BindSparseLayers(image, layers);
         if (bound.empty()) {
             runtime.TransitionImageLayout(image);
             return;
@@ -1445,6 +1451,21 @@ void TextureCache<P>::RefreshSparseLayers(Image& image) {
 }
 
 template <class P>
+std::vector<u32> TextureCache<P>::BindSparseLayers(Image& image, std::span<const u32> layers) {
+    if constexpr (requires { &Image::BindLayers; }) {
+        const u64 bound_bytes = ImageMemoryBytes(image);
+        std::vector<u32> bound = image.BindLayers(layers);
+        // RegisterImage counts what an image has bound by then; later binds are added here.
+        if (True(image.flags & ImageFlagBits::Registered)) {
+            total_used_memory += ImageMemoryBytes(image) - bound_bytes;
+        }
+        return bound;
+    } else {
+        return {};
+    }
+}
+
+template <class P>
 void TextureCache<P>::UploadLayers(Image& image, std::span<const u32> layers) {
     if constexpr (requires { &Runtime::AccelerateImageUploadLayers; }) {
         if (False(image.flags & ImageFlagBits::AcceleratedUpload)) {
@@ -1714,6 +1735,26 @@ u64 TextureCache<P>::GetScaledImageSizeBytes(const ImageBase& image) {
 }
 
 template <class P>
+u64 TextureCache<P>::ImageMemoryBytes(const Image& image) const {
+    if (True(image.flags & ImageFlagBits::Unbacked)) {
+        // No device memory but what BindLayers gave a SparseLayers image: at its full size (GBs
+        // for title 1's sparse terrain images), deleting it would look to the collector like
+        // freeing that much.
+        if constexpr (requires { image.SparseBoundBytes(); }) {
+            return image.SparseBoundBytes();
+        }
+        return 0;
+    }
+    u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes);
+    if ((IsPixelFormatASTC(image.info.format) &&
+         True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
+        True(image.flags & ImageFlagBits::Converted)) {
+        tentative_size = TranscodedAstcSize(tentative_size, image.info.format);
+    }
+    return Common::AlignUp(tentative_size, 1024);
+}
+
+template <class P>
 void TextureCache<P>::QueueAsyncDecode(Image& image, ImageId image_id) {
     UNIMPLEMENTED_IF(False(image.flags & ImageFlagBits::Converted));
     LOG_DEBUG(HW_GPU, "Queuing async texture decode {:#x}", image.gpu_addr);
@@ -2613,13 +2654,7 @@ void TextureCache<P>::RegisterImage(ImageId image_id) {
     ASSERT_MSG(False(image.flags & ImageFlagBits::Registered),
                "Trying to register an already registered image");
     image.flags |= ImageFlagBits::Registered;
-    u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes);
-    if ((IsPixelFormatASTC(image.info.format) &&
-         True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
-        True(image.flags & ImageFlagBits::Converted)) {
-        tentative_size = TranscodedAstcSize(tentative_size, image.info.format);
-    }
-    total_used_memory += Common::AlignUp(tentative_size, 1024);
+    total_used_memory += ImageMemoryBytes(slot_images[image_id]);
     image.lru_index = lru_cache.Insert(image_id, frame_tick);
 
     ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, image_id](u64 page) {
@@ -2795,13 +2830,8 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) {
     if (image.HasScaled()) {
         total_used_memory -= GetScaledImageSizeBytes(image);
     }
-    u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes);
-    if ((IsPixelFormatASTC(image.info.format) &&
-         True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
-        True(image.flags & ImageFlagBits::Converted)) {
-        tentative_size = TranscodedAstcSize(tentative_size, image.info.format);
-    }
-    total_used_memory -= Common::AlignUp(tentative_size, 1024);
+    // TickFrame sets the total from the driver, which can be under the sum of these estimates.
+    total_used_memory -= (std::min)(total_used_memory, ImageMemoryBytes(slot_images[image_id]));
     const GPUVAddr gpu_addr = image.gpu_addr;
     const auto alloc_it = image_allocs_table.find(gpu_addr);
     if (alloc_it == image_allocs_table.end()) {
@@ -3000,7 +3030,7 @@ void TextureCache<P>::PrepareImage(ImageId image_id, bool is_modification, bool
                 std::vector<u32> layers(static_cast<size_t>(image.info.resources.layers));
                 std::iota(layers.begin(), layers.end(), 0U);
                 // Layers over the image's memory cap stay unbacked (logged once).
-                image.BindLayers(layers);
+                BindSparseLayers(image, layers);
             }
         }
         if (True(image.flags & ImageFlagBits::UnwrittenSparse) &&