vulkan: don't make small images unbacked when device memory runs out

When VRAM ran out, any image that failed to allocate became a sparse
image with nothing bound, which AMD's drivers don't survive being
sampled (an RX 6600 died mid-flood, an RX 7800 XT made 211 of them,
8x8 ones included). Try host memory first, and count the images that
didn't fit: the texture cache then keeps more room under the budget
(512 MiB a step, up to 3 GiB, decaying) so its collector runs earlier.
The sparse fallback stays for images too large to back.

2 files changed, 29 insertions, 0 deletions

diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
index 79fb331de0..a0a39abe39 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
@@ -245,11 +245,32 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat
                         ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels,
                         ci.arrayLayers, static_cast<u32>(ci.format));
         }
+        pressure_events.fetch_add(1, std::memory_order_relaxed);
         VmaAllocationCreateInfo relaxed = alloc_ci;
         relaxed.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT;
         result = vmaCreateImage(allocator, &ci, &relaxed, &handle, &allocation, &alloc_info);
     }
+    if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && !huge) {
+        // Still out of device memory: a real image in host memory (slow, but valid) beats one
+        // with nothing bound, which AMD's drivers do not survive being sampled.
+        VmaAllocationCreateInfo host = alloc_ci;
+        host.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT;
+        host.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST;
+        host.preferredFlags = 0;
+        result = vmaCreateImage(allocator, &ci, &host, &handle, &allocation, &alloc_info);
+        static u32 host_logged{};
+        if (result == VK_SUCCESS && host_logged++ < 16) {
+            LOG_WARNING(Render_Vulkan,
+                        "Image {}x{}x{} ({} levels, {} layers, format {}) is out of device memory; "
+                        "placed in host memory",
+                        ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels,
+                        ci.arrayLayers, static_cast<u32>(ci.format));
+        }
+    }
     if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && ci.imageType != VK_IMAGE_TYPE_1D) {
+        if (!huge) {
+            pressure_events.fetch_add(1, std::memory_order_relaxed);
+        }
         // Too big to back at all, e.g. title 1's sparse virtual texture arrays (8192x8192,
         // 68 layers, ~23 GB): on the console only resident tiles have memory. Use an image with
         // nothing bound, so views and copies still work, instead of aborting.
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h
index ffca933ee7..8b9e72b3e3 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.h
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h
@@ -6,6 +6,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <memory>
 #include <mutex>
 #include <span>
@@ -126,6 +127,12 @@ namespace Vulkan {
 
         void FreeSparseMemory(VmaAllocation allocation) const;
 
+        /// Counts the images that did not fit the device memory budget (made anyway, in host
+        /// memory or with nothing bound): the texture cache collects harder when it grows.
+        u32 MemoryPressureEvents() const noexcept {
+            return pressure_events.load(std::memory_order_relaxed);
+        }
+
         vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const;
 
         vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage,
@@ -161,6 +168,7 @@ namespace Vulkan {
         const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties.
         VkDeviceSize buffer_image_granularity;            ///< Adjacent buffer/image granularity
         u32 valid_memory_types{~0u};
+        mutable std::atomic<u32> pressure_events{0};
         mutable std::mutex buffer_type_mutex;
         /// Memory type of device-local buffers by usage and flags (CreateBufferInSeparatePool).
         mutable std::unordered_map<u64, u32> buffer_memory_types;