Vulkan: unbacked sparse images for textures too large to back

Title 1 samples an 8192x8192, 12-level, 68-layer array (a sparse virtual
texture, ~23 GB): vkCreateImage failed and Eden aborted, or VMA put it in
system memory and staged a full upload (34 GB private). Images over 2^28
texels are now created sparse-resident with no memory bound; the texture
cache neither uploads nor downloads them. Images merely over the VRAM
budget are allocated without the budget limit instead of failing.

2 files changed, 10 insertions, 2 deletions

diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index 79fc8505d6..846a6f7fc1 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -45,6 +45,7 @@ enum class ImageFlagBits : u32 {
 
     AsynchronousDecode = 1 << 16,
     IsDecoding = 1 << 17, ///< Is currently being decoded asynchronously.
+    Unbacked = 1 << 18,   ///< Host image has no memory (too big to back); never upload/download.
 };
 DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits)
 
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index cd4ef82dbe..f3da5684a5 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -597,7 +597,7 @@ void TextureCache<P>::WriteMemory(DAddr cpu_addr, size_t size) {
 
 template <class P>
 bool TextureCache<P>::IsDownloadable(const ImageBase& image) const noexcept {
-    if (!image.IsSafeGpuCopy()) {
+    if (!image.IsSafeGpuCopy() || True(image.flags & ImageFlagBits::Unbacked)) {
         return false;
     }
     if (image.info.num_samples == 1) {
@@ -1146,6 +1146,12 @@ void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) {
 
     TrackImage(image, image_id);
 
+    if (True(image.flags & ImageFlagBits::Unbacked)) {
+        // Nothing to upload into (and staging it could take tens of GB).
+        runtime.TransitionImageLayout(image);
+        return;
+    }
+
     if (image.info.num_samples > 1 && !runtime.CanUploadMSAA()) {
         LOG_WARNING(HW_GPU, "MSAA image uploads are not implemented");
         runtime.TransitionImageLayout(image);
@@ -2669,7 +2675,8 @@ void TextureCache<P>::BindRenderTarget(ImageViewId* old_id, ImageViewId new_id)
     }
     if (new_id) {
         const ImageViewBase& old_view = slot_image_views[new_id];
-        if (True(old_view.flags & ImageViewFlagBits::PreemtiveDownload)) {
+        if (True(old_view.flags & ImageViewFlagBits::PreemtiveDownload) &&
+            False(slot_images[old_view.image_id].flags & ImageFlagBits::Unbacked)) {
             const PendingDownload new_download{true, 0, old_view.image_id};
             uncommitted_downloads.emplace_back(new_download);
         }