vulkan: bind memory to sparse terrain arrays layer by layer

The written part of Mario Kart World's sparse terrain arrays is small:
13-26% of their blocks. The two 1.45 GB ASTC arrays were backed in full
on the host, and the two 5.8 GB BC6H arrays, too large to back, stayed
unbacked, so what the game streamed into them never showed.

A color array of 32 MiB or more lying in one sparse region, in a format
uploaded layer by layer (huge ASTC arrays decoded on the GPU, or formats
sampled as they are), is now a sparse resident image (SparseLayers).
When it is refreshed, the layers written since (or, for a new image,
the layers any of whose blocks was written) get device memory bound
with vkQueueBindSparse (under the submit lock, waited on, as the
multi-range buffer cache does) and only those layers are uploaded.
Unwritten layers read zero. One image binds at most 768 MiB (384 MiB on
GPUs with less than 7 GiB); rendering into one binds all its layers.

Mario Kart World, 180 s: the ASTC arrays hold 1.4 MiB per written layer,
four BC6H layers (341 MiB) now have their data, device memory in use
4.8-5.0 GB instead of 6.0-7.1 GB with the texture cache below its
collection threshold, 16.8-17.2 ms average frames, peak working set
13.4 GB.

5 files changed, 55 insertions, 0 deletions

diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h
index b1fa8db183..39ae25f43a 100644
--- a/src/video_core/vulkan_common/vulkan_device.h
+++ b/src/video_core/vulkan_common/vulkan_device.h
@@ -321,6 +321,11 @@ public:
         return features.features.sparseBinding && graphics_family_sparse_binding;
     }
 
+    /// 2D images whose memory can be bound in parts, on the graphics queue.
+    bool IsSparseResidencyImage2DSupported() const {
+        return IsSparseBindingSupported() && features.features.sparseResidencyImage2D;
+    }
+
     /// Returns true for tile-based deferred renderers.
     bool IsTiler() const {
         switch (GetDriverID()) {
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
index 92f537aa3c..a0c96d1b45 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
@@ -295,6 +295,35 @@ vk::Image MemoryAllocator::CreateUnbackedImage(const VkImageCreateInfo &ci) cons
                      device.GetDispatchLoader());
 }
 
+VmaAllocation MemoryAllocator::AllocateSparseMemory(const VkMemoryRequirements &requirements,
+                                                   VkDeviceMemory &memory,
+                                                   VkDeviceSize &offset) const {
+    const VmaAllocationCreateInfo alloc_ci{
+        .flags = 0,
+        .usage = VMA_MEMORY_USAGE_UNKNOWN,
+        .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
+        .preferredFlags = 0,
+        .memoryTypeBits = requirements.memoryTypeBits,
+        .pool = VK_NULL_HANDLE,
+        .pUserData = nullptr,
+        .priority = 0.f,
+    };
+    VmaAllocation allocation{};
+    VmaAllocationInfo info{};
+    if (vmaAllocateMemory(allocator, &requirements, &alloc_ci, &allocation, &info) != VK_SUCCESS) {
+        return VK_NULL_HANDLE;
+    }
+    memory = info.deviceMemory;
+    offset = info.offset;
+    return allocation;
+}
+
+void MemoryAllocator::FreeSparseMemory(VmaAllocation allocation) const {
+    if (allocation != VK_NULL_HANDLE) {
+        vmaFreeMemory(allocator, allocation);
+    }
+}
+
 vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const {
     // MESA will do memcpy() if not marked as host cached, so just force mark it for most buffers
     auto const anv_flags = (usage == MemoryUsage::Stream
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h
index 99649b9aef..f3c6c177a2 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.h
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h
@@ -111,6 +111,12 @@ namespace Vulkan {
         /// a null image when the device can't make one.
         vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci) const;
 
+        /// Device memory to bind to part of a sparse resource. Null when there is none left.
+        VmaAllocation AllocateSparseMemory(const VkMemoryRequirements &requirements,
+                                           VkDeviceMemory &memory, VkDeviceSize &offset) const;
+
+        void FreeSparseMemory(VmaAllocation allocation) const;
+
         vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const;
 
         vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage,
diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp
index bb43fe399f..dbaaf450a0 100644
--- a/src/video_core/vulkan_common/vulkan_wrapper.cpp
+++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp
@@ -219,6 +219,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept {
     X(vkGetEventStatus);
     X(vkGetFenceStatus);
     X(vkGetImageMemoryRequirements);
+    X(vkGetImageSparseMemoryRequirements);
     X(vkGetPipelineCacheData);
     X(vkGetMemoryFdKHR);
 #ifdef _WIN32
@@ -866,6 +867,16 @@ VkMemoryRequirements Device::GetImageMemoryRequirements(VkImage image) const noe
     return requirements;
 }
 
+std::vector<VkSparseImageMemoryRequirements> Device::GetImageSparseMemoryRequirements(
+    VkImage image) const {
+    u32 count = 0;
+    dld->vkGetImageSparseMemoryRequirements(handle, image, &count, nullptr);
+    std::vector<VkSparseImageMemoryRequirements> requirements(count);
+    dld->vkGetImageSparseMemoryRequirements(handle, image, &count, requirements.data());
+    requirements.resize(count);
+    return requirements;
+}
+
 std::vector<VkPipelineExecutablePropertiesKHR> Device::GetPipelineExecutablePropertiesKHR(
     VkPipeline pipeline) const {
     const VkPipelineInfoKHR info{
diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h
index 947150648d..3010368973 100644
--- a/src/video_core/vulkan_common/vulkan_wrapper.h
+++ b/src/video_core/vulkan_common/vulkan_wrapper.h
@@ -335,6 +335,7 @@ struct DeviceDispatch : InstanceDispatch {
     PFN_vkGetEventStatus vkGetEventStatus{};
     PFN_vkGetFenceStatus vkGetFenceStatus{};
     PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements{};
+    PFN_vkGetImageSparseMemoryRequirements vkGetImageSparseMemoryRequirements{};
     PFN_vkGetPipelineCacheData vkGetPipelineCacheData{};
     PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR{};
 #ifdef _WIN32
@@ -1111,6 +1112,9 @@ public:
 
     VkMemoryRequirements GetImageMemoryRequirements(VkImage image) const noexcept;
 
+    std::vector<VkSparseImageMemoryRequirements> GetImageSparseMemoryRequirements(
+        VkImage image) const;
+
     std::vector<VkPipelineExecutablePropertiesKHR> GetPipelineExecutablePropertiesKHR(
         VkPipeline pipeline) const;