[shader_recompiler, vulkan] Virtual buffer pages with multi-range + sparse buffers binding (#4362)

Storage buffers can span GPU pages that aren't contiguous in host memory, but the buffer cache assumed one contiguous buffer per binding, so anything crossing a mapping boundary read the wrong bytes. Multi-range binding resolves the real segments and presents them to the shader as one buffer, aliasing their memory into a sparse VkBuffer or gathering them into a copy, all of this was mostly fixed by making a path to use sparse on buffers (the same way as texture cache has it) and retrieve properly the mapping ranges to the virtual pages, including the actual structure on the use of binding sparse.

Meanwhile this fixes Monster Hunter Sunbreak z-fighting, bad performance and mostly vertex explosions (coming from the contiguous buffer read from the shader's game) it's only the basis for a further investigation towards how are we gonna treat phi calculations, optimized them and mostly eradicate the currently excess of exposure on the textures where the lightning trace should not being reflected.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4362
Reviewed-by: lizzie <lizzie@eden-emu.dev>
Reviewed-by: Maufeat <sahyno1996@gmail.com>

1 files changed, 134 insertions, 26 deletions

diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index e40aad1fb5..4c345dd213 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -113,6 +113,13 @@ void BufferCache<P>::TickFrame() {
 }
 
 template <class P>
+void BufferCache<P>::UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size) {
+    if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) {
+        virtual_ranges.Unmap(as_id, gpu_addr, size);
+    }
+}
+
+template <class P>
 void BufferCache<P>::WriteMemory(DAddr device_addr, u64 size) {
     if (memory_tracker.IsRegionGpuModified(device_addr, size)) {
         ClearDownload(device_addr, size);
@@ -208,8 +215,8 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am
     BufferId buffer_b;
     do {
         channel_state->has_deleted_buffers = false;
-        buffer_a = FindBuffer(*cpu_src_address, static_cast<u32>(amount));
-        buffer_b = FindBuffer(*cpu_dest_address, static_cast<u32>(amount));
+        buffer_a = FindBuffer(*cpu_src_address, static_cast<u32>(amount), false);
+        buffer_b = FindBuffer(*cpu_dest_address, static_cast<u32>(amount), false);
     } while (channel_state->has_deleted_buffers);
     auto& src_buffer = slot_buffers[buffer_a];
     auto& dest_buffer = slot_buffers[buffer_b];
@@ -265,7 +272,7 @@ bool BufferCache<P>::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) {
     ClearDownload(*cpu_dst_address, size);
     gpu_modified_ranges.Subtract(*cpu_dst_address, size);
 
-    const BufferId buffer = FindBuffer(*cpu_dst_address, static_cast<u32>(size));
+    const BufferId buffer = FindBuffer(*cpu_dst_address, static_cast<u32>(size), false);
     Buffer& dest_buffer = slot_buffers[buffer];
     const u32 offset = dest_buffer.Offset(*cpu_dst_address);
     runtime.ClearBuffer(dest_buffer, offset, size, value);
@@ -287,7 +294,7 @@ std::pair<typename P::Buffer*, u32> BufferCache<P>::ObtainBuffer(GPUVAddr gpu_ad
 template <class P>
 std::pair<typename P::Buffer*, u32> BufferCache<P>::ObtainCPUBuffer(
     DAddr device_addr, u32 size, ObtainBufferSynchronize sync_info, ObtainBufferOperation post_op) {
-    const BufferId buffer_id = FindBuffer(device_addr, size);
+    const BufferId buffer_id = FindBuffer(device_addr, size, false);
     Buffer& buffer = slot_buffers[buffer_id];
 
     // synchronize op
@@ -999,10 +1006,84 @@ void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32
 }
 
 template <class P>
+void BufferCache<P>::ResolveMultiRangeStorage(Binding& binding, bool is_written,
+                                              std::vector<MultiRangeSegment>& pool) {
+    binding.segment_first = 0;
+    binding.segment_count = 0;
+    if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) {
+        if (binding.gpu_addr == 0 || binding.size == 0) {
+            return;
+        }
+        if (is_written && !runtime.PrefersSparseSources()) {
+            return;
+        }
+        const VirtualSegments* found =
+            virtual_ranges.Query(*gpu_memory, binding.gpu_addr, binding.size);
+        if (!found || found->size() < 2) {
+            return;
+        }
+        const VirtualSegments segments = *found;
+        const u32 first = static_cast<u32>(pool.size());
+        const bool prefer_sparse = runtime.PrefersSparseSources();
+        for (const VirtualSegment& segment : segments) {
+            const BufferId buffer_id =
+                FindBuffer(segment.device_addr, segment.size, prefer_sparse);
+            if (!buffer_id) {
+                pool.resize(first);
+                return;
+            }
+            pool.push_back(MultiRangeSegment{
+                .buffer_id = buffer_id,
+                .device_addr = segment.device_addr,
+                .size = segment.size,
+            });
+        }
+        binding.segment_first = first;
+        binding.segment_count = static_cast<u32>(segments.size());
+    }
+}
+
+template <class P>
+bool BufferCache<P>::BindMultiRangeStorage(const Binding& binding, bool is_written,
+                                           std::span<const MultiRangeSegment> pool) {
+    if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) {
+        if (binding.segment_count < 2) {
+            return false;
+        }
+        if (binding.segment_first + binding.segment_count > pool.size()) {
+            return false;
+        }
+        const u64 key = (static_cast<u64>(gpu_memory->GetID()) << 48) ^ binding.gpu_addr;
+        runtime.ResetMultiRange();
+        for (u32 index = 0; index < binding.segment_count; ++index) {
+            const MultiRangeSegment& segment = pool[binding.segment_first + index];
+            Buffer& buffer = slot_buffers[segment.buffer_id];
+            TouchBuffer(buffer, segment.buffer_id);
+            if (SynchronizeBuffer(buffer, segment.device_addr, segment.size)) {
+                runtime.InvalidateMultiRange(key);
+            }
+            const u32 offset = buffer.Offset(segment.device_addr);
+            buffer.MarkUsage(offset, segment.size);
+            if (is_written) {
+                MarkWrittenBuffer(segment.buffer_id, segment.device_addr, segment.size);
+            }
+            runtime.PushMultiRangeSource(buffer, offset, segment.size);
+        }
+        return runtime.BindMultiRangeStorageBuffer(key, is_written);
+    } else {
+        return false;
+    }
+}
+
+template <class P>
 void BufferCache<P>::BindHostGraphicsStorageBuffers(size_t stage) {
     u32 binding_index = 0;
     ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) {
         const Binding& binding = channel_state->storage_buffers[stage][index];
+        const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0;
+        if (BindMultiRangeStorage(binding, is_written, graphics_segments)) {
+            return;
+        }
         Buffer& buffer = slot_buffers[binding.buffer_id];
         TouchBuffer(buffer, binding.buffer_id);
         const u32 size = binding.size;
@@ -1010,7 +1091,6 @@ void BufferCache<P>::BindHostGraphicsStorageBuffers(size_t stage) {
 
         const u32 offset = buffer.Offset(binding.device_addr);
         buffer.MarkUsage(offset, size);
-        const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0;
 
         if (is_written) {
             MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size);
@@ -1139,6 +1219,11 @@ void BufferCache<P>::BindHostComputeStorageBuffers() {
     u32 binding_index = 0;
     ForEachEnabledBit(channel_state->enabled_compute_storage_buffers, [&](u32 index) {
         const Binding& binding = channel_state->compute_storage_buffers[index];
+        const bool is_written =
+            ((channel_state->written_compute_storage_buffers >> index) & 1) != 0;
+        if (BindMultiRangeStorage(binding, is_written, compute_segments)) {
+            return;
+        }
         Buffer& buffer = slot_buffers[binding.buffer_id];
         TouchBuffer(buffer, binding.buffer_id);
         const u32 size = binding.size;
@@ -1146,8 +1231,6 @@ void BufferCache<P>::BindHostComputeStorageBuffers() {
 
         const u32 offset = buffer.Offset(binding.device_addr);
         buffer.MarkUsage(offset, size);
-        const bool is_written =
-            ((channel_state->written_compute_storage_buffers >> index) & 1) != 0;
 
         if (is_written) {
             MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size);
@@ -1193,6 +1276,7 @@ void BufferCache<P>::BindHostComputeTextureBuffers() {
 
 template <class P>
 void BufferCache<P>::DoUpdateGraphicsBuffers(bool is_indexed) {
+    graphics_segments.clear();
     BufferOperations([&]() {
         if (is_indexed) {
             UpdateIndexBuffer();
@@ -1212,6 +1296,7 @@ void BufferCache<P>::DoUpdateGraphicsBuffers(bool is_indexed) {
 
 template <class P>
 void BufferCache<P>::DoUpdateComputeBuffers() {
+    compute_segments.clear();
     BufferOperations([&]() {
         UpdateComputeUniformBuffers();
         UpdateComputeStorageBuffers();
@@ -1234,11 +1319,11 @@ void BufferCache<P>::UpdateIndexBuffer() {
         auto inline_index_size = static_cast<u32>(draw_state.inline_index_draw_indexes.size());
         u32 buffer_size = Common::AlignUp(inline_index_size, CACHING_PAGESIZE);
         if (inline_buffer_id == NULL_BUFFER_ID) [[unlikely]] {
-            inline_buffer_id = CreateBuffer(0, buffer_size);
+            inline_buffer_id = CreateBuffer(0, buffer_size, false);
         }
         if (slot_buffers[inline_buffer_id].SizeBytes() < buffer_size) [[unlikely]] {
             slot_buffers.erase(inline_buffer_id);
-            inline_buffer_id = CreateBuffer(0, buffer_size);
+            inline_buffer_id = CreateBuffer(0, buffer_size, false);
         }
         channel_state->index_buffer = Binding{
             .device_addr = 0,
@@ -1261,7 +1346,7 @@ void BufferCache<P>::UpdateIndexBuffer() {
     channel_state->index_buffer = Binding{
         .device_addr = *device_addr,
         .size = size,
-        .buffer_id = FindBuffer(*device_addr, size),
+        .buffer_id = FindBuffer(*device_addr, size, false),
     };
 }
 
@@ -1298,7 +1383,7 @@ void BufferCache<P>::UpdateVertexBuffer(u32 index) {
     if (!gpu_memory->IsWithinGPUAddressRange(gpu_addr_end) || size >= 64_MiB) {
         size = static_cast<u32>(gpu_memory->MaxContinuousRange(gpu_addr_begin, size));
     }
-    const BufferId buffer_id = FindBuffer(*device_addr, size);
+    const BufferId buffer_id = FindBuffer(*device_addr, size, false);
     const Binding binding{
         .device_addr = *device_addr,
         .size = size,
@@ -1319,7 +1404,7 @@ void BufferCache<P>::UpdateDrawIndirect() {
         binding = Binding{
             .device_addr = *device_addr,
             .size = static_cast<u32>(size),
-            .buffer_id = FindBuffer(*device_addr, static_cast<u32>(size)),
+            .buffer_id = FindBuffer(*device_addr, static_cast<u32>(size), false),
         };
     };
     if (current_draw_indirect->include_count) {
@@ -1343,7 +1428,7 @@ void BufferCache<P>::UpdateUniformBuffers(size_t stage) {
             channel_state->dirty_uniform_buffers[stage] |= 1U << index;
         }
         // Resolve buffer
-        binding.buffer_id = FindBuffer(binding.device_addr, binding.size);
+        binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false);
     });
 }
 
@@ -1352,8 +1437,10 @@ void BufferCache<P>::UpdateStorageBuffers(size_t stage) {
     ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) {
         // Resolve buffer
         Binding& binding = channel_state->storage_buffers[stage][index];
-        const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size);
+        const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size, false);
         binding.buffer_id = buffer_id;
+        const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0;
+        ResolveMultiRangeStorage(binding, is_written, graphics_segments);
     });
 }
 
@@ -1361,7 +1448,7 @@ template <class P>
 void BufferCache<P>::UpdateTextureBuffers(size_t stage) {
     ForEachEnabledBit(channel_state->enabled_texture_buffers[stage], [&](u32 index) {
         Binding& binding = channel_state->texture_buffers[stage][index];
-        binding.buffer_id = FindBuffer(binding.device_addr, binding.size);
+        binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false);
     });
 }
 
@@ -1385,7 +1472,7 @@ void BufferCache<P>::UpdateTransformFeedbackBuffer(u32 index) {
         channel_state->transform_feedback_buffers[index] = NULL_BINDING;
         return;
     }
-    const BufferId buffer_id = FindBuffer(*device_addr, size);
+    const BufferId buffer_id = FindBuffer(*device_addr, size, false);
     channel_state->transform_feedback_buffers[index] = Binding{
         .device_addr = *device_addr,
         .size = size,
@@ -1407,7 +1494,7 @@ void BufferCache<P>::UpdateComputeUniformBuffers() {
                 binding.size = cbuf.size;
             }
         }
-        binding.buffer_id = FindBuffer(binding.device_addr, binding.size);
+        binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false);
     });
 }
 
@@ -1416,7 +1503,10 @@ void BufferCache<P>::UpdateComputeStorageBuffers() {
     ForEachEnabledBit(channel_state->enabled_compute_storage_buffers, [&](u32 index) {
         // Resolve buffer
         Binding& binding = channel_state->compute_storage_buffers[index];
-        binding.buffer_id = FindBuffer(binding.device_addr, binding.size);
+        binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false);
+        const bool is_written =
+            ((channel_state->written_compute_storage_buffers >> index) & 1) != 0;
+        ResolveMultiRangeStorage(binding, is_written, compute_segments);
     });
 }
 
@@ -1424,7 +1514,7 @@ template <class P>
 void BufferCache<P>::UpdateComputeTextureBuffers() {
     ForEachEnabledBit(channel_state->enabled_compute_texture_buffers, [&](u32 index) {
         Binding& binding = channel_state->compute_texture_buffers[index];
-        binding.buffer_id = FindBuffer(binding.device_addr, binding.size);
+        binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false);
     });
 }
 
@@ -1440,7 +1530,7 @@ void BufferCache<P>::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3
 }
 
 template <class P>
-BufferId BufferCache<P>::FindBuffer(DAddr device_addr, u32 size) {
+BufferId BufferCache<P>::FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible) {
     if (device_addr == 0) {
         return NULL_BUFFER_ID;
     }
@@ -1450,10 +1540,18 @@ BufferId BufferCache<P>::FindBuffer(DAddr device_addr, u32 size) {
         Buffer& buffer = slot_buffers[buffer_id];
         WaitForGpuFenceIfNeeded(buffer);
         if (buffer.IsInBounds(device_addr, size)) {
-            return buffer_id;
+            bool usable = true;
+            if constexpr (requires { buffer.IsSparseCompatible(); }) {
+                if (sparse_compatible && !buffer.IsSparseCompatible()) {
+                    usable = false;
+                }
+            }
+            if (usable) {
+                return buffer_id;
+            }
         }
     }
-    return CreateBuffer(device_addr, size);
+    return CreateBuffer(device_addr, size, sparse_compatible);
 }
 
 template <class P>
@@ -1575,13 +1673,15 @@ void BufferCache<P>::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id,
 }
 
 template <class P>
-BufferId BufferCache<P>::CreateBuffer(DAddr device_addr, u32 wanted_size) {
+BufferId BufferCache<P>::CreateBuffer(DAddr device_addr, u32 wanted_size,
+                                      bool sparse_compatible) {
     DAddr device_addr_end = Common::AlignUp(device_addr + wanted_size, CACHING_PAGESIZE);
     device_addr = Common::AlignDown(device_addr, CACHING_PAGESIZE);
     wanted_size = static_cast<u32>(device_addr_end - device_addr);
     const OverlapResult overlap = ResolveOverlaps(device_addr, wanted_size);
     const u32 size = static_cast<u32>(overlap.end - overlap.begin);
-    const BufferId new_buffer_id = slot_buffers.insert(runtime, overlap.begin, size);
+    const BufferId new_buffer_id =
+        slot_buffers.insert(runtime, overlap.begin, size, sparse_compatible);
     auto& new_buffer = slot_buffers[new_buffer_id];
     const size_t size_bytes = new_buffer.SizeBytes();
     runtime.ClearBuffer(new_buffer, 0, size_bytes, 0);
@@ -1745,7 +1845,7 @@ void BufferCache<P>::InlineMemoryImplementation(DAddr dest_address, size_t copy_
     ClearDownload(dest_address, copy_size);
     gpu_modified_ranges.Subtract(dest_address, copy_size);
 
-    BufferId buffer_id = FindBuffer(dest_address, static_cast<u32>(copy_size));
+    BufferId buffer_id = FindBuffer(dest_address, static_cast<u32>(copy_size), false);
     auto& buffer = slot_buffers[buffer_id];
     SynchronizeBuffer(buffer, dest_address, static_cast<u32>(copy_size));
 
@@ -1831,6 +1931,9 @@ void BufferCache<P>::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64
 
 template <class P>
 void BufferCache<P>::DeleteBuffer(BufferId buffer_id, bool do_not_mark) {
+    if constexpr (requires { runtime.OnBufferDeleted(slot_buffers[buffer_id]); }) {
+        runtime.OnBufferDeleted(slot_buffers[buffer_id]);
+    }
     bool dirty_index{false};
     boost::container::small_vector<u64, NUM_VERTEX_BUFFERS> dirty_vertex_buffers;
     const auto scalar_replace = [buffer_id](Binding& binding) {
@@ -1934,9 +2037,14 @@ Binding BufferCache<P>::StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index,
     // The end address used for size calculation does not need to be aligned
     const DAddr cpu_end = Common::AlignUp(*device_addr + size, Core::DEVICE_PAGESIZE);
 
+    u32 binding_size = static_cast<u32>(cpu_end - *aligned_device_addr);
+    if (is_written) {
+        binding_size = aligned_size;
+    }
     const Binding binding{
         .device_addr = *aligned_device_addr,
-        .size = is_written ? aligned_size : static_cast<u32>(cpu_end - *aligned_device_addr),
+        .gpu_addr = aligned_gpu_addr,
+        .size = binding_size,
         .buffer_id = BufferId{},
     };
     return binding;