vulkan: repack 69-byte vertices only when needed

SetVertexRepackLayout returned old_mask | new_mask, which stays non-zero
on every draw that uses a 69-byte binding. PrepareDraw then re-dirtied the
vertex buffers and vertex input each time, so the repack compute pass ran
and the render pass ended once per draw, which is costly on tiled GPUs.

Return old_mask ^ new_mask so only a layout change re-dirties the
bindings. The repacked copy only stays valid while the source data is
unchanged, so the buffer cache now flags the vertex buffers overlapping a
CPU write, inline or DMA write, or GPU write, and re-dirties the repacked
ones before binding. Command buffer submission already invalidates all
vertex buffer state, which keeps the staging copy within its tick.

Also pass the source buffer size to the repack pass and clamp the
word-rounded storage descriptor range to it; it could run up to three
bytes past the end of the buffer. A binding that does not fit is skipped
like one over the device limits.

Follow-up to e38b738b84.

Assisted-by: Claude Sonnet 5.5 (implementation and commit message)

5 files changed, 23 insertions, 12 deletions

diff --git a/src/video_core/renderer_vulkan/vertex_repack.h b/src/video_core/renderer_vulkan/vertex_repack.h
index b0849c25c7..456d1c6f20 100644
--- a/src/video_core/renderer_vulkan/vertex_repack.h
+++ b/src/video_core/renderer_vulkan/vertex_repack.h
@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <algorithm>
 #include <cstdlib>
 #include <cstring>
 #include <limits>
@@ -31,8 +32,11 @@ struct VertexRepackParameters {
     u32 dispatch_groups;
 };
 
-inline std::optional<VertexRepackParameters> MakeVertexRepackParameters(
-    u32 offset, u32 size, u64 alignment, u64 max_storage_range, u32 max_dispatch_groups) {
+inline std::optional<VertexRepackParameters> MakeVertexRepackParameters(u32 offset, u32 size,
+                                                                        u64 alignment,
+                                                                        u64 max_storage_range,
+                                                                        u32 max_dispatch_groups,
+                                                                        u64 buffer_size) {
     if (size == 0 || alignment == 0) {
         return std::nullopt;
     }
@@ -40,9 +44,11 @@ inline std::optional<VertexRepackParameters> MakeVertexRepackParameters(
     const u64 output_size = vertices * 72;
     const u64 base = u64{offset} / alignment * alignment;
     const u64 bias = offset - base;
-    const u64 range = (bias + size + 3) / 4 * 4;
+    // The word-rounded range must not run past the end of the source buffer
+    const u64 available = buffer_size > base ? buffer_size - base : 0;
+    const u64 range = (std::min)((bias + size + 3) / 4 * 4, available);
     const u64 groups = (vertices * 18 + 127) / 128;
-    if (output_size > std::numeric_limits<u32>::max() ||
+    if (range < bias + size || output_size > std::numeric_limits<u32>::max() ||
         range > std::numeric_limits<u32>::max() || output_size > max_storage_range ||
         range > max_storage_range || groups > max_dispatch_groups) {
         return std::nullopt;
diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
index 2449265509..4d5ccb6520 100644
--- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
@@ -394,7 +394,7 @@ u32 BufferCacheRuntime::SetVertexRepackLayout(const Tegra::Engines::Maxwell3D::R
         LOG_DEBUG(Render_Vulkan, "Vertex repack selection: mask={:#x}, dynamic_input={}",
                   vertex_repack_mask, dynamic_input);
     }
-    return old_mask | vertex_repack_mask;
+    return old_mask ^ vertex_repack_mask;
 }
 
 StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) {
@@ -657,7 +657,8 @@ void BufferCacheRuntime::BindVertexBuffer(u32 index, VkBuffer buffer, u32 offset
         return;
     }
     if (buffer != VK_NULL_HANDLE && size != 0 && IsVertexRepacked(index)) {
-        const auto repacked = vertex_repack_pass->Assemble(buffer, offset, size);
+        const auto repacked =
+            vertex_repack_pass->Assemble(buffer, offset, size, u64{offset} + size);
         if (!repacked) {
             return;
         }
@@ -702,7 +703,8 @@ void BufferCacheRuntime::BindVertexBuffers(VideoCommon::HostBindings<Buffer>& bi
         if (handle != VK_NULL_HANDLE && IsVertexRepacked(binding) && bindings.strides[i] == 69 &&
             bindings.sizes[i] > 0) {
             const auto repacked = vertex_repack_pass->Assemble(
-                handle, static_cast<u32>(bindings.offsets[i]), static_cast<u32>(bindings.sizes[i]));
+                handle, static_cast<u32>(bindings.offsets[i]), static_cast<u32>(bindings.sizes[i]),
+                bindings.buffers[i]->SizeBytes());
             if (!repacked) {
                 continue;
             }
diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h
index d3295ec6ed..a8d0274016 100644
--- a/src/video_core/renderer_vulkan/vk_buffer_cache.h
+++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h
@@ -110,6 +110,7 @@ public:
 
     void TickFrame(Common::SlotVector<Buffer>& slot_buffers) noexcept;
 
+    /// Returns the mask of bindings whose repacked state changed.
     u32 SetVertexRepackLayout(const Tegra::Engines::Maxwell3D::Regs& regs, bool dynamic_input);
 
     bool IsVertexRepacked(u32 binding) const {
diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp
index cb8dcde42f..5902b7d8f5 100644
--- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp
+++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp
@@ -335,10 +335,11 @@ VertexRepackPass::VertexRepackPass(const Device& device_, Scheduler& scheduler_,
       scheduler{scheduler_}, staging_pool{staging_pool_}, descriptor_queue{descriptor_queue_} {}
 
 std::optional<std::pair<VkBuffer, VkDeviceSize>> VertexRepackPass::Assemble(VkBuffer source,
-                                                                           u32 offset, u32 size) {
+                                                                            u32 offset, u32 size,
+                                                                            u64 buffer_size) {
     const auto parameters = MakeVertexRepackParameters(
         offset, size, device.GetStorageBufferAlignment(), device.GetMaxStorageBufferRange(),
-        device.GetMaxComputeWorkGroupCount()[0]);
+        device.GetMaxComputeWorkGroupCount()[0], buffer_size);
     if (!parameters) {
         LOG_ERROR(Render_Vulkan, "Vertex repack exceeds device limits: offset={}, size={}", offset, size);
         return std::nullopt;
diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h
index 27fffbbbb7..9687bfff91 100644
--- a/src/video_core/renderer_vulkan/vk_compute_pass.h
+++ b/src/video_core/renderer_vulkan/vk_compute_pass.h
@@ -60,9 +60,10 @@ class VertexRepackPass final : public ComputePass {
 public:
     VertexRepackPass(const Device& device, Scheduler& scheduler, DescriptorPool& descriptor_pool,
                      StagingBufferPool& staging_pool, ComputePassDescriptorQueue& descriptor_queue);
-    /// Returns nothing when the range exceeds the device limits; the binding is then left as is.
-    std::optional<std::pair<VkBuffer, VkDeviceSize>> Assemble(VkBuffer source, u32 offset,
-                                                              u32 size);
+    /// Returns nothing when the range exceeds the device limits or the source buffer; the binding
+    /// is then left as is.
+    std::optional<std::pair<VkBuffer, VkDeviceSize>> Assemble(VkBuffer source, u32 offset, u32 size,
+                                                              u64 buffer_size);
 
 private:
     Scheduler& scheduler;