video_core: less waiting on the GPU caches' locks

- The texture and buffer caches' mutexes spin briefly before sleeping
  (Common::SpinRecursiveMutex): the GPU thread, the fencing thread and
  CPU write traps take them tens of thousands of times a second for
  microseconds, and a sleeping waiter's wake-up took longer than that.
  Title 1's GPU thread slept 7.7% of its time there, now 3.7%.
- Fences commit and pop texture and buffer downloads separately, and
  only when there are any (one fence per compute dispatch in Switch 2
  titles): the fencing thread no longer holds both caches while it
  copies buffer downloads.
- Inline uploads and pre-draw invalidation take the caches one at a
  time again: holding one while std::lock waited for the other waited
  longer.

8 files changed, 161 insertions, 17 deletions

diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
index aae4afd180..d3e7b03cd2 100644
--- a/src/common/CMakeLists.txt
+++ b/src/common/CMakeLists.txt
@@ -96,6 +96,7 @@ add_library(
     ${CMAKE_CURRENT_BINARY_DIR}/scm_rev.cpp
     scm_rev.h
     scope_exit.h
+    spin_recursive_mutex.h
     scratch_buffer.h
     settings.cpp
     settings.h
diff --git a/src/common/spin_recursive_mutex.h b/src/common/spin_recursive_mutex.h
new file mode 100644
index 0000000000..291dd4e41e
--- /dev/null
+++ b/src/common/spin_recursive_mutex.h
@@ -0,0 +1,98 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <atomic>
+#include <cstdint>
+#include <thread>
+
+#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)
+#include <immintrin.h>
+#endif
+
+namespace Common {
+
+/// A recursive mutex that spins briefly before sleeping. The GPU caches' locks are taken tens of
+/// thousands of times a second by the GPU thread, the fencing thread and CPU write traps, and
+/// held for microseconds: std::recursive_mutex put a waiting thread to sleep at once, and its
+/// wake-up took longer than the critical section (title 1's GPU thread spent ~8% asleep there).
+class SpinRecursiveMutex {
+public:
+    void lock() {
+        const std::thread::id self = std::this_thread::get_id();
+        if (owner.load(std::memory_order_relaxed) == self) {
+            ++depth;
+            return;
+        }
+        std::uint32_t expected = 0;
+        if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire,
+                                           std::memory_order_relaxed)) {
+            LockContended();
+        }
+        owner.store(self, std::memory_order_relaxed);
+        depth = 1;
+    }
+
+    bool try_lock() {
+        const std::thread::id self = std::this_thread::get_id();
+        if (owner.load(std::memory_order_relaxed) == self) {
+            ++depth;
+            return true;
+        }
+        std::uint32_t expected = 0;
+        if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire,
+                                           std::memory_order_relaxed)) {
+            return false;
+        }
+        owner.store(self, std::memory_order_relaxed);
+        depth = 1;
+        return true;
+    }
+
+    void unlock() {
+        if (--depth != 0) {
+            return;
+        }
+        owner.store(std::thread::id{}, std::memory_order_relaxed);
+        if (state.exchange(0, std::memory_order_release) == 2) {
+            state.notify_one();
+        }
+    }
+
+private:
+    // About 10 microseconds of spinning before sleeping.
+    static constexpr int SpinCount = 1024;
+
+    static void Pause() {
+#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)
+        _mm_pause();
+#elif defined(__aarch64__)
+        asm volatile("yield");
+#endif
+    }
+
+    void LockContended() {
+        for (int spin = 0; spin < SpinCount; ++spin) {
+            Pause();
+            if (state.load(std::memory_order_relaxed) == 0) {
+                std::uint32_t expected = 0;
+                if (state.compare_exchange_weak(expected, 1, std::memory_order_acquire,
+                                                std::memory_order_relaxed)) {
+                    return;
+                }
+            }
+        }
+        // 2: held with (possible) sleepers; taken this way it stays marked, so its unlock wakes
+        // the next sleeper.
+        while (state.exchange(2, std::memory_order_acquire) != 0) {
+            state.wait(2, std::memory_order_relaxed);
+        }
+    }
+
+    std::atomic<std::uint32_t> state{0}; ///< 0 free, 1 held, 2 held with sleepers
+    std::atomic<std::thread::id> owner{};
+    std::uint32_t depth{}; ///< Recursion depth, touched only by the owner
+};
+
+} // namespace Common
diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h
index e4aa5896e5..57011752cc 100644
--- a/src/video_core/buffer_cache/buffer_cache_base.h
+++ b/src/video_core/buffer_cache/buffer_cache_base.h
@@ -21,6 +21,7 @@
 #include <boost/container/small_vector.hpp>
 
 #include "common/common_types.h"
+#include "common/spin_recursive_mutex.h"
 #include "common/div_ceil.h"
 #include "common/literals.h"
 #include "common/lru_cache.h"
@@ -339,7 +340,7 @@ public:
         } while (channel_state->has_deleted_buffers);
     }
 
-    std::recursive_mutex mutex;
+    Common::SpinRecursiveMutex mutex;
     Runtime& runtime;
     bool any_buffer_uploaded = false;
 
diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h
index 65d8f24aae..eb72529da0 100644
--- a/src/video_core/fence_manager.h
+++ b/src/video_core/fence_manager.h
@@ -81,7 +81,7 @@ public:
         if constexpr (!can_async_check) {
             TryReleasePendingFences<false>();
         }
-        CommitAsyncFlushes();
+        const u8 committed_flushes = CommitAsyncFlushes();
         TFence new_fence = CreateFence(!should_flush);
         if constexpr (can_async_check) {
             guard.lock();
@@ -94,6 +94,7 @@ public:
             pending_operations.emplace_back(std::move(uncommitted_operations));
             uncommitted_operations.clear(); // moved-from: make it empty explicitly
         }
+        pending_flushes.push_back(committed_flushes);
         QueueFence(new_fence);
         if (!delay_fence) {
             func();
@@ -186,7 +187,8 @@ private:
                     return;
                 }
             }
-            PopAsyncFlushes();
+            PopAsyncFlushes(pending_flushes.front());
+            pending_flushes.pop_front();
             auto operations = std::move(pending_operations.front());
             pending_operations.pop_front();
             for (auto& operation : operations) {
@@ -207,6 +209,7 @@ private:
 
         TFence current_fence;
         std::deque<std::function<void()>> current_operations;
+        u8 current_flushes{};
         while (!stop_token.stop_requested()) {
             {
                 std::unique_lock lock(guard);
@@ -216,13 +219,15 @@ private:
                 }
                 current_fence = std::move(fences.front());
                 current_operations = std::move(pending_operations.front());
+                current_flushes = pending_flushes.front();
                 fences.pop();
                 pending_operations.pop_front();
+                pending_flushes.pop_front();
             }
             if (!current_fence->IsStubbed()) {
                 WaitFence(current_fence);
             }
-            PopAsyncFlushes();
+            PopAsyncFlushes(current_flushes);
             for (auto& operation : current_operations) {
                 operation();
             }
@@ -245,27 +250,51 @@ private:
                query_cache.HasUncommittedFlushes();
     }
 
-    void PopAsyncFlushes() {
-        {
-            std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
+    static constexpr u8 TextureFlushes = 1;
+    static constexpr u8 BufferFlushes = 2;
+
+    /// committed: what CommitAsyncFlushes returned for this fence.
+    void PopAsyncFlushes(u8 committed) {
+        if (committed & TextureFlushes) {
+            std::scoped_lock lock{texture_cache.mutex};
             texture_cache.PopAsyncFlushes();
+        }
+        if (committed & BufferFlushes) {
+            std::scoped_lock lock{buffer_cache.mutex};
             buffer_cache.PopAsyncFlushes();
         }
         query_cache.PopAsyncFlushes();
     }
 
-    void CommitAsyncFlushes() {
+    /// Returns which caches committed downloads for this fence (TextureFlushes, BufferFlushes).
+    /// Switch 2 titles signal a fence per compute dispatch (thousands a second), mostly with
+    /// nothing or only buffer data to download: the fencing thread then takes only the lock of
+    /// the cache with work, instead of holding both while it copies (the GPU thread's draws and
+    /// uploads waited for it).
+    u8 CommitAsyncFlushes() {
+        u8 committed = 0;
         {
-            std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
-            texture_cache.CommitAsyncFlushes();
-            buffer_cache.CommitAsyncFlushes();
+            std::scoped_lock lock{texture_cache.mutex};
+            if (texture_cache.HasAsyncFlushWork()) {
+                texture_cache.CommitAsyncFlushes();
+                committed |= TextureFlushes;
+            }
+        }
+        {
+            std::scoped_lock lock{buffer_cache.mutex};
+            if (buffer_cache.HasUncommittedFlushes()) {
+                buffer_cache.CommitAsyncFlushes();
+                committed |= BufferFlushes;
+            }
         }
         query_cache.CommitAsyncFlushes();
+        return committed;
     }
 
     std::queue<TFence> fences;
     std::deque<std::function<void()>> uncommitted_operations;
     std::deque<std::deque<std::function<void()>>> pending_operations;
+    std::deque<u8> pending_flushes; ///< Per fence: the caches that committed downloads
 
     std::mutex guard;
     std::mutex ring_guard;
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp
index 1d77d28c46..4e0972d1d2 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp
@@ -716,7 +716,7 @@ void RasterizerOpenGL::AccelerateInlineToMemory(GPUVAddr address, size_t copy_si
     }
     gpu_memory->WriteBlockUnsafe(address, memory.data(), copy_size);
     {
-        std::unique_lock<std::recursive_mutex> lock{buffer_cache.mutex};
+        std::unique_lock lock{buffer_cache.mutex};
         if (!buffer_cache.InlineMemory(*cpu_addr, copy_size, memory)) {
             buffer_cache.WriteMemory(*cpu_addr, copy_size);
         }
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index 886c5dbfe4..429b2d63ef 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -1036,11 +1036,13 @@ void RasterizerVulkan::InvalidateRegion(DAddr addr, u64 size, VideoCommon::Cache
 
 void RasterizerVulkan::InnerInvalidation(std::span<const std::pair<DAddr, std::size_t>> sequences) {
     {
-        // Both caches under one lock: this runs before most draws (constant buffer updates).
-        std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
+        std::scoped_lock lock{texture_cache.mutex};
         for (const auto& [addr, size] : sequences) {
             texture_cache.WriteMemory(addr, size);
         }
+    }
+    {
+        std::scoped_lock lock{buffer_cache.mutex};
         for (const auto& [addr, size] : sequences) {
             buffer_cache.WriteMemory(addr, size);
         }
@@ -1248,12 +1250,16 @@ void RasterizerVulkan::AccelerateInlineToMemory(GPUVAddr address, size_t copy_si
         return;
     }
     gpu_memory->WriteBlockUnsafe(address, memory.data(), copy_size);
+    // One cache at a time: holding both while waiting for either (std::lock) waited longer, the
+    // fencing thread and CPU write traps holding one of them most of the time.
     {
-        // Both caches under one lock (title 1 makes thousands of inline uploads a second).
-        std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
+        std::scoped_lock lock{buffer_cache.mutex};
         if (!buffer_cache.InlineMemory(*cpu_addr, copy_size, memory)) {
             buffer_cache.WriteMemory(*cpu_addr, copy_size);
         }
+    }
+    {
+        std::scoped_lock lock{texture_cache.mutex};
         texture_cache.WriteMemory(*cpu_addr, copy_size);
     }
     pipeline_cache.InvalidateRegion(*cpu_addr, copy_size);
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 13dfbba20e..fb8adef161 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -990,6 +990,11 @@ bool TextureCache<P>::HasUncommittedFlushes() const noexcept {
 }
 
 template <class P>
+bool TextureCache<P>::HasAsyncFlushWork() const noexcept {
+    return !uncommitted_downloads.empty() || !uncommitted_async_buffers.empty();
+}
+
+template <class P>
 bool TextureCache<P>::ShouldWaitAsyncFlushes() const noexcept {
     return !committed_downloads.empty() && !committed_downloads.front().empty();
 }
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index 3c96c91a93..11ac8a2421 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -23,6 +23,7 @@
 #include <queue>
 
 #include "common/common_types.h"
+#include "common/spin_recursive_mutex.h"
 #include "common/hash.h"
 #include "common/literals.h"
 #include "common/lru_cache.h"
@@ -244,6 +245,9 @@ public:
     /// Return true when there are uncommitted images to be downloaded
     [[nodiscard]] bool HasUncommittedFlushes() const noexcept;
 
+    /// Whether CommitAsyncFlushes would commit anything (downloads or their buffers).
+    [[nodiscard]] bool HasAsyncFlushWork() const noexcept;
+
     /// Return true when the caller should wait for async downloads
     [[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept;
 
@@ -276,7 +280,7 @@ public:
     /// Prepare an image to be used
     void PrepareImage(ImageId image_id, bool is_modification, bool invalidate);
 
-    std::recursive_mutex mutex;
+    Common::SpinRecursiveMutex mutex;
 
 private:
     /// Iterate over all page indices in a range