video_core: skip empty fences; one lock per cache per fence; single-pass overlap subtract

Switch 2 titles signal a fence for each compute dispatch's release (5,000-7,000 a
second in title 1). With fast GPU fences the callback runs at once, so a fence with
no downloads, queries or queued operations does nothing: those (~20%) are no longer
made (one in 64 still is, for the operations fences carry). Each cache's lock is now
taken once per fence (the flush check took both together first). The overlap range
set's Subtract restarted its search after every erase (quadratic); the fencing thread
runs it for every downloaded range while holding the buffer cache. The GPU profiler
counts fences, skipped fences and buffer download bytes.

6 files changed, 105 insertions, 38 deletions

diff --git a/src/common/range_sets.inc b/src/common/range_sets.inc
index 3edd8c8a43..57b07d05f6 100644
--- a/src/common/range_sets.inc
+++ b/src/common/range_sets.inc
@@ -116,28 +116,25 @@ struct OverlapRangeSet<AddressType>::OverlapRangeSetImpl {
         }
         AddressType end_address = base_address + static_cast<AddressType>(size);
         IntervalType interval{base_address, end_address};
-        bool any_removals = false;
         m_split_ranges_set += std::make_pair(interval, -amount);
-        do {
-            any_removals = false;
-            auto it = m_split_ranges_set.lower_bound(interval);
-            if (it == m_split_ranges_set.end()) {
-                return;
+        // One pass, erasing the segments that dropped to zero or below in address order (the
+        // search restarted after every erase: quadratic in the segments of the range, and the
+        // GPU fencing thread does this for each downloaded buffer range while holding the buffer
+        // cache's lock). Erasing a node leaves the other iterators valid.
+        auto it = m_split_ranges_set.lower_bound(interval);
+        const auto end_it = m_split_ranges_set.upper_bound(interval);
+        while (it != end_it) {
+            if (it->second > 0) {
+                ++it;
+                continue;
             }
-            auto end_it = m_split_ranges_set.upper_bound(interval);
-            for (; it != end_it; it++) {
-                if (it->second <= 0) {
-                    if constexpr (has_on_delete) {
-                        if (it->second == 0) {
-                            on_delete(it->first.lower(), it->first.upper());
-                        }
-                    }
-                    any_removals = true;
-                    m_split_ranges_set.erase(it);
-                    break;
+            if constexpr (has_on_delete) {
+                if (it->second == 0) {
+                    on_delete(it->first.lower(), it->first.upper());
                 }
             }
-        } while (any_removals);
+            m_split_ranges_set.erase(it++);
+        }
     }
 
     template <typename Func>
diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index ad2e200438..e2de0eede1 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -17,6 +17,7 @@
 #include "video_core/buffer_cache/buffer_cache_base.h"
 #include "video_core/guest_memory.h"
 #include "video_core/host1x/gpu_device_memory_manager.h"
+#include "video_core/nx2_profile.h"
 #include "video_core/texture_cache/util.h"
 
 namespace VideoCommon {
@@ -666,6 +667,8 @@ void BufferCache<P>::CommitAsyncFlushesHigh() {
         async_buffers.emplace_back(std::optional<Async_Buffer>{});
         return;
     }
+    Nx2Profile::Add(Nx2Profile::Counter::DownloadBytes, total_size_bytes);
+    Nx2Profile::Add(Nx2Profile::Counter::DownloadCopies, downloads.size());
     auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes, true);
     boost::container::small_vector<BufferCopy, 4> normalized_copies;
     runtime.PreCopyBarrier();
diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h
index eb72529da0..f8ee6261bb 100644
--- a/src/video_core/fence_manager.h
+++ b/src/video_core/fence_manager.h
@@ -22,6 +22,7 @@
 #include "video_core/gpu.h"
 #include "video_core/host1x/host1x.h"
 #include "video_core/host1x/syncpoint_manager.h"
+#include "video_core/nx2_profile.h"
 #include "video_core/rasterizer_interface.h"
 
 namespace VideoCommon {
@@ -76,12 +77,21 @@ public:
     }
 
     void SignalFence(std::function<void()>&& func) {
+        Nx2Profile::Scope profile_scope{Nx2Profile::Zone::Fence};
         const bool delay_fence = Settings::IsGPUFenceBehaviorDefault() ? Settings::IsGPULevelHigh() : Settings::IsGPUFenceBehaviorBalanced() || Settings::IsGPUFenceBehaviorAccurate();
-        const bool should_flush = ShouldFlush();
         if constexpr (!can_async_check) {
             TryReleasePendingFences<false>();
         }
-        const u8 committed_flushes = CommitAsyncFlushes();
+        auto [should_flush, committed_flushes] = CommitCacheFlushes();
+        if (!delay_fence && committed_flushes == 0 && SkipEmptyFence()) {
+            Nx2Profile::Scope skipped_scope{Nx2Profile::Zone::FenceSkipped};
+            func();
+            rasterizer.InvalidateGPUCache();
+            return;
+        }
+        skipped_fences = 0;
+        should_flush |= query_cache.HasUncommittedFlushes();
+        query_cache.CommitAsyncFlushes();
         TFence new_fence = CreateFence(!should_flush);
         if constexpr (can_async_check) {
             guard.lock();
@@ -244,12 +254,6 @@ private:
                query_cache.ShouldWaitAsyncFlushes();
     }
 
-    bool ShouldFlush() const {
-        std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
-        return texture_cache.HasUncommittedFlushes() || buffer_cache.HasUncommittedFlushes() ||
-               query_cache.HasUncommittedFlushes();
-    }
-
     static constexpr u8 TextureFlushes = 1;
     static constexpr u8 BufferFlushes = 2;
 
@@ -266,35 +270,66 @@ private:
         query_cache.PopAsyncFlushes();
     }
 
-    /// Returns which caches committed downloads for this fence (TextureFlushes, BufferFlushes).
-    /// Switch 2 titles signal a fence per compute dispatch (thousands a second), mostly with
-    /// nothing or only buffer data to download: the fencing thread then takes only the lock of
-    /// the cache with work, instead of holding both while it copies (the GPU thread's draws and
-    /// uploads waited for it).
-    u8 CommitAsyncFlushes() {
-        u8 committed = 0;
+    struct CacheFlushes {
+        bool should_flush;  ///< Downloads to wait for: the fence must reach the host GPU
+        u8 committed;       ///< TextureFlushes, BufferFlushes: the caches that committed downloads
+    };
+
+    /// Commits the texture and buffer caches' downloads for a new fence (the query cache's are
+    /// committed once a fence is made). Switch 2 titles signal a fence per compute dispatch
+    /// (thousands a second), mostly with nothing or only buffer data to download: each cache's
+    /// lock is taken once, alone, and the fencing thread later takes only the lock of the cache
+    /// with work, instead of holding both while it copies (the GPU thread's draws and uploads
+    /// waited for it).
+    CacheFlushes CommitCacheFlushes() {
+        CacheFlushes flushes{};
         {
             std::scoped_lock lock{texture_cache.mutex};
+            flushes.should_flush = texture_cache.HasUncommittedFlushes();
             if (texture_cache.HasAsyncFlushWork()) {
                 texture_cache.CommitAsyncFlushes();
-                committed |= TextureFlushes;
+                flushes.committed |= TextureFlushes;
             }
         }
         {
             std::scoped_lock lock{buffer_cache.mutex};
             if (buffer_cache.HasUncommittedFlushes()) {
                 buffer_cache.CommitAsyncFlushes();
-                committed |= BufferFlushes;
+                flushes.should_flush = true;
+                flushes.committed |= BufferFlushes;
             }
         }
-        query_cache.CommitAsyncFlushes();
-        return committed;
+        return flushes;
+    }
+
+    /// Whether a fence with no downloads would do nothing, so need not be made: with fast GPU
+    /// fences its callback runs at once anyway, and most of the per-dispatch fences of Switch 2
+    /// titles have no queries or queued operations either. Making, queueing and retiring them
+    /// cost the GPU thread ~5% and woke the fencing thread for each. One in 64 is still made, so
+    /// the operations fences carry along (query slot cleanup) keep running.
+    bool SkipEmptyFence() {
+        if constexpr (requires { query_cache.HasPendingFenceWork(); }) {
+            if (skipped_fences >= 64 || query_cache.HasPendingFenceWork()) {
+                return false;
+            }
+            {
+                std::scoped_lock lock{uncommitted_guard};
+                if (!uncommitted_operations.empty()) {
+                    return false;
+                }
+            }
+            ++skipped_fences;
+            return true;
+        } else {
+            return false;
+        }
     }
 
     std::queue<TFence> fences;
     std::deque<std::function<void()>> uncommitted_operations;
     std::deque<std::deque<std::function<void()>>> pending_operations;
     std::deque<u8> pending_flushes; ///< Per fence: the caches that committed downloads
+    u32 skipped_fences{}; ///< Empty fences skipped since the last one made (GPU thread only)
 
     std::mutex guard;
     std::mutex ring_guard;
diff --git a/src/video_core/nx2_profile.h b/src/video_core/nx2_profile.h
index a162ce16a3..088bfc5fe7 100644
--- a/src/video_core/nx2_profile.h
+++ b/src/video_core/nx2_profile.h
@@ -45,6 +45,8 @@ enum class Zone : u32 {
     DynamicState,
     DrawRecord,
     TextureUpload,
+    Fence,        ///< Fences made (SignalFence)
+    FenceSkipped, ///< Empty fences not made
     Count,
 };
 
@@ -54,7 +56,8 @@ inline constexpr std::array<const char*, static_cast<size_t>(Zone::Count)> zone_
     "clear",         "compute",       "flush work",     "pipeline lookup", "configure",
     "sync descriptors", "read handles", "fill image views", "texture buffers", "update buffers",
     "descriptor acquire", "bind stage buffers", "push descriptors", "render targets",
-    "configure draw", "dynamic state", "draw record", "texture upload",
+    "configure draw", "dynamic state", "draw record", "texture upload", "fence",
+    "fence skipped",
 };
 
 inline bool Enabled() {
@@ -71,6 +74,7 @@ inline u64 Now() {
 struct ThreadData {
     std::array<u64, static_cast<size_t>(Zone::Count)> exclusive_ns{};
     std::array<u64, static_cast<size_t>(Zone::Count)> count{};
+    std::array<u64, 2> counters{}; ///< See Counter
     u64 child_ns{};    ///< Time of the zones nested in the current one
     u64 window_start{};
 };
@@ -110,6 +114,18 @@ private:
     u64 parent_child_ns{};
 };
 
+/// Amounts counted on the GPU thread, logged per second with the zones.
+enum class Counter : u32 {
+    DownloadBytes,  ///< Buffer data copied back from the host GPU at fences
+    DownloadCopies, ///< Buffer ranges copied back
+};
+
+inline void Add(Counter counter, u64 amount) {
+    if (Enabled()) {
+        t_data.counters[static_cast<size_t>(counter)] += amount;
+    }
+}
+
 /// Called by the GPU thread between commands: logs and resets the totals every 10 s.
 inline void Report() {
     if (!Enabled()) {
@@ -142,6 +158,9 @@ inline void Report() {
         const double calls = static_cast<double>(data.count[i]) * 1e9 / window;
         line += fmt::format(" | {} {:.1f} ms ({:.0f}/s)", zone_names[i], ms, calls);
     }
+    line += fmt::format(" | buffer downloads {:.0f} KiB in {:.0f} copies",
+                        static_cast<double>(data.counters[0]) * 1e9 / 1024.0 / window,
+                        static_cast<double>(data.counters[1]) * 1e9 / window);
     LOG_INFO(HW_GPU, "GPU thread per second:{}", line);
     data = {};
     data.window_start = now;
diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h
index 245ab1996c..60ca50eb1f 100644
--- a/src/video_core/query_cache/query_cache.h
+++ b/src/video_core/query_cache/query_cache.h
@@ -520,6 +520,16 @@ bool QueryCacheBase<Traits>::HasUncommittedFlushes() const {
 }
 
 template <typename Traits>
+bool QueryCacheBase<Traits>::HasPendingFenceWork() const {
+    bool result = false;
+    impl->ForEachStreamer([&result](StreamerInterface* streamer) {
+        result |= streamer->HasUnsyncedQueries() || streamer->HasPendingSync();
+        return result;
+    });
+    return result;
+}
+
+template <typename Traits>
 bool QueryCacheBase<Traits>::ShouldWaitAsyncFlushes() {
     std::scoped_lock lk(impl->flush_guard);
     return !impl->flushes_pending.empty() && impl->flushes_pending.front() != 0ULL;
diff --git a/src/video_core/query_cache/query_cache_base.h b/src/video_core/query_cache/query_cache_base.h
index d73e9039f7..5a60e42986 100644
--- a/src/video_core/query_cache/query_cache_base.h
+++ b/src/video_core/query_cache/query_cache_base.h
@@ -109,6 +109,9 @@ public:
 
     bool HasUncommittedFlushes() const;
 
+    /// Whether a new fence has queries to flush or host writes to sync for this cache.
+    bool HasPendingFenceWork() const;
+
     bool ShouldWaitAsyncFlushes();
 
     void PopAsyncFlushes();