nx2: semaphore releases latch the acquires they satisfy

Switch 2 channels that wait for an exact semaphore value poll on the console
and see every release. Eden resumes a waiting channel only between command
lists and writes releases when fences signal, often several back to back, so
an acquire-equal could miss its value and wait forever. Title 4's 3D queue
ticks a counter that two other queues wait on value by value; under the
loading after the post-customization save one of them missed a tick and the
game froze (channel 4 waiting for 0x5293 with the counter at 0x5310).

Pending acquires now register with Nx2SemaphoreWaiters before fences run,
and every query-cache payload write and QMD add/max release latches the
waiters it meets at the moment it is written.

4 files changed, 113 insertions, 2 deletions

diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp
index 85b1f971f7..dc3f9b25b6 100644
--- a/src/video_core/engines/kepler_compute.cpp
+++ b/src/video_core/engines/kepler_compute.cpp
@@ -14,6 +14,7 @@
 #include "common/logging.h"
 #include "shader_recompiler/exception.h"
 #include "core/core.h"
+#include "video_core/nx2.h"
 #include "video_core/engines/fermi_2d.h"
 #include "video_core/engines/kepler_compute.h"
 #include "video_core/engines/maxwell_3d.h"
@@ -240,6 +241,9 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar
                     rasterizer->SignalFence([this, address, payload] {
                         const u32 value = memory_manager.Read<u32>(address) + payload;
                         memory_manager.Write<u32>(address, value);
+                        if (const auto device = memory_manager.GpuToCpuAddress(address)) {
+                            Tegra::Nx2SemaphoreWaiters::Get().OnWrite(*device, value);
+                        }
                         static const u64 watch = [] {
                             const char* text = std::getenv("EDEN_NX2_SEM_WATCH");
                             return text ? u64{std::strtoull(text, nullptr, 16)} : u64{};
@@ -257,6 +261,9 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar
                     rasterizer->SignalFence([this, address, payload] {
                         const u32 value = std::max(memory_manager.Read<u32>(address), payload);
                         memory_manager.Write<u32>(address, value);
+                        if (const auto device = memory_manager.GpuToCpuAddress(address)) {
+                            Tegra::Nx2SemaphoreWaiters::Get().OnWrite(*device, value);
+                        }
                     });
                 } else {
                     static const u64 watch = [] {
diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp
index 54abf790d2..b607f20161 100644
--- a/src/video_core/engines/puller.cpp
+++ b/src/video_core/engines/puller.cpp
@@ -62,6 +62,15 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) {
         return true;
     }
     const u32 current = ReadSemaphoreWord(dma_pusher, pending_acquire->address);
+    if (Nx2SemaphoreWaiters::Get().Latched(this)) {
+        if (static std::atomic<u32> rescued{};
+            !pending_acquire->Matches(current) && rescued.fetch_add(1) < 64) {
+            LOG_INFO(HW_GPU, "Channel {} met semaphore {:#x} value {:#x} at its release (now {:#x})",
+                     dma_pusher.channel_state.bind_id, pending_acquire->address,
+                     pending_acquire->value, current);
+        }
+        return true;
+    }
     if (pending_acquire->Matches(current)) {
         return true;
     }
@@ -87,6 +96,7 @@ bool Puller::ResumeAcquire(DmaPusher& dma_pusher) {
         return false;
     }
     pending_acquire.reset();
+    Nx2SemaphoreWaiters::Get().Remove(this);
     return true;
 }
 
@@ -210,9 +220,17 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
         if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) {
             return;
         }
+        // Registered before the fences run, so a release among them that meets the value counts
+        // even if a later one overwrites it.
+        auto& waiters = Nx2SemaphoreWaiters::Get();
+        if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address)) {
+            waiters.Add(this, *device, acquire.value, acquire.operation);
+        }
         dma_pusher.rasterizer->ReleaseFences();
-        if (!acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) {
+        if (!waiters.Latched(this) && !acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) {
             pending_acquire = acquire;
+        } else {
+            waiters.Remove(this);
         }
     } else {
         do {
diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h
index 7bbbf14785..5b6d89abdd 100644
--- a/src/video_core/nx2.h
+++ b/src/video_core/nx2.h
@@ -3,8 +3,11 @@
 
 #pragma once
 
+#include <algorithm>
 #include <array>
 #include <atomic>
+#include <mutex>
+#include <vector>
 
 #include "common/common_types.h"
 
@@ -57,4 +60,84 @@ inline u64 Nx2SemaphoreMirror(u64 address) {
     return 0;
 }
 
+/// Channels suspended on a semaphore acquire. A console channel polls and sees every release;
+/// Eden writes releases in bursts when fences signal and resumes waiters only between command
+/// lists, so an acquire-equal could miss its value. Writes latch the waiters they satisfy.
+class Nx2SemaphoreWaiters {
+public:
+    // Puller GpuSemaphoreOperation values.
+    static constexpr u32 AcquireEqual = 0x1;
+    static constexpr u32 AcquireGequal = 0x4;
+    static constexpr u32 AcquireMask = 0x8;
+
+    static Nx2SemaphoreWaiters& Get() {
+        static Nx2SemaphoreWaiters waiters;
+        return waiters;
+    }
+
+    void Add(const void* owner, u64 device_address, u32 value, u32 operation) {
+        std::scoped_lock lk{mutex};
+        std::erase_if(waiters, [owner](const Waiter& w) { return w.owner == owner; });
+        waiters.push_back({owner, device_address, value, operation, false});
+        count.store(waiters.size(), std::memory_order_release);
+    }
+
+    void Remove(const void* owner) {
+        if (count.load(std::memory_order_acquire) == 0) {
+            return;
+        }
+        std::scoped_lock lk{mutex};
+        std::erase_if(waiters, [owner](const Waiter& w) { return w.owner == owner; });
+        count.store(waiters.size(), std::memory_order_release);
+    }
+
+    [[nodiscard]] bool Latched(const void* owner) {
+        if (count.load(std::memory_order_acquire) == 0) {
+            return false;
+        }
+        std::scoped_lock lk{mutex};
+        return std::ranges::any_of(waiters,
+                                   [owner](const Waiter& w) { return w.owner == owner && w.latched; });
+    }
+
+    /// A GPU write of `value` to the u32 at `device_address`.
+    void OnWrite(u64 device_address, u32 value) {
+        if (count.load(std::memory_order_acquire) == 0) {
+            return;
+        }
+        std::scoped_lock lk{mutex};
+        for (auto& w : waiters) {
+            if (w.address == device_address && !w.latched && Matches(w, value)) {
+                w.latched = true;
+            }
+        }
+    }
+
+private:
+    struct Waiter {
+        const void* owner;
+        u64 address;
+        u32 value;
+        u32 operation;
+        bool latched;
+    };
+
+    static bool Matches(const Waiter& w, u32 current) {
+        switch (w.operation) {
+        case AcquireEqual:
+            return current == w.value;
+        case AcquireGequal:
+            return current >= w.value;
+        case AcquireMask:
+            return (current & w.value) != 0;
+        default:
+            return false;
+        }
+    }
+
+    std::mutex mutex;
+    std::vector<Waiter> waiters;
+    std::atomic<size_t> count{};
+};
+
 } // namespace Tegra
diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h
index f94407b026..245ab1996c 100644
--- a/src/video_core/query_cache/query_cache.h
+++ b/src/video_core/query_cache/query_cache.h
@@ -23,6 +23,7 @@
 #include "video_core/gpu.h"
 #include "video_core/host1x/gpu_device_memory_manager.h"
 #include "video_core/memory_manager.h"
+#include "video_core/nx2.h"
 #include "video_core/query_cache/bank_base.h"
 #include "video_core/query_cache/query_base.h"
 #include "video_core/query_cache/query_cache_base.h"
@@ -273,7 +274,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type
     u8* pointer_timestamp = impl->device_memory.template GetPointer<u8>(cpu_addr + 8);
     bool is_synced = (Settings::IsGPUFenceBehaviorDefault() ? !Settings::IsGPULevelHigh() : !Settings::IsGPUFenceBehaviorBalanced() && !Settings::IsGPUFenceBehaviorAccurate()) && is_fence;
     std::function<void()> operation([this, is_synced, streamer, query_base = query, query_location,
-                                     pointer, pointer_timestamp] {
+                                     pointer, pointer_timestamp, cpu_addr] {
         if (True(query_base->flags & QueryFlagBits::IsInvalidated)) {
             if (!is_synced) [[likely]] {
                 impl->pending_unregister.push_back(query_location);
@@ -298,6 +299,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type
             u32 value = static_cast<u32>(query_base->value);
             std::memcpy(pointer, &value, sizeof(value));
         }
+        Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, static_cast<u32>(query_base->value));
         if (!is_synced) [[likely]] {
             impl->pending_unregister.push_back(query_location);
         }
@@ -314,6 +316,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type
             } else {
                 std::memcpy(pointer, &payload, sizeof(payload));
             }
+            Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, payload);
             streamer->Free(new_query_id);
             return;
         }