1 files changed, 211 insertions, 5 deletions

diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp
index 0b05d31718..b607f20161 100644
--- a/src/video_core/engines/puller.cpp
+++ b/src/video_core/engines/puller.cpp
@@ -4,6 +4,9 @@
 // SPDX-FileCopyrightText: 2022 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
+#include <atomic>
+#include <cstdlib>
+
 #include "common/assert.h"
 #include "common/logging.h"
 #include "common/settings.h"
@@ -18,6 +21,7 @@
 #include "video_core/engines/puller.h"
 #include "video_core/gpu.h"
 #include "video_core/memory_manager.h"
+#include "video_core/nx2.h"
 #include "video_core/rasterizer_interface.h"
 
 namespace Tegra::Engines {
@@ -27,14 +31,102 @@ namespace {
 constexpr u32 Gf100BindClassMask = 0xffff;
 constexpr u32 Gf100BindValidMask =  0x1f0000 | Gf100BindClassMask;
 
+// Reads a semaphore word, flushing a GPU-written copy out of the buffer cache first.
+u32 ReadSemaphoreWord(DmaPusher& dma_pusher, GPUVAddr address) {
+    if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address);
+        device && dma_pusher.rasterizer->MustFlushRegion(*device, sizeof(u32),
+                                                          VideoCommon::CacheType::BufferCache)) {
+        dma_pusher.rasterizer->FlushRegion(*device, sizeof(u32),
+                                           VideoCommon::CacheType::BufferCache);
+    }
+    return dma_pusher.memory_manager.Read<u32>(address);
+}
+
 } // Anonymous namespace
 
+bool Puller::PendingAcquire::Matches(u32 current) const {
+    switch (static_cast<GpuSemaphoreOperation>(operation)) {
+    case GpuSemaphoreOperation::AcquireEqual:
+        return current == value;
+    case GpuSemaphoreOperation::AcquireGequal:
+        return current >= value;
+    case GpuSemaphoreOperation::AcquireMask:
+        return (current & value) != 0;
+    default:
+        return true;
+    }
+}
+
+bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) {
+    if (!pending_acquire) {
+        return true;
+    }
+    const u32 current = ReadSemaphoreWord(dma_pusher, pending_acquire->address);
+    if (Nx2SemaphoreWaiters::Get().Latched(this)) {
+        if (static std::atomic<u32> rescued{};
+            !pending_acquire->Matches(current) && rescued.fetch_add(1) < 64) {
+            LOG_INFO(HW_GPU, "Channel {} met semaphore {:#x} value {:#x} at its release (now {:#x})",
+                     dma_pusher.channel_state.bind_id, pending_acquire->address,
+                     pending_acquire->value, current);
+        }
+        return true;
+    }
+    if (pending_acquire->Matches(current)) {
+        return true;
+    }
+    if (!pending_acquire->reported &&
+        std::chrono::steady_clock::now() - pending_acquire->since > std::chrono::seconds{2}) {
+        pending_acquire->reported = true;
+        const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address);
+        LOG_WARNING(HW_GPU, "Channel {} waits on semaphore {:#x} (op {}, {:#x}, now {:#x}) for 2 s; "
+                    "device {:#x}, buffer cache holds GPU data: {}, words {:08x} {:08x} {:08x} {:08x}",
+                    dma_pusher.channel_state.bind_id, pending_acquire->address,
+                    pending_acquire->operation, pending_acquire->value, current, device.value_or(0),
+                    device && dma_pusher.rasterizer->MustFlushRegion(*device, 16, VideoCommon::CacheType::All),
+                    dma_pusher.memory_manager.Read<u32>(pending_acquire->address & ~GPUVAddr{0xf}),
+                    dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 4),
+                    dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 8),
+                    dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 12));
+    }
+    return false;
+}
+
+bool Puller::ResumeAcquire(DmaPusher& dma_pusher) {
+    if (!CanResumeAcquire(dma_pusher)) {
+        return false;
+    }
+    pending_acquire.reset();
+    Nx2SemaphoreWaiters::Get().Remove(this);
+    return true;
+}
+
 void Puller::ProcessBindMethod(DmaPusher& dma_pusher, const MethodCall& method_call) {
     LOG_DEBUG(HW_GPU, "Binding subchannel {} to engine {:#x}", method_call.subchannel, method_call.argument);
     u32 engine = method_call.argument;
     if ((engine & ~Gf100BindClassMask) != 0 && (engine & ~Gf100BindValidMask) == 0) {
         engine &= Gf100BindClassMask;
     }
+    // Switch 2 binds Ampere classes; run them on the Maxwell engines (same method layout for
+    // most state). Class 0 unbinds the subchannel.
+    switch (engine & 0xFFFF) {
+    case 0xC797: // AMPERE_B
+        engine = static_cast<u32>(EngineID::MAXWELL_B);
+        break;
+    case 0xC7C0: // AMPERE_COMPUTE_B
+        engine = static_cast<u32>(EngineID::KEPLER_COMPUTE_B);
+        break;
+    case 0xC7B5: // AMPERE_DMA_COPY_B
+        engine = static_cast<u32>(EngineID::MAXWELL_DMA_COPY_A);
+        break;
+    case 0x0:
+        if (g_nx2_ampere) {
+            LOG_DEBUG(HW_GPU, "Switch 2 unbinds subchannel {}", method_call.subchannel);
+            return;
+        }
+        break;
+    default:
+        break;
+    }
 
     const auto engine_id = static_cast<EngineID>(engine);
     bound_engines[method_call.subchannel] = engine_id;
@@ -82,10 +174,64 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) {
 void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
     const auto semaphoreOperationMask = 0xF;
     const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask);
+    // EDEN_NX2_SEM_WATCH=<gpu address>: log every release/acquire on the 16 bytes there.
+    static const GPUVAddr watch = [] {
+        const char* value = std::getenv("EDEN_NX2_SEM_WATCH");
+        return value ? GPUVAddr{std::strtoull(value, nullptr, 16)} : GPUVAddr{};
+    }();
+    if (watch != 0 && (regs.semaphore_address.SemaphoreAddress() & ~GPUVAddr{0xf}) == watch) {
+        LOG_INFO(HW_GPU, "Semaphore watch: channel {} as {} device {:#x} op {} {:#x} value {:#x} (now {:#x})",
+                 dma_pusher.channel_state.bind_id, dma_pusher.memory_manager.GetID(),
+                 dma_pusher.memory_manager.GpuToCpuAddress(regs.semaphore_address.SemaphoreAddress()).value_or(0),
+                 static_cast<u32>(op),
+                 regs.semaphore_address.SemaphoreAddress(), regs.semaphore_sequence,
+                 dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress()));
+    }
     if (op == GpuSemaphoreOperation::WriteLong) {
         const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()};
         const u32 payload = regs.semaphore_sequence;
-        dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::HasTimeout, payload, 0);
+        // RELEASE_SIZE (bit 24) set means a 4-byte release without the timestamp; writing 16
+        // bytes there clobbers the words next to it (Switch 2 packs semaphores tightly).
+        const auto properties = g_nx2_ampere && (regs.semaphore_trigger & (1U << 24)) != 0
+                                    ? VideoCommon::QueryPropertiesFlags::IsAFence
+                                    : VideoCommon::QueryPropertiesFlags::HasTimeout;
+        dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0);
+        if (const GPUVAddr mirror = g_nx2_ampere ? Nx2SemaphoreMirror(sequence_address) : 0;
+            mirror != 0) {
+            LOG_DEBUG(HW_GPU, "Switch 2 semaphore {:#x}={:#x} mirrored to {:#x}", sequence_address, payload, mirror);
+            dma_pusher.rasterizer->Query(mirror, VideoCommon::QueryType::Payload, properties, payload, 0);
+        }
+    } else if (g_nx2_ampere) {
+        // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has
+        // written the commands this channel runs next: suspend the channel until the value is
+        // met instead of reading ahead (approach from changes.patch).
+        PendingAcquire acquire{
+            .address = regs.semaphore_address.SemaphoreAddress(),
+            .value = regs.semaphore_sequence,
+            .operation = static_cast<u32>(op),
+            .since = std::chrono::steady_clock::now(),
+            .reported = false,
+        };
+        if (op != GpuSemaphoreOperation::AcquireEqual && op != GpuSemaphoreOperation::AcquireGequal &&
+            op != GpuSemaphoreOperation::AcquireMask) {
+            LOG_ERROR(HW_GPU, "Invalid semaphore operation {}", static_cast<u32>(op));
+            return;
+        }
+        if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) {
+            return;
+        }
+        // Registered before the fences run, so a release among them that meets the value counts
+        // even if a later one overwrites it.
+        auto& waiters = Nx2SemaphoreWaiters::Get();
+        if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address)) {
+            waiters.Add(this, *device, acquire.value, acquire.operation);
+        }
+        dma_pusher.rasterizer->ReleaseFences();
+        if (!waiters.Latched(this) && !acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) {
+            pending_acquire = acquire;
+        } else {
+            waiters.Remove(this);
+        }
     } else {
         do {
             const u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress());
@@ -137,8 +283,43 @@ void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) {
     }
 }
 
+// Ampere SEM_EXECUTE (method 0x1B): operation in bits 2:0 on the semaphore at
+// SEM_ADDR_HI:SEM_ADDR_LO (methods 0x18/0x17) with payload SEM_PAYLOAD_LO (0x19).
+void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) {
+    const GPUVAddr address = (GPUVAddr{regs.reg_array[0x18] & 0x1FFFFFF} << 32) |
+                             (regs.reg_array[0x17] & ~3U);
+    const u32 payload = regs.reg_array[0x19];
+    const u32 operation = argument & 7;
+    if (operation == 1) { // RELEASE
+        dma_pusher.rasterizer->Query(address, VideoCommon::QueryType::Payload,
+                                     VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+        return;
+    }
+    // Acquires: check once after flushing pending fences; never block the GPU thread on them.
+    dma_pusher.rasterizer->ReleaseFences();
+    const u32 word = dma_pusher.memory_manager.Read<u32>(address);
+    const bool ready = operation == 0   ? word == payload
+                       : operation == 2 ? word >= payload
+                       : operation == 3 ? static_cast<s32>(word - payload) >= 0
+                       : operation == 4 ? (word & payload) != 0
+                       : operation == 5 ? (word & payload) == 0
+                                        : true;
+    if (!ready) {
+        static std::atomic<u32> reported{};
+        if (reported.fetch_add(1) < 8) {
+            LOG_WARNING(HW_GPU, "SEM_EXECUTE op {} at {:#x}: {:#x} vs {:#x} not met, continuing",
+                        operation, address, word, payload);
+        }
+    }
+}
+
 /// Calls a GPU puller method.
 void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_call) {
+    static const bool nx2_trace = std::getenv("EDEN_NX2_PULLER_TRACE") != nullptr;
+    if (nx2_trace) {
+        LOG_INFO(HW_GPU, "nx2 puller sub={} method={:#x} arg={:#x}", method_call.subchannel,
+                 method_call.method, method_call.argument);
+    }
     regs.reg_array[method_call.method] = method_call.argument;
     const auto method = static_cast<BufferMethods>(method_call.method);
 
@@ -172,7 +353,7 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
         break;
     }
     case BufferMethods::MemOpA: {
-        LOG_ERROR(HW_GPU, "Memory Operation A");
+        LOG_TRACE(HW_GPU, "Memory Operation A");
         break;
     }
     case BufferMethods::MemOpB: {
@@ -182,15 +363,22 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
     }
     case BufferMethods::MemOpC:
     case BufferMethods::MemOpD: {
-        LOG_ERROR(HW_GPU, "Memory Operation C,D");
+        LOG_TRACE(HW_GPU, "Memory Operation C,D");
         break;
     }
     case BufferMethods::SemaphoreAcquire: {
-        ProcessSemaphoreAcquire(dma_pusher);
+        // On Ampere host classes 0x17..0x1A are SEM_ADDR_LO/HI and SEM_PAYLOAD_LO/HI.
+        if (!g_nx2_ampere) {
+            ProcessSemaphoreAcquire(dma_pusher);
+        }
         break;
     }
     case BufferMethods::SemaphoreRelease: {
-        ProcessSemaphoreRelease(dma_pusher);
+        if (g_nx2_ampere) {
+            ProcessSemaphoreExecute(dma_pusher, method_call.argument);
+        } else {
+            ProcessSemaphoreRelease(dma_pusher);
+        }
         break;
     }
     case BufferMethods::Yield: {
@@ -200,13 +388,27 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
     }
     default:
         LOG_ERROR(HW_GPU, "Special puller engine method {:X} not implemented", method);
+        if (static bool dumped = false; g_nx2_ampere && !dumped) {
+            dumped = true;
+            dma_pusher.DumpCurrentEntry("unknown host method");
+        }
         break;
     }
 }
 
 /// Calls a GPU engine method.
+static bool Nx2MethodTrace() {
+    static const bool trace = std::getenv("EDEN_NX2_METHOD_TRACE") != nullptr;
+    return trace;
+}
+
 void Puller::CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_call) {
     const EngineID engine = bound_engines[method_call.subchannel];
+    if (Nx2MethodTrace()) {
+        LOG_INFO(HW_GPU, "nx2 method sub={} engine={:04X} method={:#x} arg={:#x}",
+                 method_call.subchannel, static_cast<u32>(engine), method_call.method,
+                 method_call.argument);
+    }
     switch (engine) {
     case EngineID::FERMI_TWOD_A:
         dma_pusher.channel_state.payload->fermi_2d.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall());
@@ -235,6 +437,10 @@ void Puller::CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
 /// Calls a GPU engine multivalue method.
 void Puller::CallEngineMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending) {
     const EngineID engine = bound_engines[subchannel];
+    if (Nx2MethodTrace()) {
+        LOG_INFO(HW_GPU, "nx2 multimethod sub={} engine={:04X} method={:#x} amount={} first={:#x}",
+                 subchannel, static_cast<u32>(engine), method, amount, base_start[0]);
+    }
     switch (engine) {
     case EngineID::FERMI_TWOD_A:
         dma_pusher.channel_state.payload->fermi_2d.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending);