// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later #include #include #include "common/assert.h" #include "common/logging.h" #include "common/settings.h" #include "core/core.h" #include "video_core/nx2_profile.h" #include "video_core/control/channel_state.h" #include "video_core/dma_pusher.h" #include "video_core/engines/fermi_2d.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/kepler_memory.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/maxwell_dma.h" #include "video_core/engines/puller.h" #include "video_core/gpu.h" #include "video_core/gpu_split.h" #include "video_core/memory_manager.h" #include "video_core/nx2.h" #include "video_core/rasterizer_interface.h" namespace Tegra::Engines { namespace { constexpr u32 Gf100BindClassMask = 0xffff; constexpr u32 Gf100BindValidMask = 0x1f0000 | Gf100BindClassMask; // Reads a semaphore word, flushing a GPU-written copy out of the buffer cache first. u32 ReadSemaphoreWord(DmaPusher& dma_pusher, GPUVAddr address) { if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); device && dma_pusher.rasterizer->MustFlushRegion(*device, sizeof(u32), VideoCommon::CacheType::BufferCache)) { dma_pusher.rasterizer->FlushRegion(*device, sizeof(u32), VideoCommon::CacheType::BufferCache); } return dma_pusher.memory_manager.Read(address); } // With the GPU thread split (gpu_split.h) the rasterizer belongs to the back thread: work that // only feeds it runs there in order; reads of what it wrote first wait for it (DrainBack). template void OnBack(const Func& func) { if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) { split.Defer(func); } else { func(); } } void DrainBack() { VideoCommon::GpuSplit::Get().Drain(); } } // Anonymous namespace bool Puller::PendingAcquire::Matches(u32 current) const { switch (static_cast(operation)) { case GpuSemaphoreOperation::AcquireEqual: return current == value; case GpuSemaphoreOperation::AcquireGequal: return current >= value; case GpuSemaphoreOperation::AcquireMask: return (current & value) != 0; default: return true; } } bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { if (!pending_acquire) { return true; } // The release may still be queued for the split GPU thread's back half (and the caches // ReadSemaphoreWord asks are its state). DrainBack(); const u32 current = ReadSemaphoreWord(dma_pusher, pending_acquire->address); if (Nx2SemaphoreWaiters::Get().Latched(this)) { if (static std::atomic rescued{}; !pending_acquire->Matches(current) && rescued.fetch_add(1) < 64) { LOG_INFO(HW_GPU, "Channel {} met semaphore {:#x} value {:#x} at its release (now {:#x})", dma_pusher.channel_state.bind_id, pending_acquire->address, pending_acquire->value, current); } return true; } if (pending_acquire->Matches(current)) { return true; } if (!pending_acquire->reported && std::chrono::steady_clock::now() - pending_acquire->since > std::chrono::seconds{2}) { pending_acquire->reported = true; const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address); LOG_WARNING(HW_GPU, "Channel {} waits on semaphore {:#x} (op {}, {:#x}, now {:#x}) for 2 s; " "device {:#x}, buffer cache holds GPU data: {}, words {:08x} {:08x} {:08x} {:08x}", dma_pusher.channel_state.bind_id, pending_acquire->address, pending_acquire->operation, pending_acquire->value, current, device.value_or(0), device && dma_pusher.rasterizer->MustFlushRegion(*device, 16, VideoCommon::CacheType::All), dma_pusher.memory_manager.Read(pending_acquire->address & ~GPUVAddr{0xf}), dma_pusher.memory_manager.Read((pending_acquire->address & ~GPUVAddr{0xf}) + 4), dma_pusher.memory_manager.Read((pending_acquire->address & ~GPUVAddr{0xf}) + 8), dma_pusher.memory_manager.Read((pending_acquire->address & ~GPUVAddr{0xf}) + 12)); } return false; } bool Puller::ResumeAcquire(DmaPusher& dma_pusher) { if (!CanResumeAcquire(dma_pusher)) { return false; } pending_acquire.reset(); dma_pusher.suspensions.fetch_add(1, std::memory_order_release); Nx2SemaphoreWaiters::Get().Remove(this); return true; } void Puller::ProcessBindMethod(DmaPusher& dma_pusher, const MethodCall& method_call) { LOG_DEBUG(HW_GPU, "Binding subchannel {} to engine {:#x}", method_call.subchannel, method_call.argument); u32 engine = method_call.argument; if ((engine & ~Gf100BindClassMask) != 0 && (engine & ~Gf100BindValidMask) == 0) { engine &= Gf100BindClassMask; } // Switch 2 binds Ampere classes; run them on the Maxwell engines (same method layout for // most state). Class 0 unbinds the subchannel. switch (engine & 0xFFFF) { case 0xC797: // AMPERE_B engine = static_cast(EngineID::MAXWELL_B); break; case 0xC7C0: // AMPERE_COMPUTE_B engine = static_cast(EngineID::KEPLER_COMPUTE_B); break; case 0xC7B5: // AMPERE_DMA_COPY_B engine = static_cast(EngineID::MAXWELL_DMA_COPY_A); break; case 0x0: if (g_nx2_ampere) { LOG_DEBUG(HW_GPU, "Switch 2 unbinds subchannel {}", method_call.subchannel); return; } break; default: break; } const auto engine_id = static_cast(engine); bound_engines[method_call.subchannel] = engine_id; switch (engine_id) { case EngineID::FERMI_TWOD_A: dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->fermi_2d, method_call.subchannel, EngineTypes::Fermi2D); break; case EngineID::MAXWELL_B: dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->maxwell_3d, method_call.subchannel, EngineTypes::Maxwell3D); break; case EngineID::KEPLER_COMPUTE_B: dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->kepler_compute, method_call.subchannel, EngineTypes::KeplerCompute); break; case EngineID::MAXWELL_DMA_COPY_A: dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->maxwell_dma, method_call.subchannel, EngineTypes::MaxwellDMA); break; case EngineID::KEPLER_INLINE_TO_MEMORY_B: dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->kepler_memory, method_call.subchannel, EngineTypes::KeplerMemory); break; case EngineID::NV01_TIMER: dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->nv01_timer, method_call.subchannel, EngineTypes::Nv01Timer); break; default: UNIMPLEMENTED_MSG("Unimplemented engine {:04X}", engine_id); break; } } void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { switch (regs.fence_action.op) { case Puller::FenceOperation::Acquire: // UNIMPLEMENTED_MSG("Channel Scheduling pending."); // WaitFence(regs.fence_action.syncpoint_id, regs.fence_value); OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); }); break; case Puller::FenceOperation::Increment: OnBack([rasterizer = dma_pusher.rasterizer, id = static_cast(regs.fence_action.syncpoint_id.Value())] { rasterizer->SignalSyncPoint(id); }); break; default: UNIMPLEMENTED_MSG("Unimplemented operation {}", regs.fence_action.op.Value()); break; } } void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { const auto semaphoreOperationMask = 0xF; const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask); // EDEN_NX2_SEM_WATCH=: log every release/acquire on the 16 bytes there. static const GPUVAddr watch = [] { const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); return value ? GPUVAddr{std::strtoull(value, nullptr, 16)} : GPUVAddr{}; }(); if (watch != 0 && (regs.semaphore_address.SemaphoreAddress() & ~GPUVAddr{0xf}) == watch) { LOG_INFO(HW_GPU, "Semaphore watch: channel {} as {} device {:#x} op {} {:#x} value {:#x} (now {:#x})", dma_pusher.channel_state.bind_id, dma_pusher.memory_manager.GetID(), dma_pusher.memory_manager.GpuToCpuAddress(regs.semaphore_address.SemaphoreAddress()).value_or(0), static_cast(op), regs.semaphore_address.SemaphoreAddress(), regs.semaphore_sequence, dma_pusher.memory_manager.Read(regs.semaphore_address.SemaphoreAddress())); } if (op == GpuSemaphoreOperation::WriteLong) { const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()}; const u32 payload = regs.semaphore_sequence; // RELEASE_SIZE (bit 24) set means a 4-byte release without the timestamp; writing 16 // bytes there clobbers the words next to it (Switch 2 packs semaphores tightly). const auto properties = g_nx2_ampere && (regs.semaphore_trigger & (1U << 24)) != 0 ? VideoCommon::QueryPropertiesFlags::IsAFence : VideoCommon::QueryPropertiesFlags::HasTimeout; const GPUVAddr mirror = g_nx2_ampere ? Nx2SemaphoreMirror(sequence_address) : 0; if (mirror != 0) { LOG_DEBUG(HW_GPU, "Switch 2 semaphore {:#x}={:#x} mirrored to {:#x}", sequence_address, payload, mirror); } OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, mirror, properties, payload] { rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0); if (mirror != 0) { rasterizer->Query(mirror, VideoCommon::QueryType::Payload, properties, payload, 0); } }); } else if (g_nx2_ampere) { // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has // written the commands this channel runs next: suspend the channel until the value is // met instead of reading ahead (approach from changes.patch). PendingAcquire acquire{ .address = regs.semaphore_address.SemaphoreAddress(), .value = regs.semaphore_sequence, .operation = static_cast(op), .since = std::chrono::steady_clock::now(), .reported = false, }; if (op != GpuSemaphoreOperation::AcquireEqual && op != GpuSemaphoreOperation::AcquireGequal && op != GpuSemaphoreOperation::AcquireMask) { LOG_ERROR(HW_GPU, "Invalid semaphore operation {}", static_cast(op)); return; } VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1); DrainBack(); if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { return; } // Registered before the fences run, so a release among them that meets the value counts // even if a later one overwrites it. auto& waiters = Nx2SemaphoreWaiters::Get(); if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address)) { waiters.Add(this, *device, acquire.value, acquire.operation); } dma_pusher.rasterizer->ReleaseFences(); if (!waiters.Latched(this) && !acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { pending_acquire = acquire; dma_pusher.suspensions.fetch_add(1, std::memory_order_release); VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireWaits, 1); } else { waiters.Remove(this); } } else { DrainBack(); do { const u32 word = dma_pusher.memory_manager.Read(regs.semaphore_address.SemaphoreAddress()); regs.acquire_source = true; regs.acquire_value = regs.semaphore_sequence; if (op == GpuSemaphoreOperation::AcquireEqual) { regs.acquire_active = true; regs.acquire_mode = false; if (word != regs.acquire_value) { dma_pusher.rasterizer->ReleaseFences(); continue; } } else if (op == GpuSemaphoreOperation::AcquireGequal) { regs.acquire_active = true; regs.acquire_mode = true; if (word < regs.acquire_value) { dma_pusher.rasterizer->ReleaseFences(); continue; } } else if (op == GpuSemaphoreOperation::AcquireMask) { if (word && regs.semaphore_sequence == 0) { dma_pusher.rasterizer->ReleaseFences(); continue; } } else { LOG_ERROR(HW_GPU, "Invalid semaphore operation"); } } while (false); } } void Puller::ProcessSemaphoreRelease(DmaPusher& dma_pusher) { const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()}; const u32 payload = regs.semaphore_release; OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, payload] { rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }); } void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) { DrainBack(); u32 word = dma_pusher.memory_manager.Read(regs.semaphore_address.SemaphoreAddress()); const auto value = regs.semaphore_acquire; while (word != value) { regs.acquire_active = true; regs.acquire_value = value; dma_pusher.rasterizer->ReleaseFences(); word = dma_pusher.memory_manager.Read(regs.semaphore_address.SemaphoreAddress()); // TODO(kemathe73) figure out how to do the acquire_timeout regs.acquire_mode = false; regs.acquire_source = false; } } // Ampere SEM_EXECUTE (method 0x1B): operation in bits 2:0 on the semaphore at // SEM_ADDR_HI:SEM_ADDR_LO (methods 0x18/0x17) with payload SEM_PAYLOAD_LO (0x19). void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { const GPUVAddr address = (GPUVAddr{regs.reg_array[0x18] & 0x1FFFFFF} << 32) | (regs.reg_array[0x17] & ~3U); const u32 payload = regs.reg_array[0x19]; const u32 operation = argument & 7; if (operation == 1) { // RELEASE OnBack([rasterizer = dma_pusher.rasterizer, address, payload] { rasterizer->Query(address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); }); return; } // Acquires: check once after flushing pending fences; never block the GPU thread on them. DrainBack(); dma_pusher.rasterizer->ReleaseFences(); const u32 word = dma_pusher.memory_manager.Read(address); const bool ready = operation == 0 ? word == payload : operation == 2 ? word >= payload : operation == 3 ? static_cast(word - payload) >= 0 : operation == 4 ? (word & payload) != 0 : operation == 5 ? (word & payload) == 0 : true; if (!ready) { static std::atomic reported{}; if (reported.fetch_add(1) < 8) { LOG_WARNING(HW_GPU, "SEM_EXECUTE op {} at {:#x}: {:#x} vs {:#x} not met, continuing", operation, address, word, payload); } } } /// Calls a GPU puller method. void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_call) { static const bool nx2_trace = std::getenv("EDEN_NX2_PULLER_TRACE") != nullptr; if (nx2_trace) { LOG_INFO(HW_GPU, "nx2 puller sub={} method={:#x} arg={:#x}", method_call.subchannel, method_call.method, method_call.argument); } regs.reg_array[method_call.method] = method_call.argument; const auto method = static_cast(method_call.method); switch (method) { case BufferMethods::BindObject: { ProcessBindMethod(dma_pusher, method_call); break; } case BufferMethods::Nop: case BufferMethods::SemaphoreAddressHigh: case BufferMethods::SemaphoreAddressLow: case BufferMethods::SemaphoreSequencePayload: case BufferMethods::SyncpointPayload: case BufferMethods::WrcacheFlush: break; case BufferMethods::RefCnt: OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->SignalReference(); }); break; case BufferMethods::SyncpointOperation: ProcessFenceActionMethod(dma_pusher); break; case BufferMethods::WaitForIdle: OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->WaitForIdle(); }); break; case BufferMethods::SemaphoreOperation: { ProcessSemaphoreTriggerMethod(dma_pusher); break; } case BufferMethods::NonStallInterrupt: { LOG_ERROR(HW_GPU, "Special puller engine method NonStallInterrupt not implemented"); break; } case BufferMethods::MemOpA: { LOG_TRACE(HW_GPU, "Memory Operation A"); break; } case BufferMethods::MemOpB: { // Implement this better. OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->InvalidateGPUCache(); }); break; } case BufferMethods::MemOpC: case BufferMethods::MemOpD: { LOG_TRACE(HW_GPU, "Memory Operation C,D"); break; } case BufferMethods::SemaphoreAcquire: { // On Ampere host classes 0x17..0x1A are SEM_ADDR_LO/HI and SEM_PAYLOAD_LO/HI. if (!g_nx2_ampere) { ProcessSemaphoreAcquire(dma_pusher); } break; } case BufferMethods::SemaphoreRelease: { if (g_nx2_ampere) { ProcessSemaphoreExecute(dma_pusher, method_call.argument); } else { ProcessSemaphoreRelease(dma_pusher); } break; } case BufferMethods::Yield: { // TODO(Kmather73): Research and implement this method. LOG_ERROR(HW_GPU, "Special puller engine method Yield not implemented"); break; } default: LOG_ERROR(HW_GPU, "Special puller engine method {:X} not implemented", method); if (static bool dumped = false; g_nx2_ampere && !dumped) { dumped = true; dma_pusher.DumpCurrentEntry("unknown host method"); } break; } } /// Calls a GPU engine method. static bool Nx2MethodTrace() { static const bool trace = std::getenv("EDEN_NX2_METHOD_TRACE") != nullptr; return trace; } void Puller::CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_call) { const EngineID engine = bound_engines[method_call.subchannel]; if (Nx2MethodTrace()) { LOG_INFO(HW_GPU, "nx2 method sub={} engine={:04X} method={:#x} arg={:#x}", method_call.subchannel, static_cast(engine), method_call.method, method_call.argument); } switch (engine) { case EngineID::FERMI_TWOD_A: dma_pusher.channel_state.payload->fermi_2d.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); break; case EngineID::MAXWELL_B: dma_pusher.channel_state.payload->maxwell_3d.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); break; case EngineID::KEPLER_COMPUTE_B: dma_pusher.channel_state.payload->kepler_compute.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); break; case EngineID::MAXWELL_DMA_COPY_A: dma_pusher.channel_state.payload->maxwell_dma.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); break; case EngineID::KEPLER_INLINE_TO_MEMORY_B: dma_pusher.channel_state.payload->kepler_memory.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); break; case EngineID::NV01_TIMER: dma_pusher.channel_state.payload->nv01_timer.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); break; default: UNIMPLEMENTED_MSG("Unimplemented engine"); break; } } /// Calls a GPU engine multivalue method. void Puller::CallEngineMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending) { const EngineID engine = bound_engines[subchannel]; if (Nx2MethodTrace()) { LOG_INFO(HW_GPU, "nx2 multimethod sub={} engine={:04X} method={:#x} amount={} first={:#x}", subchannel, static_cast(engine), method, amount, base_start[0]); } switch (engine) { case EngineID::FERMI_TWOD_A: dma_pusher.channel_state.payload->fermi_2d.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); break; case EngineID::MAXWELL_B: dma_pusher.channel_state.payload->maxwell_3d.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); break; case EngineID::KEPLER_COMPUTE_B: dma_pusher.channel_state.payload->kepler_compute.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); break; case EngineID::MAXWELL_DMA_COPY_A: dma_pusher.channel_state.payload->maxwell_dma.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); break; case EngineID::KEPLER_INLINE_TO_MEMORY_B: dma_pusher.channel_state.payload->kepler_memory.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); break; case EngineID::NV01_TIMER: dma_pusher.channel_state.payload->nv01_timer.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); break; default: UNIMPLEMENTED_MSG("Unimplemented engine"); break; } } /// Calls a GPU method. void Puller::CallMethod(DmaPusher& dma_pusher, const MethodCall& method_call) { LOG_TRACE(HW_GPU, "Processing method {:08X} on subchannel {}", method_call.method, method_call.subchannel); ASSERT(method_call.subchannel < bound_engines.size()); if (ExecuteMethodOnEngine(dma_pusher, method_call.method)) { CallEngineMethod(dma_pusher, method_call); } else { CallPullerMethod(dma_pusher, method_call); } } /// Calls a GPU multivalue method. void Puller::CallMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending) { LOG_TRACE(HW_GPU, "Processing method {:08X} on subchannel {}", method, subchannel); ASSERT(subchannel < bound_engines.size()); if (ExecuteMethodOnEngine(dma_pusher, method)) { CallEngineMultiMethod(dma_pusher, method, subchannel, base_start, amount, methods_pending); } else { for (u32 i = 0; i < amount; i++) { CallPullerMethod(dma_pusher, MethodCall{ method, base_start[i], subchannel, methods_pending - i, }); } } } /// Determines where the method should be executed. [[nodiscard]] bool Puller::ExecuteMethodOnEngine(DmaPusher& dma_pusher, u32 method) { const auto buffer_method = BufferMethods(method); return buffer_method >= BufferMethods::NonPullerMethods; } } // namespace Tegra::Engines