aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/video_core/CMakeLists.txt2
-rw-r--r--src/video_core/control/scheduler.cpp22
-rw-r--r--src/video_core/control/scheduler.h2
-rw-r--r--src/video_core/dma_pusher.cpp83
-rw-r--r--src/video_core/dma_pusher.h1
-rw-r--r--src/video_core/engines/kepler_compute.cpp164
-rw-r--r--src/video_core/engines/kepler_compute.h25
-rw-r--r--src/video_core/engines/maxwell_3d.cpp425
-rw-r--r--src/video_core/engines/maxwell_3d.h34
-rw-r--r--src/video_core/engines/puller.cpp58
-rw-r--r--src/video_core/gpu_split.cpp363
-rw-r--r--src/video_core/gpu_split.h142
-rw-r--r--src/video_core/gpu_thread.cpp55
-rw-r--r--src/video_core/nx2_profile.h11
-rw-r--r--src/video_core/renderer_vulkan/vk_scheduler.cpp9
-rw-r--r--src/video_core/renderer_vulkan/vk_scheduler.h4
16 files changed, 1368 insertions, 32 deletions
diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt
index e97b425412..e619c4f2de 100644
--- a/src/video_core/CMakeLists.txt
+++ b/src/video_core/CMakeLists.txt
@@ -95,6 +95,8 @@ add_library(video_core STATIC
fence_manager.h
gpu.cpp
gpu.h
+ gpu_split.cpp
+ gpu_split.h
gpu_thread.cpp
gpu_thread.h
guest_memory.h
diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp
index 24f1db1f7c..c120533c37 100644
--- a/src/video_core/control/scheduler.cpp
+++ b/src/video_core/control/scheduler.cpp
@@ -12,9 +12,24 @@
#include "video_core/control/channel_state.h"
#include "video_core/control/scheduler.h"
#include "video_core/gpu.h"
+#include "video_core/gpu_split.h"
namespace Tegra::Control {
+void Scheduler::BindChannel(GPU& gpu, s32 bind_id) {
+ // With the GPU thread split the rasterizer's channel switches with the methods around it, on
+ // the back thread (deferred outside the lock: the back takes it to bind).
+ if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) {
+ split.Defer([this, gpu_ptr = &gpu, bind_id] {
+ std::scoped_lock lk{scheduling_guard};
+ gpu_ptr->BindChannel(bind_id);
+ });
+ return;
+ }
+ std::scoped_lock lk{scheduling_guard};
+ gpu.BindChannel(bind_id);
+}
+
void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) {
std::shared_ptr<ChannelState> channel_state;
{
@@ -22,8 +37,8 @@ void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) {
auto it = channels.find(channel);
ASSERT(it != channels.end());
channel_state = it->second;
- gpu.BindChannel(channel_state->bind_id);
}
+ BindChannel(gpu, channel_state->bind_id);
// Process commands outside the lock to reduce contention.
// Multiple channels can prepare their commands in parallel.
channel_state->payload->dma_pusher.Push(std::move(entries));
@@ -47,10 +62,7 @@ void Scheduler::ResumePending(GPU& gpu) {
if (!dma_pusher.HasPendingAcquire() || !dma_pusher.CanResumeAcquire()) {
continue;
}
- {
- std::scoped_lock lk{scheduling_guard};
- gpu.BindChannel(state->bind_id);
- }
+ BindChannel(gpu, state->bind_id);
dma_pusher.DispatchCalls();
progress = true;
}
diff --git a/src/video_core/control/scheduler.h b/src/video_core/control/scheduler.h
index 3a3dcc58cf..9a0f1b919f 100644
--- a/src/video_core/control/scheduler.h
+++ b/src/video_core/control/scheduler.h
@@ -34,6 +34,8 @@ public:
void DeclareChannel(std::shared_ptr<ChannelState> new_channel);
private:
+ void BindChannel(GPU& gpu, s32 bind_id);
+
::Common::unordered_map<s32, std::shared_ptr<ChannelState>> channels;
std::mutex scheduling_guard;
std::vector<std::shared_ptr<ChannelState>> pending_channels; ///< Suspended on an acquire
diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp
index 44a63d789a..62f7e6bfdb 100644
--- a/src/video_core/dma_pusher.cpp
+++ b/src/video_core/dma_pusher.cpp
@@ -15,8 +15,10 @@
#include "video_core/nx2_profile.h"
#include "video_core/dma_pusher.h"
#include "video_core/control/channel_state.h"
+#include "video_core/engines/kepler_compute.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/gpu.h"
+#include "video_core/gpu_split.h"
#include "video_core/guest_memory.h"
#include "video_core/memory_manager.h"
#include "video_core/nx2.h"
@@ -45,6 +47,14 @@ void DmaPusher::DispatchCalls() {
break;
}
}
+ if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) {
+ split.Defer([gpu = &system.GPU()] {
+ gpu->FlushCommands();
+ gpu->OnCommandListEnd();
+ });
+ split.Publish();
+ return;
+ }
system.GPU().FlushCommands();
system.GPU().OnCommandListEnd();
}
@@ -98,7 +108,16 @@ bool DmaPusher::Step() {
const bool macro_payload = engine == Engines::EngineTypes::Maxwell3D && dma_state.method >= MacroRegistersStart;
if (kepler_payload || macro_payload) {
const size_t words = std::min<size_t>(dma_state.method_count, header.size);
- subchannels[dma_state.subchannel]->current_dirty = memory_manager.IsMemoryDirty(dma_state.dma_get, words * sizeof(u32));
+ auto* const engine_state = subchannels[dma_state.subchannel];
+ if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) {
+ // The caches behind IsMemoryDirty belong to the split's back thread.
+ split.Defer([engine_state, memory = &memory_manager, address = GPUVAddr{dma_state.dma_get},
+ bytes = words * sizeof(u32)] {
+ engine_state->current_dirty = memory->IsMemoryDirty(address, bytes);
+ });
+ } else {
+ engine_state->current_dirty = memory_manager.IsMemoryDirty(dma_state.dma_get, words * sizeof(u32));
+ }
}
}
if (command_resume_offset == 0) {
@@ -116,9 +135,9 @@ bool DmaPusher::Step() {
u64{header.allow_flush}, split_method_end, parked_methods.size());
std::array<u32, 8> next_words{};
std::array<u32, 4> prev_words{};
- memory_manager.ReadBlock(header.addr, next_words.data(),
+ memory_manager.ReadBlockUnsafe(header.addr, next_words.data(),
std::min<size_t>(header.size, 8) * sizeof(u32));
- memory_manager.ReadBlock(split_method_end - sizeof(prev_words), prev_words.data(),
+ memory_manager.ReadBlockUnsafe(split_method_end - sizeof(prev_words), prev_words.data(),
sizeof(prev_words));
LOG_DEBUG(HW_GPU, " previous entry ends {:08x}; next entry starts {:08x}",
fmt::join(prev_words, " "), fmt::join(next_words, " "));
@@ -176,6 +195,7 @@ bool DmaPusher::Step() {
}
if (signal_sync) {
+ VideoCommon::GpuSplit::Get().Drain();
rasterizer->SignalFence([this]() {
std::scoped_lock lk(sync_mutex);
synced = true;
@@ -316,10 +336,18 @@ void DmaPusher::WatchArgument(u32 method, u32 argument) const {
}
}
+bool DmaPusher::WantsSegment() const {
+ // Only macro parameters and compute uploads read the engine's current_dma_segment.
+ const auto engine = subchannel_type[dma_state.subchannel];
+ return engine == Engines::EngineTypes::KeplerCompute ||
+ (engine == Engines::EngineTypes::Maxwell3D && dma_state.method >= MacroRegistersStart);
+}
+
void DmaPusher::CallMethod(u32 argument) {
if (g_semaphore_watch != 0) [[unlikely]] {
WatchArgument(dma_state.method, argument);
}
+ auto& split = VideoCommon::GpuSplit::Get();
if (dma_state.method < non_puller_methods) {
puller.CallPullerMethod(*this, Engines::Puller::MethodCall{
dma_state.method,
@@ -333,6 +361,30 @@ void DmaPusher::CallMethod(u32 argument) {
ReportUnboundSubchannel();
return;
}
+ if (split.Active()) {
+ const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset;
+ if (split.FrontMacros()) {
+ const auto type = subchannel_type[dma_state.subchannel];
+ if (type == Engines::EngineTypes::Maxwell3D) {
+ auto* const maxwell3d = static_cast<Engines::Maxwell3D*>(subchannel);
+ if (maxwell3d->GetNativeMacroEngine().native_turing) {
+ maxwell3d->FrontCallMethod(system, split, dma_state.method, argument,
+ dma_state.is_last_call);
+ return;
+ }
+ } else if (type == Engines::EngineTypes::KeplerCompute) {
+ auto* const compute = static_cast<Engines::KeplerCompute*>(subchannel);
+ if (compute->FrontRunsMacros()) {
+ compute->FrontCallMethod(system, split, dma_state.method, argument,
+ dma_state.is_last_call, &segment);
+ return;
+ }
+ }
+ }
+ split.Method(subchannel, dma_state.method, argument, dma_state.is_last_call,
+ WantsSegment() ? &segment : nullptr);
+ return;
+ }
if (!subchannel->execution_mask[dma_state.method]) {
subchannel->method_sink.emplace_back(dma_state.method, argument);
} else {
@@ -350,6 +402,7 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) {
WatchArgument(dma_state.method + (dma_state.non_incrementing ? 0 : i), base_start[i]);
}
}
+ auto& split = VideoCommon::GpuSplit::Get();
if (dma_state.method < non_puller_methods) {
puller.CallMultiMethod(*this, dma_state.method, dma_state.subchannel, base_start, num_methods, dma_state.method_count);
} else {
@@ -358,6 +411,30 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) {
ReportUnboundSubchannel();
return;
}
+ if (split.Active()) {
+ const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset;
+ if (split.FrontMacros()) {
+ const auto type = subchannel_type[dma_state.subchannel];
+ if (type == Engines::EngineTypes::Maxwell3D) {
+ auto* const maxwell3d = static_cast<Engines::Maxwell3D*>(subchannel);
+ if (maxwell3d->GetNativeMacroEngine().native_turing) {
+ maxwell3d->FrontCallMultiMethod(system, split, dma_state.method, base_start,
+ num_methods, dma_state.method_count);
+ return;
+ }
+ } else if (type == Engines::EngineTypes::KeplerCompute) {
+ auto* const compute = static_cast<Engines::KeplerCompute*>(subchannel);
+ if (compute->FrontRunsMacros()) {
+ compute->FrontCallMultiMethod(system, split, dma_state.method, base_start,
+ num_methods, dma_state.method_count, &segment);
+ return;
+ }
+ }
+ }
+ split.MultiMethod(subchannel, dma_state.method, base_start, num_methods,
+ dma_state.method_count, WantsSegment() ? &segment : nullptr);
+ return;
+ }
VideoCommon::Nx2Profile::Scope method_scope{VideoCommon::Nx2Profile::Zone::Method};
subchannel->ConsumeSink(system);
subchannel->current_dma_segment = dma_state.dma_get + dma_state.dma_word_offset;
diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h
index bbcc345c84..82d29edb4b 100644
--- a/src/video_core/dma_pusher.h
+++ b/src/video_core/dma_pusher.h
@@ -172,6 +172,7 @@ private:
void CallMethod(u32 argument);
void CallMultiMethod(const u32* base_start, u32 num_methods);
+ [[nodiscard]] bool WantsSegment() const;
void ReportUnboundSubchannel();
void WatchArgument(u32 method, u32 argument) const;
diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp
index ed4e9674b6..6ef639fa91 100644
--- a/src/video_core/engines/kepler_compute.cpp
+++ b/src/video_core/engines/kepler_compute.cpp
@@ -17,6 +17,7 @@
#include "common/logging.h"
#include "shader_recompiler/exception.h"
#include "core/core.h"
+#include "video_core/gpu_split.h"
#include "video_core/nx2_profile.h"
#include "video_core/nx2.h"
#include "video_core/engines/fermi_2d.h"
@@ -120,7 +121,7 @@ void KeplerCompute::ProcessNativeReportSemaphore(u32 argument) {
}
bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call) {
- if (native_context) {
+ if (native_context && !front_macros) {
if (method == 0x45 || method == 0x47) native_compute_uploaded = true;
native_macro = native_compute_uploaded ? &native_context->compute_engine : &native_context->engine;
native_macro_positions = native_compute_uploaded ? std::span<u32>{native_context->compute_positions} : std::span<u32>{native_context->positions};
@@ -308,6 +309,9 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar
}
return true;
}
+ if (front_macros && method >= 0x45 && method <= 0x48) {
+ return true; // Loaded and bound by the split GPU thread's front (FrontApply).
+ }
switch (method) {
case 0x45: native_macro->ClearCode(argument); return true;
case 0x46: native_macro->AddCode(0, argument); return true;
@@ -754,4 +758,162 @@ Texture::TSCEntry KeplerCompute::GetTSCEntry(u32 tsc_index) const {
return tsc_entry;
}
+// Split GPU thread with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2): the front
+// runs compute macros against its own copy of the native registers (which hold the last value
+// written to each, on the back too) and loads and binds the compute macro code itself.
+struct KeplerCompute::FrontState {
+ std::array<u32, 0x1000> registers{};
+ std::optional<u32> executing_macro;
+ std::vector<u32> parameters;
+ bool compute_uploaded{};
+ u32 bind_pointer{};
+ MacroEngine* macro{}; ///< Without a native context: the one macro engine
+};
+
+KeplerCompute::FrontState& KeplerCompute::Front(VideoCommon::GpuSplit& split) {
+ if (!front) [[unlikely]] {
+ // Copy the back's state and take the macros over while it is idle.
+ split.Drain();
+ front = std::make_unique<FrontState>();
+ front->registers = native_registers;
+ front->executing_macro = native_executing_macro;
+ front->parameters = std::move(native_parameters);
+ native_parameters.clear();
+ native_executing_macro.reset();
+ front->compute_uploaded = native_compute_uploaded;
+ front->bind_pointer = native_bind_pointer;
+ front->macro = native_macro;
+ front_macros = true;
+ }
+ return *front;
+}
+
+void KeplerCompute::FrontApply(FrontState& fs, u32 method, u32 argument) {
+ // As ProcessNativeMethod does with the macro code and bindings.
+ if (method == 0x45 || method == 0x47) {
+ fs.compute_uploaded = true;
+ }
+ fs.registers[method] = argument;
+ switch (method) {
+ case 0x45:
+ FrontMacroEngine(fs).ClearCode(argument);
+ break;
+ case 0x46:
+ FrontMacroEngine(fs).AddCode(0, argument);
+ break;
+ case 0x47:
+ fs.bind_pointer = argument;
+ break;
+ case 0x48: {
+ const auto positions = FrontMacroPositions(fs);
+ if (fs.bind_pointer < positions.size()) {
+ positions[fs.bind_pointer++] = argument;
+ } else {
+ LOG_ERROR(HW_GPU, "Native compute macro bind out of bounds");
+ }
+ break;
+ }
+ default:
+ break;
+ }
+}
+
+MacroEngine& KeplerCompute::FrontMacroEngine(FrontState& fs) {
+ if (native_context) {
+ return fs.compute_uploaded ? native_context->compute_engine : native_context->engine;
+ }
+ return *fs.macro;
+}
+
+std::span<u32> KeplerCompute::FrontMacroPositions(FrontState& fs) {
+ if (native_context) {
+ return fs.compute_uploaded ? std::span<u32>{native_context->compute_positions}
+ : std::span<u32>{native_context->positions};
+ }
+ return native_macro_positions;
+}
+
+void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split,
+ u32 method, u32 argument, bool is_last_call,
+ const GPUVAddr* segment) {
+ auto& fs = Front(split);
+ if (method >= fs.registers.size()) {
+ split.Method(this, method, argument, is_last_call, segment);
+ return;
+ }
+ if (method >= 0xe00) {
+ if (!fs.executing_macro) {
+ if (method & 1) {
+ LOG_ERROR(HW_GPU, "Native compute macro starts on argument method");
+ return;
+ }
+ fs.executing_macro = method;
+ }
+ fs.parameters.push_back(argument);
+ if (is_last_call) {
+ FrontRunMacro(system, split, fs);
+ }
+ return;
+ }
+ FrontApply(fs, method, argument);
+ split.Method(this, method, argument, is_last_call, segment);
+}
+
+void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split,
+ u32 method, const u32* base_start, u32 amount,
+ u32 methods_pending, const GPUVAddr* segment) {
+ auto& fs = Front(split);
+ if (method >= 0xe00 && method < fs.registers.size()) {
+ for (u32 i = 0; i < amount; ++i) {
+ FrontCallMethod(system, split, method, base_start[i], methods_pending - i <= 1, segment);
+ }
+ return;
+ }
+ if (method < fs.registers.size()) {
+ for (u32 i = 0; i < amount; ++i) {
+ FrontApply(fs, method, base_start[i]);
+ }
+ }
+ split.MultiMethod(this, method, base_start, amount, methods_pending, segment);
+}
+
+void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split,
+ FrontState& fs) {
+#ifdef ARCHITECTURE_x86_64
+ const auto positions = FrontMacroPositions(fs);
+ const u32 start = positions[((*fs.executing_macro - 0xe00) / 2) % positions.size()];
+ fs.executing_macro.reset();
+ auto& macro = FrontMacroEngine(fs);
+ if (native_data_ram) {
+ macro.native_interpreter.active_dram = native_data_ram;
+ }
+ VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro};
+ if (!macro.native_jit) {
+ macro.native_jit = std::make_shared<TuringMacroJit>();
+ }
+ macro.native_jit->Prepare(macro.native_code, macro.native_code_generation);
+ const auto result = macro.native_jit->Execute(
+ start, fs.parameters, std::span<u32>{*macro.native_interpreter.active_dram},
+ [&fs](u32 address) {
+ return address < fs.registers.size() ? fs.registers[address] : 0U;
+ },
+ [&](u32 address, u32 value) {
+ if (address >= 0xe00) {
+ return false;
+ }
+ FrontApply(fs, address, value);
+ split.MacroWrite(this, address, value);
+ return true;
+ });
+ if (!result.success) {
+ macro.native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Native MME (JIT, split front) failed: compute start={:#x}, consumed={}, error={}",
+ start, result.consumed, result.error);
+ }
+#else
+ fs.executing_macro.reset();
+#endif
+ fs.parameters.clear();
+}
+
} // namespace Tegra::Engines
diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h
index 02e2f92f71..b525636dde 100644
--- a/src/video_core/engines/kepler_compute.h
+++ b/src/video_core/engines/kepler_compute.h
@@ -9,7 +9,9 @@
#include <array>
#include <atomic>
#include <cstddef>
+#include <memory>
#include <optional>
+#include <span>
#include <vector>
#include "common/bit_field.h"
#include "common/common_funcs.h"
@@ -24,6 +26,10 @@ namespace Core {
class System;
}
+namespace VideoCommon {
+class GpuSplit;
+}
+
namespace Tegra {
class MemoryManager;
}
@@ -237,7 +243,26 @@ public:
bool IsNativeMode() const { return native_mode; }
+ /// GPU thread split with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2), called by
+ /// the front instead of CallMethod for native compute engines: runs their macros and macro
+ /// uploads against a copy of the registers and sends the register writes to the back.
+ bool FrontRunsMacros() const { return native_mode; }
+ void FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method,
+ u32 argument, bool is_last_call, const GPUVAddr* segment);
+ void FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method,
+ const u32* base_start, u32 amount, u32 methods_pending,
+ const GPUVAddr* segment);
+
private:
+ struct FrontState;
+ FrontState& Front(VideoCommon::GpuSplit& split);
+ void FrontApply(FrontState& state, u32 method, u32 argument);
+ MacroEngine& FrontMacroEngine(FrontState& state);
+ std::span<u32> FrontMacroPositions(FrontState& state);
+ void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state);
+ std::unique_ptr<FrontState> front;
+ bool front_macros{}; ///< The front owns the macro code and bindings (the back skips them)
+
bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor);
void ProcessNativeReportSemaphore(u32 argument);
void DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd);
diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp
index 997ef474f5..10527c2473 100644
--- a/src/video_core/engines/maxwell_3d.cpp
+++ b/src/video_core/engines/maxwell_3d.cpp
@@ -19,10 +19,14 @@
#include "video_core/engines/maxwell_3d.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/gpu.h"
+#include "video_core/gpu_split.h"
#include "video_core/memory_manager.h"
#include "video_core/nx2_profile.h"
#include "video_core/rasterizer_interface.h"
#include "video_core/textures/texture.h"
+#ifdef ARCHITECTURE_x86_64
+#include "video_core/macro_turing_jit.h"
+#endif
namespace Tegra::Engines {
@@ -347,11 +351,18 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume
case MAXWELL3D_REG_INDEX(shadow_ram_control):
shadow_state.shadow_ram_control = static_cast<Regs::ShadowRamControl>(nonshadow_argument);
return;
+ // With macros on the split GPU thread's front (FrontWrite), the front loads and binds them.
case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+ if (front_macros) return;
return GetNativeMacroEngine().ClearCode(regs.load_mme.instruction_ptr);
case MAXWELL3D_REG_INDEX(load_mme.instruction):
+ if (front_macros) return;
return GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr, argument);
case MAXWELL3D_REG_INDEX(load_mme.start_address):
+ if (front_macros) {
+ ++regs.load_mme.start_address_ptr;
+ return;
+ }
return ProcessMacroBind(argument);
case MAXWELL3D_REG_INDEX(falcon[4]):
return ProcessFirmwareCall4();
@@ -460,6 +471,10 @@ void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument
VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::SameValueWrites, 1);
}
}
+ if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && front_macros) [[unlikely]] {
+ back_control_writes[back_control_count++ % back_control_writes.size()] = {
+ method_argument, static_cast<u32>(shadow_state.shadow_ram_control), is_last_call ? 1U : 0U};
+ }
const u32 argument = ProcessShadowRam(method, method_argument);
// A state register written with the value it holds changes nothing: no dirty flags (they
// would make the next draw re-check that state), no method processing. Title 1 makes ~6
@@ -767,4 +782,414 @@ void Maxwell3D::SetHLEReplacementAttributeType(u32 bank, u32 offset,
replace_table.emplace(key, name);
}
+// Split GPU thread with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2). The front keeps
+// what macros read: the registers as the back thread will hold them when it reaches the same
+// point of the method stream. That is every write the front sends, with the shadow RAM applied,
+// plus the few changes the back makes on its own (constant buffer offset, shadow_scratch[0] on
+// falcon[4], the macro bind pointer); the front loads and binds the macro code itself.
+struct Maxwell3D::FrontState {
+ std::array<ControlWrite, 16> control_writes{};
+ u32 control_count{};
+ std::array<u32, Regs::NUM_REGS> regs{};
+ std::array<u32, Regs::NUM_REGS> shadow{};
+ Regs::ShadowRamControl control{};
+ u32 executing_macro{};
+ std::vector<u32> parameters;
+ std::vector<u32> fifo;
+ u64 calls{};
+ /// A method the draw manager handles was sent since the back last ran DrawDeferred: only
+ /// then can it have a deferred instanced draw to issue at the end of a macro.
+ bool draw_pending{true};
+};
+
+namespace {
+
+// EDEN_NX2_SPLIT_VERIFY=1: every 4096 front macro calls, wait for the back and compare the
+// front's registers with the back's (logs and resynchronizes differences).
+bool FrontVerifyEnabled() {
+ static const bool verify = std::getenv("EDEN_NX2_SPLIT_VERIFY") != nullptr;
+ return verify;
+}
+
+// Executable methods ProcessMethodCall handles itself; the rest go to the draw manager.
+bool FrontHandledOutsideDrawManager(u32 method) {
+ constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer);
+ if (method >= cb_data && method < cb_data + 16) {
+ return true;
+ }
+ switch (method) {
+ case MAXWELL3D_REG_INDEX(wait_for_idle):
+ case MAXWELL3D_REG_INDEX(shadow_ram_control):
+ case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+ case MAXWELL3D_REG_INDEX(load_mme.instruction):
+ case MAXWELL3D_REG_INDEX(load_mme.start_address):
+ case MAXWELL3D_REG_INDEX(falcon[4]):
+ case MAXWELL3D_REG_INDEX(bind_groups[0].raw_config):
+ case MAXWELL3D_REG_INDEX(bind_groups[1].raw_config):
+ case MAXWELL3D_REG_INDEX(bind_groups[2].raw_config):
+ case MAXWELL3D_REG_INDEX(bind_groups[3].raw_config):
+ case MAXWELL3D_REG_INDEX(bind_groups[4].raw_config):
+ case MAXWELL3D_REG_INDEX(report_semaphore.query):
+ case MAXWELL3D_REG_INDEX(render_enable.mode):
+ case MAXWELL3D_REG_INDEX(clear_report_value):
+ case MAXWELL3D_REG_INDEX(sync_info):
+ case MAXWELL3D_REG_INDEX(launch_dma):
+ case MAXWELL3D_REG_INDEX(inline_data):
+ case MAXWELL3D_REG_INDEX(fragment_barrier):
+ case MAXWELL3D_REG_INDEX(invalidate_texture_data_cache):
+ case MAXWELL3D_REG_INDEX(tiled_cache_barrier):
+ return true;
+ default:
+ return false;
+ }
+}
+
+// EDEN_NX2_SPLIT_SKIP=0: send state writes that keep the register's value too.
+bool FrontSkipSame() {
+ static const bool skip = [] {
+ const char* value = std::getenv("EDEN_NX2_SPLIT_SKIP");
+ return value == nullptr || value[0] != '0';
+ }();
+ return skip;
+}
+
+} // Anonymous namespace
+
+Maxwell3D::FrontState& Maxwell3D::Front(Core::System& system, VideoCommon::GpuSplit& split) {
+ if (!front) [[unlikely]] {
+ // Copy the back's state and take the macros over while it is idle (state writes it
+ // still holds in its method sink first).
+ split.Drain();
+ ConsumeSink(system);
+ front = std::make_unique<FrontState>();
+ std::memcpy(front->regs.data(), regs.reg_array.data(), sizeof(front->regs));
+ std::memcpy(front->shadow.data(), shadow_state.reg_array.data(), sizeof(front->shadow));
+ front->control = shadow_state.shadow_ram_control;
+ front->executing_macro = executing_macro;
+ front->parameters = std::move(macro_params);
+ macro_params.clear();
+ executing_macro = 0;
+ front_macros = true;
+ }
+ return *front;
+}
+
+void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 method,
+ u32 argument, bool is_last_call, bool from_macro) {
+ namespace Profile = VideoCommon::Nx2Profile;
+ // As ProcessShadowRam does on the back.
+ u32 value = argument;
+ bool shadow_kept = true;
+ if (fs.control == Regs::ShadowRamControl::Track ||
+ fs.control == Regs::ShadowRamControl::TrackWithFilter) {
+ shadow_kept = fs.shadow[method] == argument;
+ fs.shadow[method] = argument;
+ } else if (fs.control == Regs::ShadowRamControl::Replay) {
+ value = fs.shadow[method];
+ }
+ if (Profile::Enabled()) [[unlikely]] {
+ Profile::Add(Profile::Counter::MethodWrites, 1);
+ }
+ if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && FrontVerifyEnabled()) [[unlikely]] {
+ fs.control_writes[fs.control_count++ % fs.control_writes.size()] = {
+ argument, static_cast<u32>(fs.control), from_macro ? 2U : 1U};
+ }
+ if (!execution_mask[method]) {
+ // Nothing would change on the back: CallMethod and ConsumeSinkImpl skip same-value state
+ // writes, and the shadow RAM (tracked by default) holds the value already.
+ if (fs.regs[method] == value && shadow_kept && SkipSameValueWrites() && FrontSkipSame()) {
+ if (Profile::Enabled()) [[unlikely]] {
+ Profile::Add(Profile::Counter::SameValueWrites, 1);
+ }
+ return;
+ }
+ fs.regs[method] = value;
+ } else {
+ fs.regs[method] = value;
+ constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer);
+ switch (method) {
+ case MAXWELL3D_REG_INDEX(shadow_ram_control):
+ // The back's mode is shadow_state.shadow_ram_control: the shadow copy of this very
+ // register, which a later Replay write reads back.
+ fs.control = static_cast<Regs::ShadowRamControl>(argument);
+ fs.shadow[method] = argument;
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+ GetNativeMacroEngine().ClearCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)]);
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.instruction):
+ GetNativeMacroEngine().AddCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)], value);
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.start_address): {
+ const auto positions = GetNativeMacroPositions();
+ u32& pointer = fs.regs[MAXWELL3D_REG_INDEX(load_mme.start_address_ptr)];
+ if (pointer < positions.size()) {
+ positions[pointer] = value;
+ }
+ ++pointer;
+ break;
+ }
+ case MAXWELL3D_REG_INDEX(falcon[4]):
+ fs.regs[MAXWELL3D_REG_INDEX(shadow_scratch)] = 1;
+ break;
+ default:
+ if (method >= cb_data && method < cb_data + 16) {
+ fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)] += sizeof(u32);
+ } else if (!FrontHandledOutsideDrawManager(method)) {
+ fs.draw_pending = true;
+ }
+ break;
+ }
+ }
+ if (from_macro) {
+ split.MacroWrite(this, method, argument);
+ } else {
+ split.Method(this, method, argument, is_last_call, nullptr);
+ }
+}
+
+void Maxwell3D::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method,
+ u32 argument, bool is_last_call) {
+ auto& fs = Front(system, split);
+ if (method >= MacroRegistersStart) {
+ if (fs.executing_macro == 0) {
+ fs.executing_macro = method;
+ }
+ fs.parameters.push_back(argument);
+ if (is_last_call) {
+ FrontRunMacro(system, split, fs);
+ }
+ return;
+ }
+ FrontWrite(split, fs, method, argument, is_last_call, false);
+}
+
+void Maxwell3D::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split,
+ u32 method, const u32* base_start, u32 amount,
+ u32 methods_pending) {
+ auto& fs = Front(system, split);
+ if (method >= MacroRegistersStart) {
+ if (fs.executing_macro == 0) {
+ fs.executing_macro = method;
+ }
+ fs.parameters.insert(fs.parameters.end(), base_start, base_start + amount);
+ if (amount == methods_pending) {
+ FrontRunMacro(system, split, fs);
+ }
+ return;
+ }
+ // What CallMultiMethod does to the registers on the back, then one entry for all the words.
+ constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer);
+ const auto tracked = [&fs] {
+ return fs.control == Regs::ShadowRamControl::Track ||
+ fs.control == Regs::ShadowRamControl::TrackWithFilter;
+ };
+ if (method >= cb_data && method < cb_data + 16) {
+ fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)] += amount * sizeof(u32);
+ } else if (method == MAXWELL3D_REG_INDEX(inline_data)) {
+ // Upload data only.
+ } else if ((method == MAXWELL3D_REG_INDEX(draw_inline_index) ||
+ method == MAXWELL3D_REG_INDEX(inline_index_2x16.even) ||
+ method == MAXWELL3D_REG_INDEX(inline_index_4x8.index0)) &&
+ fs.control != Regs::ShadowRamControl::Replay) {
+ if (amount != 0) {
+ const u32 last = base_start[amount - 1];
+ if (tracked()) {
+ fs.shadow[method] = last;
+ }
+ fs.regs[method] = last;
+ }
+ fs.draw_pending = true;
+ } else {
+ // CallMethod for each word, macro uploads among them (load_mme.instruction). A word may
+ // change the shadow RAM mode for the next (a run of SET_SHADOW_RAM_CONTROL writes).
+ for (u32 i = 0; i < amount; ++i) {
+ const u32 argument = base_start[i];
+ u32 value = argument;
+ if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && FrontVerifyEnabled()) [[unlikely]] {
+ fs.control_writes[fs.control_count++ % fs.control_writes.size()] = {
+ argument, static_cast<u32>(fs.control), 3U};
+ }
+ if (tracked()) {
+ fs.shadow[method] = argument;
+ } else if (fs.control == Regs::ShadowRamControl::Replay) {
+ value = fs.shadow[method];
+ }
+ fs.regs[method] = value;
+ if (!execution_mask[method]) {
+ continue;
+ }
+ switch (method) {
+ case MAXWELL3D_REG_INDEX(shadow_ram_control):
+ fs.control = static_cast<Regs::ShadowRamControl>(argument);
+ fs.shadow[method] = argument; // The same word on the back (see FrontWrite)
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+ GetNativeMacroEngine().ClearCode(value);
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.instruction):
+ GetNativeMacroEngine().AddCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)], value);
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.start_address): {
+ const auto positions = GetNativeMacroPositions();
+ u32& pointer = fs.regs[MAXWELL3D_REG_INDEX(load_mme.start_address_ptr)];
+ if (pointer < positions.size()) {
+ positions[pointer] = value;
+ }
+ ++pointer;
+ break;
+ }
+ case MAXWELL3D_REG_INDEX(falcon[4]):
+ fs.regs[MAXWELL3D_REG_INDEX(shadow_scratch)] = 1;
+ break;
+ default:
+ if (!FrontHandledOutsideDrawManager(method)) {
+ fs.draw_pending = true;
+ }
+ break;
+ }
+ }
+ }
+ split.MultiMethod(this, method, base_start, amount, methods_pending, nullptr);
+}
+
+void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split,
+ FrontState& fs) {
+#ifdef ARCHITECTURE_x86_64
+ const u32 method = fs.executing_macro;
+ fs.executing_macro = 0;
+ auto& engine = GetNativeMacroEngine();
+ const auto positions = GetNativeMacroPositions();
+ const u32 start = positions[((method - MacroRegistersStart) >> 1) % positions.size()];
+ {
+ VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro};
+ engine.native_interpreter.active_dram = &native_data_ram;
+ if (!engine.native_jit) {
+ engine.native_jit = std::make_shared<TuringMacroJit>();
+ }
+ engine.native_jit->Prepare(engine.native_code, engine.native_code_generation);
+ fs.fifo.assign(fs.parameters.begin(), fs.parameters.end());
+ const auto result = engine.native_jit->Execute(
+ start, fs.fifo, std::span<u32>{native_data_ram},
+ [&fs](u32 address) { return address < Regs::NUM_REGS ? fs.regs[address] : 0U; },
+ [&](u32 address, u32 value) {
+ if (address >= MacroRegistersStart) {
+ return false;
+ }
+ FrontWrite(split, fs, address, value, true, true);
+ // MME_DMA_READ (into data RAM) and MME_DMA_READ_FIFOED (into the load FIFO).
+ if (address == 0x157 || address == 0x158) {
+ FrontDmaRead(split, fs, address, value);
+ }
+ return true;
+ });
+ if (!result.success) {
+ engine.native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Native MME (JIT, split front) failed: start={:#x}, consumed={}/{}, error={}",
+ start, result.consumed, fs.parameters.size(), result.error);
+ }
+ }
+ fs.parameters.clear();
+ if (fs.draw_pending) {
+ fs.draw_pending = false;
+ split.Defer([this, system_ptr = &system] { FinishFrontMacro(*system_ptr); });
+ }
+ if (FrontVerifyEnabled() && (++fs.calls & 4095) == 0) [[unlikely]] {
+ FrontVerify(split, fs);
+ }
+#else
+ fs.executing_macro = 0;
+ fs.parameters.clear();
+#endif
+}
+
+void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 method,
+ u32 count) {
+ // The words may be written by methods still queued for the back (indirect draw arguments
+ // from a culling dispatch), and the caches that may hold newer ones belong to it.
+ split.Drain();
+ const GPUVAddr source = (GPUVAddr{fs.regs[0x154] & 0x1ffffff} << 32) | fs.regs[0x155];
+ const u32 ram = fs.regs[0x156];
+ const u32 words = std::min<u32>(count, 0x1000);
+ const size_t read_bytes = size_t{words} * sizeof(u32);
+ if (VideoCommon::Nx2Profile::Enabled()) {
+ VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::DmaReadBytes, read_bytes);
+ }
+ if (memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) {
+ memory_manager.FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache);
+ }
+ for (u32 i = 0; i < words; ++i) {
+ const u32 word = memory_manager.Read<u32>(source + u64{i} * sizeof(u32));
+ if (method == 0x158) {
+ fs.fifo.push_back(word);
+ } else if (ram + i < native_data_ram.size()) {
+ native_data_ram[ram + i] = word;
+ }
+ }
+}
+
+void Maxwell3D::FrontVerify(VideoCommon::GpuSplit& split, FrontState& fs) {
+ split.Drain();
+ static u64 checks{};
+ static u64 differences{};
+ ++checks;
+ // State writes the back still holds in its method sink count as done.
+ std::array<u32, Regs::NUM_REGS> expected;
+ std::memcpy(expected.data(), regs.reg_array.data(), sizeof(expected));
+ for (const auto& [method, value] : method_sink) {
+ expected[method] = shadow_state.shadow_ram_control == Regs::ShadowRamControl::Replay
+ ? shadow_state.reg_array[method]
+ : value;
+ }
+ static u64 control_differences{};
+ if ((fs.control != shadow_state.shadow_ram_control ||
+ fs.shadow[MAXWELL3D_REG_INDEX(shadow_ram_control)] != shadow_state.reg_array[MAXWELL3D_REG_INDEX(shadow_ram_control)] ||
+ fs.regs[MAXWELL3D_REG_INDEX(shadow_ram_control)] != expected[MAXWELL3D_REG_INDEX(shadow_ram_control)]) &&
+ control_differences < 4) {
+ LOG_WARNING(HW_GPU, "Split shadow RAM control writes, front ({} total) / back ({} total):",
+ fs.control_count, back_control_count);
+ for (u32 i = 0; i < 16; ++i) {
+ const auto& f = fs.control_writes[(fs.control_count + i) % 16];
+ const auto& b = back_control_writes[(back_control_count + i) % 16];
+ LOG_WARNING(HW_GPU, " front {:#x} (mode {}, path {}) | back {:#x} (mode {}, last {})",
+ f.argument, f.before, f.path, b.argument, b.before, b.path);
+ }
+ }
+ for (u32 i = 0; i < Regs::NUM_REGS; ++i) {
+ if (fs.regs[i] != expected[i]) {
+ if (differences++ < 64) {
+ LOG_WARNING(HW_GPU, "Split front register {:#x}: front {:#x}, back {:#x}", i,
+ fs.regs[i], expected[i]);
+ }
+ fs.regs[i] = expected[i];
+ }
+ }
+ if (fs.control != shadow_state.shadow_ram_control) {
+ if (control_differences++ < 16) {
+ LOG_WARNING(HW_GPU, "Split front shadow RAM control {} vs back {}",
+ static_cast<u32>(fs.control),
+ static_cast<u32>(shadow_state.shadow_ram_control));
+ }
+ fs.control = shadow_state.shadow_ram_control;
+ }
+ for (u32 i = 0; i < Regs::NUM_REGS; ++i) {
+ if (fs.shadow[i] != shadow_state.reg_array[i]) {
+ if (control_differences++ < 16) {
+ LOG_WARNING(HW_GPU, "Split front shadow register {:#x}: front {:#x}, back {:#x}", i,
+ fs.shadow[i], shadow_state.reg_array[i]);
+ }
+ fs.shadow[i] = shadow_state.reg_array[i];
+ }
+ }
+ if ((checks & 63) == 1) {
+ LOG_INFO(HW_GPU, "Split front register checks: {} done, {} differences", checks, differences);
+ }
+}
+
+void Maxwell3D::FinishFrontMacro(Core::System& system) {
+ // What ProcessMacro and CallMacroMethod do around the macro.
+ ConsumeSink(system);
+ draw_manager.DrawDeferred(*this);
+}
+
} // namespace Tegra::Engines
diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h
index 8a806cc070..0121f1a01c 100644
--- a/src/video_core/engines/maxwell_3d.h
+++ b/src/video_core/engines/maxwell_3d.h
@@ -37,6 +37,10 @@ namespace Tegra {
class MemoryManager;
}
+namespace VideoCommon {
+class GpuSplit;
+}
+
namespace VideoCore {
class RasterizerInterface;
}
@@ -3163,6 +3167,14 @@ public:
/// Write multiple values to the register identified by method.
void CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) override;
+ /// GPU thread split with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2), called by
+ /// the front instead of CallMethod: runs macros and macro uploads against a copy of the
+ /// registers and sends the register writes to the back thread.
+ void FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method,
+ u32 argument, bool is_last_call);
+ void FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method,
+ const u32* base_start, u32 amount, u32 methods_pending);
+
bool ShouldExecute() const {
return execute_on;
}
@@ -3324,6 +3336,28 @@ private:
std::vector<std::pair<GPUVAddr, size_t>> macro_segments;
bool current_macro_dirty{};
+
+ // Split GPU thread, macros on the front (FrontCallMethod): the front's state, and whether
+ // the front owns the macro code and bindings (the back thread then leaves them alone).
+ struct FrontState;
+ FrontState& Front(Core::System& system, VideoCommon::GpuSplit& split);
+ void FrontWrite(VideoCommon::GpuSplit& split, FrontState& state, u32 method, u32 argument,
+ bool is_last_call, bool from_macro);
+ void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state);
+ void FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& state, u32 method, u32 count);
+ void FrontVerify(VideoCommon::GpuSplit& split, FrontState& state);
+ void FinishFrontMacro(Core::System& system);
+ std::unique_ptr<FrontState> front;
+ bool front_macros{};
+ // EDEN_NX2_SPLIT_VERIFY: the last SET_SHADOW_RAM_CONTROL writes each side saw (argument,
+ // mode before, path), printed when the sides disagree.
+ struct ControlWrite {
+ u32 argument;
+ u32 before;
+ u32 path;
+ };
+ std::array<ControlWrite, 16> back_control_writes{};
+ u32 back_control_count{};
};
#define ASSERT_REG_POSITION(field_name, position) \
diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp
index c9073f1eb0..0bb1809aa3 100644
--- a/src/video_core/engines/puller.cpp
+++ b/src/video_core/engines/puller.cpp
@@ -21,6 +21,7 @@
#include "video_core/engines/maxwell_dma.h"
#include "video_core/engines/puller.h"
#include "video_core/gpu.h"
+#include "video_core/gpu_split.h"
#include "video_core/memory_manager.h"
#include "video_core/nx2.h"
#include "video_core/rasterizer_interface.h"
@@ -43,6 +44,21 @@ u32 ReadSemaphoreWord(DmaPusher& dma_pusher, GPUVAddr address) {
return dma_pusher.memory_manager.Read<u32>(address);
}
+// With the GPU thread split (gpu_split.h) the rasterizer belongs to the back thread: work that
+// only feeds it runs there in order; reads of what it wrote first wait for it (DrainBack).
+template <typename Func>
+void OnBack(const Func& func) {
+ if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) {
+ split.Defer(func);
+ } else {
+ func();
+ }
+}
+
+void DrainBack() {
+ VideoCommon::GpuSplit::Get().Drain();
+}
+
} // Anonymous namespace
bool Puller::PendingAcquire::Matches(u32 current) const {
@@ -62,6 +78,9 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) {
if (!pending_acquire) {
return true;
}
+ // The release may still be queued for the split GPU thread's back half (and the caches
+ // ReadSemaphoreWord asks are its state).
+ DrainBack();
const u32 current = ReadSemaphoreWord(dma_pusher, pending_acquire->address);
if (Nx2SemaphoreWaiters::Get().Latched(this)) {
if (static std::atomic<u32> rescued{};
@@ -161,10 +180,12 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) {
case Puller::FenceOperation::Acquire:
// UNIMPLEMENTED_MSG("Channel Scheduling pending.");
// WaitFence(regs.fence_action.syncpoint_id, regs.fence_value);
- dma_pusher.rasterizer->ReleaseFences();
+ OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); });
break;
case Puller::FenceOperation::Increment:
- dma_pusher.rasterizer->SignalSyncPoint(regs.fence_action.syncpoint_id);
+ OnBack([rasterizer = dma_pusher.rasterizer, id = static_cast<u32>(regs.fence_action.syncpoint_id.Value())] {
+ rasterizer->SignalSyncPoint(id);
+ });
break;
default:
UNIMPLEMENTED_MSG("Unimplemented operation {}", regs.fence_action.op.Value());
@@ -196,12 +217,16 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
const auto properties = g_nx2_ampere && (regs.semaphore_trigger & (1U << 24)) != 0
? VideoCommon::QueryPropertiesFlags::IsAFence
: VideoCommon::QueryPropertiesFlags::HasTimeout;
- dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0);
- if (const GPUVAddr mirror = g_nx2_ampere ? Nx2SemaphoreMirror(sequence_address) : 0;
- mirror != 0) {
+ const GPUVAddr mirror = g_nx2_ampere ? Nx2SemaphoreMirror(sequence_address) : 0;
+ if (mirror != 0) {
LOG_DEBUG(HW_GPU, "Switch 2 semaphore {:#x}={:#x} mirrored to {:#x}", sequence_address, payload, mirror);
- dma_pusher.rasterizer->Query(mirror, VideoCommon::QueryType::Payload, properties, payload, 0);
}
+ OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, mirror, properties, payload] {
+ rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0);
+ if (mirror != 0) {
+ rasterizer->Query(mirror, VideoCommon::QueryType::Payload, properties, payload, 0);
+ }
+ });
} else if (g_nx2_ampere) {
// Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has
// written the commands this channel runs next: suspend the channel until the value is
@@ -219,6 +244,7 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
return;
}
VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1);
+ DrainBack();
if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) {
return;
}
@@ -236,6 +262,7 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
waiters.Remove(this);
}
} else {
+ DrainBack();
do {
const u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress());
regs.acquire_source = true;
@@ -269,10 +296,14 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
void Puller::ProcessSemaphoreRelease(DmaPusher& dma_pusher) {
const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()};
const u32 payload = regs.semaphore_release;
- dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+ OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, payload] {
+ rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload,
+ VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+ });
}
void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) {
+ DrainBack();
u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress());
const auto value = regs.semaphore_acquire;
while (word != value) {
@@ -294,11 +325,14 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) {
const u32 payload = regs.reg_array[0x19];
const u32 operation = argument & 7;
if (operation == 1) { // RELEASE
- dma_pusher.rasterizer->Query(address, VideoCommon::QueryType::Payload,
- VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+ OnBack([rasterizer = dma_pusher.rasterizer, address, payload] {
+ rasterizer->Query(address, VideoCommon::QueryType::Payload,
+ VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+ });
return;
}
// Acquires: check once after flushing pending fences; never block the GPU thread on them.
+ DrainBack();
dma_pusher.rasterizer->ReleaseFences();
const u32 word = dma_pusher.memory_manager.Read<u32>(address);
const bool ready = operation == 0 ? word == payload
@@ -339,13 +373,13 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
case BufferMethods::WrcacheFlush:
break;
case BufferMethods::RefCnt:
- dma_pusher.rasterizer->SignalReference();
+ OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->SignalReference(); });
break;
case BufferMethods::SyncpointOperation:
ProcessFenceActionMethod(dma_pusher);
break;
case BufferMethods::WaitForIdle:
- dma_pusher.rasterizer->WaitForIdle();
+ OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->WaitForIdle(); });
break;
case BufferMethods::SemaphoreOperation: {
ProcessSemaphoreTriggerMethod(dma_pusher);
@@ -361,7 +395,7 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
}
case BufferMethods::MemOpB: {
// Implement this better.
- dma_pusher.rasterizer->InvalidateGPUCache();
+ OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->InvalidateGPUCache(); });
break;
}
case BufferMethods::MemOpC:
diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp
new file mode 100644
index 0000000000..970fc0b559
--- /dev/null
+++ b/src/video_core/gpu_split.cpp
@@ -0,0 +1,363 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <algorithm>
+#include <cstdlib>
+#include <thread>
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <cstdio>
+
+#include "common/logging.h"
+#include "common/settings.h"
+#include "common/thread.h"
+#include "core/core.h"
+#include "video_core/engines/engine_interface.h"
+#include "video_core/gpu_split.h"
+#include "video_core/nx2.h"
+#include "video_core/nx2_profile.h"
+
+namespace VideoCommon {
+
+namespace {
+
+void Pause() {
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
+ _mm_pause();
+#elif defined(__x86_64__) || defined(__i386__)
+ __builtin_ia32_pause();
+#else
+ std::this_thread::yield();
+#endif
+}
+
+std::atomic<u32> g_back_thread_id{};
+std::atomic<bool> g_back_busy{};
+
+} // Anonymous namespace
+
+void GpuSplit::CheckOwner(const char* where) {
+#ifdef _WIN32
+ if (!g_back_busy.load(std::memory_order_acquire)) {
+ return;
+ }
+ const DWORD thread = GetCurrentThreadId();
+ if (thread == g_back_thread_id.load(std::memory_order_relaxed)) {
+ return;
+ }
+ static std::atomic<u32> reports{};
+ if (reports.fetch_add(1) >= 12) {
+ return;
+ }
+ std::fprintf(stderr, "GPU split owner check: %s on thread %lu while the back runs\n", where,
+ thread);
+ void* frames[40];
+ const USHORT count = RtlCaptureStackBackTrace(1, 40, frames, nullptr);
+ for (USHORT i = 0; i < count; ++i) {
+ HMODULE module{};
+ char name[MAX_PATH] = "?";
+ if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
+ GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+ static_cast<LPCSTR>(frames[i]), &module)) {
+ GetModuleFileNameA(module, name, sizeof(name));
+ }
+ const char* base_name = std::strrchr(name, '\\');
+ std::fprintf(stderr, " frame %s+%#llx\n", base_name ? base_name + 1 : name,
+ static_cast<unsigned long long>(reinterpret_cast<uintptr_t>(frames[i]) -
+ reinterpret_cast<uintptr_t>(module)));
+ }
+ std::fflush(stderr);
+#else
+ (void)where;
+#endif
+}
+
+GpuSplit& GpuSplit::Get() {
+ static GpuSplit split;
+ return split;
+}
+
+void GpuSplit::MaybeStart(Core::System& system_) {
+ static const char mode = [] {
+ const char* value = std::getenv("EDEN_NX2_SPLIT");
+ return value != nullptr ? value[0] : '0';
+ }();
+ const bool wanted = mode == '1' || mode == '2';
+ if (active || !wanted || !Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) {
+ return;
+ }
+ // The front reads command lists straight from guest memory; a read that flushes the caches
+ // first (safe DMA, accurate GPU) would touch back-owned state.
+ const bool safe_reads = Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh()
+ : Settings::IsDMALevelSafe();
+ if (safe_reads || Settings::IsGPULevelHigh() ||
+ Settings::values.renderer_backend.GetValue() != Settings::RendererBackend::Vulkan) {
+ return;
+ }
+ system = &system_;
+ check_owner = std::getenv("EDEN_NX2_SPLIT_CHECK") != nullptr;
+ ring = std::make_unique<u64[]>(RingWords);
+ back_thread = std::jthread([this](std::stop_token stop_token) { BackLoop(stop_token); });
+ active = true;
+#ifdef ARCHITECTURE_x86_64
+ // Front macros run compiled (macro_turing_jit.h) only.
+ const char* jit = std::getenv("EDEN_NX2_MME_JIT");
+ front_macros = mode == '2' && (jit == nullptr || jit[0] != '0');
+#endif
+ LOG_INFO(HW_GPU, "GPU thread split: engine methods run on a second thread{}",
+ front_macros ? ", macros on the first" : "");
+}
+
+void GpuSplit::Stop() {
+ if (!back_thread.joinable()) {
+ return;
+ }
+ back_thread.request_stop();
+ wake.fetch_add(1, std::memory_order_seq_cst);
+ wake.notify_one();
+ back_thread.join();
+ active = false;
+}
+
+u64* GpuSplit::Reserve(size_t words) {
+ size_t index = write_position & RingMask;
+ if (index + words > RingWords) {
+ // Entries never wrap: skip to the start of the ring.
+ const size_t rest = RingWords - index;
+ WaitForSpace(rest);
+ ring[index] = static_cast<u64>(Kind::Wrap);
+ write_position += rest;
+ index = 0;
+ }
+ WaitForSpace(words);
+ return &ring[index];
+}
+
+void GpuSplit::WaitForSpace(size_t words) {
+ if (write_position + words - consumed_cache <= RingWords) {
+ return;
+ }
+ Publish();
+ Nx2Profile::Scope scope{Nx2Profile::Zone::SplitDrain};
+ for (u32 spins = 0;; ++spins) {
+ consumed_cache = consumed.load(std::memory_order_acquire);
+ if (write_position + words - consumed_cache <= RingWords) {
+ return;
+ }
+ if (spins < 4096) {
+ Pause();
+ } else {
+ std::this_thread::yield();
+ }
+ }
+}
+
+void GpuSplit::Publish() {
+ if (published_local == write_position) {
+ return;
+ }
+ published_local = write_position;
+ publish_every = std::min<size_t>(publish_every * 2, PublishEvery);
+ published.store(write_position, std::memory_order_seq_cst);
+ if (back_waiting.load(std::memory_order_seq_cst)) {
+ wake.fetch_add(1, std::memory_order_seq_cst);
+ wake.notify_one();
+ }
+}
+
+void GpuSplit::Drain() {
+ consumed_cache = consumed.load(std::memory_order_acquire);
+ if (consumed_cache == write_position) {
+ return;
+ }
+ Publish();
+ Nx2Profile::Scope scope{Nx2Profile::Zone::SplitDrain};
+ for (u32 spins = 0;; ++spins) {
+ consumed_cache = consumed.load(std::memory_order_acquire);
+ if (consumed_cache == write_position) {
+ publish_every = 8;
+ return;
+ }
+ if (spins < 16384) {
+ Pause();
+ } else {
+ std::this_thread::yield();
+ }
+ }
+}
+
+void GpuSplit::SelectEngine(Tegra::Engines::EngineInterface* engine) {
+ if (engine == front_engine) {
+ return;
+ }
+ front_engine = engine;
+ u64* const entry = Reserve(2);
+ entry[0] = static_cast<u64>(Kind::Engine);
+ entry[1] = reinterpret_cast<u64>(engine);
+ Commit(2);
+}
+
+void GpuSplit::Method(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument,
+ bool is_last, const GPUVAddr* segment) {
+ SelectEngine(engine);
+ const size_t words = segment != nullptr ? 2 : 1;
+ u64* const entry = Reserve(words);
+ entry[0] = static_cast<u64>(Kind::Method) | (is_last ? IsLast : 0) |
+ (segment != nullptr ? HasSegment : 0) | (u64{method & 0xffff} << 8) |
+ (u64{argument} << 32);
+ if (segment != nullptr) {
+ entry[1] = *segment;
+ }
+ Commit(words);
+}
+
+void GpuSplit::MacroWrite(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument) {
+ SelectEngine(engine);
+ u64* const entry = Reserve(1);
+ entry[0] = static_cast<u64>(Kind::Method) | IsLast | Direct | (u64{method & 0xffff} << 8) |
+ (u64{argument} << 32);
+ Commit(1);
+}
+
+void GpuSplit::MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words,
+ u32 amount, u32 methods_pending, const GPUVAddr* segment) {
+ SelectEngine(engine);
+ // Long runs (constant buffer and inline uploads) go in pieces, as when a method's data
+ // continues in the next GPFIFO entry.
+ while (amount > 0) {
+ const u32 piece = std::min(amount, MaxMultiWords);
+ const size_t head = segment != nullptr ? 3 : 2;
+ const size_t size = head + (piece + 1) / 2;
+ u64* const entry = Reserve(size);
+ entry[0] = static_cast<u64>(Kind::Multi) | (segment != nullptr ? HasSegment : 0) |
+ (u64{method & 0xffff} << 8) | (u64{piece} << 32);
+ entry[1] = methods_pending;
+ if (segment != nullptr) {
+ entry[2] = *segment;
+ }
+ entry[size - 1] = 0;
+ std::memcpy(entry + head, words, size_t{piece} * sizeof(u32));
+ Commit(size);
+ words += piece;
+ amount -= piece;
+ methods_pending -= piece;
+ }
+}
+
+void GpuSplit::DeferFunction(std::function<void()>&& func) {
+ auto* const heap = new std::function<void()>(std::move(func));
+ Defer([heap] {
+ (*heap)();
+ delete heap;
+ });
+}
+
+size_t GpuSplit::Execute(const u64* entry) {
+ const u64 head = entry[0];
+ switch (static_cast<Kind>(head & 0xf)) {
+ case Kind::Wrap:
+ return 0;
+ case Kind::Engine:
+ back_engine = reinterpret_cast<Tegra::Engines::EngineInterface*>(entry[1]);
+ return 2;
+ case Kind::Method: {
+ const u32 method = static_cast<u32>((head >> 8) & 0xffff);
+ const u32 argument = static_cast<u32>(head >> 32);
+ auto* const engine = back_engine;
+ if ((head & Direct) != 0) {
+ Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method};
+ engine->ConsumeSink(*system);
+ engine->CallMethod(*system, method, argument, true);
+ } else if (!engine->execution_mask[method]) {
+ engine->method_sink.emplace_back(method, argument);
+ } else {
+ Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method};
+ engine->ConsumeSink(*system);
+ if ((head & HasSegment) != 0) {
+ engine->current_dma_segment = entry[1];
+ }
+ engine->CallMethod(*system, method, argument, (head & IsLast) != 0);
+ }
+ return (head & HasSegment) != 0 ? 2 : 1;
+ }
+ case Kind::Multi: {
+ const u32 method = static_cast<u32>((head >> 8) & 0xffff);
+ const u32 amount = static_cast<u32>(head >> 32);
+ const size_t data = (head & HasSegment) != 0 ? 3 : 2;
+ auto* const engine = back_engine;
+ Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method};
+ engine->ConsumeSink(*system);
+ if ((head & HasSegment) != 0) {
+ engine->current_dma_segment = entry[2];
+ }
+ engine->CallMultiMethod(*system, method, reinterpret_cast<const u32*>(entry + data),
+ amount, static_cast<u32>(entry[1]));
+ return data + (amount + 1) / 2;
+ }
+ case Kind::Call: {
+ const size_t payload = static_cast<size_t>((head >> 8) & 0xffffff);
+ reinterpret_cast<void (*)(const u64*)>(entry[1])(entry + 2);
+ return 2 + payload;
+ }
+ }
+ return 1;
+}
+
+void GpuSplit::BackLoop(std::stop_token stop_token) {
+ Common::SetCurrentThreadName("GPUBack");
+ Common::SetCurrentThreadPriority(Common::ThreadPriority::Critical);
+ Common::SetCurrentThreadToPerformanceCores();
+ system->RegisterHostThread();
+#ifdef _WIN32
+ g_back_thread_id.store(GetCurrentThreadId(), std::memory_order_relaxed);
+#endif
+ u64 read_position = 0;
+ while (!stop_token.stop_requested()) {
+ Nx2Profile::Report("GPU back thread");
+ u64 available = published.load(std::memory_order_acquire);
+ if (available == read_position) {
+ Nx2Profile::Scope scope{Nx2Profile::Zone::SplitWait};
+ // The front usually sends more within microseconds: spin briefly before sleeping.
+ for (u32 spins = 0; spins < 8192 && available == read_position; ++spins) {
+ Pause();
+ available = published.load(std::memory_order_acquire);
+ }
+ if (available == read_position) {
+ // Publish bumps `wake` when it sees back_waiting: read it before the last look
+ // at `published`, so a publish in between ends the wait at once.
+ back_waiting.store(true, std::memory_order_seq_cst);
+ const u32 seen = wake.load(std::memory_order_seq_cst);
+ available = published.load(std::memory_order_seq_cst);
+ if (available == read_position && !stop_token.stop_requested()) {
+ wake.wait(seen, std::memory_order_seq_cst);
+ }
+ back_waiting.store(false, std::memory_order_relaxed);
+ continue;
+ }
+ }
+ u64 last_store = read_position;
+ g_back_busy.store(true, std::memory_order_relaxed);
+ while (read_position != available) {
+ const size_t index = read_position & RingMask;
+ const size_t size = Execute(&ring[index]);
+ read_position += size != 0 ? size : RingWords - index;
+ // Hand ring space back now and then while working through a long batch.
+ if (read_position - last_store >= 4096 && read_position != available) {
+ consumed.store(read_position, std::memory_order_release);
+ last_store = read_position;
+ }
+ }
+ g_back_busy.store(false, std::memory_order_release);
+ consumed.store(read_position, std::memory_order_release);
+ }
+}
+
+} // namespace VideoCommon
diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h
new file mode 100644
index 0000000000..61644b9b80
--- /dev/null
+++ b/src/video_core/gpu_split.h
@@ -0,0 +1,142 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <atomic>
+#include <cstring>
+#include <functional>
+#include <memory>
+#include <thread>
+#include <type_traits>
+
+#include "common/common_types.h"
+
+namespace Core {
+class System;
+}
+
+namespace Tegra::Engines {
+class EngineInterface;
+}
+
+namespace VideoCommon {
+
+// Splits the GPU thread in two (NX2_GPU_THREAD_SPLIT.md). The GPU thread (the front) takes the
+// CPU's commands, parses command lists and runs the puller; engine method calls and everything
+// behind them (register state, macros, draws, the rasterizer and its caches) run in order on a
+// second thread (the back), fed through a single-producer single-consumer ring. Work the front
+// would do on back-owned state is deferred into the same ring; the front waits for the back to
+// catch up (Drain) only where it needs a result from that state, e.g. a semaphore it acquires.
+// Opt-in for Switch 2 titles: EDEN_NX2_SPLIT=1. EDEN_NX2_SPLIT=2 also runs the 3D and compute
+// engines' macros on the front, against copies of the registers they read (Maxwell3D and
+// KeplerCompute Front*), and sends the writes they make.
+class GpuSplit {
+public:
+ static GpuSplit& Get();
+
+ /// Whether the front sends work to the back (front thread only).
+ [[nodiscard]] bool Active() const {
+ return active;
+ }
+ /// Whether the front runs the engines' macros (EDEN_NX2_SPLIT=2).
+ [[nodiscard]] bool FrontMacros() const {
+ return front_macros;
+ }
+
+ /// Starts the back thread once the split is wanted (EDEN_NX2_SPLIT=1, a Switch 2 title,
+ /// commands read without cache flushes). Called by the front between commands.
+ void MaybeStart(Core::System& system);
+ void Stop();
+
+ /// An engine method write, executed as DmaPusher would (method sink, ConsumeSink, call).
+ /// `segment`: the engine's current_dma_segment for the call, when the engine reads it.
+ void Method(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument, bool is_last,
+ const GPUVAddr* segment);
+ void MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words,
+ u32 amount, u32 methods_pending, const GPUVAddr* segment);
+ /// A write made by a macro: CallMethod without the method sink, as the engines' macros do.
+ void MacroWrite(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument);
+
+ /// Runs `func` on the back after everything sent before it. Must be trivially copyable.
+ template <typename Func>
+ void Defer(const Func& func) {
+ static_assert(std::is_trivially_copyable_v<Func> && alignof(Func) <= alignof(u64));
+ constexpr size_t payload = (sizeof(Func) + sizeof(u64) - 1) / sizeof(u64);
+ u64* const entry = Reserve(2 + payload);
+ entry[0] = static_cast<u64>(Kind::Call) | (u64{payload} << 8);
+ entry[1] = reinterpret_cast<u64>(&CallThunk<Func>);
+ std::memcpy(entry + 2, &func, sizeof(Func));
+ Commit(2 + payload);
+ }
+ void DeferFunction(std::function<void()>&& func);
+
+ /// EDEN_NX2_SPLIT_CHECK=1 (debug): reports, with its stack, a thread other than the back
+ /// using back-owned state (the Vulkan scheduler) while the back runs entries.
+ static inline bool check_owner{};
+ static void CheckOwner(const char* where);
+
+ /// Makes what was sent visible to the back.
+ void Publish();
+ /// Waits until the back executed everything sent so far.
+ void Drain();
+
+private:
+ enum class Kind : u64 {
+ Wrap, ///< The rest of the ring is unused: continue at its start
+ Engine, ///< The engine the next methods go to: word 1
+ Method, ///< Bits 8-23 method, 32-63 argument; flags IsLast, HasSegment (word 1)
+ Multi, ///< Bits 8-23 method, 32-63 amount; word 1 methods pending, [segment], words
+ Call, ///< Bits 8-31 payload size in words; word 1 thunk, then the payload
+ };
+ static constexpr u64 IsLast = 1 << 4;
+ static constexpr u64 HasSegment = 1 << 5;
+ static constexpr u64 Direct = 1 << 6; ///< Method: call it even if not executable (macros)
+
+ static constexpr size_t RingWords = size_t{1} << 20; // 8 MiB
+ static constexpr size_t RingMask = RingWords - 1;
+ static constexpr u32 MaxMultiWords = 8192;
+ static constexpr size_t PublishEvery = 256;
+
+ template <typename Func>
+ static void CallThunk(const u64* payload) {
+ (*reinterpret_cast<const Func*>(payload))();
+ }
+
+ u64* Reserve(size_t words);
+ void Commit(size_t words) {
+ write_position += words;
+ if (write_position - published_local >= publish_every) {
+ Publish();
+ }
+ }
+ void WaitForSpace(size_t words);
+ void SelectEngine(Tegra::Engines::EngineInterface* engine);
+
+ void BackLoop(std::stop_token stop_token);
+ size_t Execute(const u64* entry);
+
+ Core::System* system{};
+ bool active{};
+ bool front_macros{};
+ std::unique_ptr<u64[]> ring;
+
+ // Front side.
+ u64 write_position{};
+ u64 published_local{};
+ u64 consumed_cache{};
+ /// Words between publishes: small right after a drain, while the back has nothing to do.
+ size_t publish_every{PublishEvery};
+ Tegra::Engines::EngineInterface* front_engine{};
+
+ // Back side.
+ Tegra::Engines::EngineInterface* back_engine{};
+
+ alignas(64) std::atomic<u64> published{};
+ alignas(64) std::atomic<u64> consumed{};
+ alignas(64) std::atomic<bool> back_waiting{};
+ std::atomic<u32> wake{};
+ std::jthread back_thread;
+};
+
+} // namespace VideoCommon
diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp
index 72da1904e1..55200b0647 100644
--- a/src/video_core/gpu_thread.cpp
+++ b/src/video_core/gpu_thread.cpp
@@ -17,12 +17,28 @@
#include "video_core/dma_pusher.h"
#include "video_core/gpu.h"
#include "video_core/gpu_thread.h"
+#include "video_core/gpu_split.h"
#include "video_core/nx2_profile.h"
#include "video_core/host1x/host1x.h"
#include "video_core/renderer_base.h"
namespace VideoCommon::GPUThread {
+namespace {
+
+// Tells a CPU thread waiting in PushCommand that its command ran.
+void SignalCommand(SynchState& state, u64 fence, bool block) {
+ state.signaled_fence.store(fence);
+ if (block) {
+ // We have to lock the write_lock to ensure that the condition_variable wait not get a
+ // race between the check and the lock itself.
+ std::scoped_lock lk{state.write_lock};
+ state.cv.notify_all();
+ }
+}
+
+} // Anonymous namespace
+
ThreadManager::ThreadManager(Core::System& system_)
: system{system_}
{}
@@ -42,8 +58,10 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten
// When a pending channel last moved or a command arrived (see the polling below).
auto last_progress = std::chrono::steady_clock::now();
namespace Profile = VideoCommon::Nx2Profile;
+ auto& split = VideoCommon::GpuSplit::Get();
while (!stop_token.stop_requested()) {
Profile::Report();
+ split.MaybeStart(system);
if (scheduler.HasPendingChannels()) {
// Suspended channels wait on semaphores the CPU releases: poll them meanwhile.
{
@@ -71,6 +89,34 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten
break;
}
last_progress = std::chrono::steady_clock::now();
+ if (split.Active()) {
+ // Only command lists are parsed here; everything else runs on the back thread
+ // in order, and so does the signal that the command ran.
+ auto* const rasterizer_ptr = renderer.ReadRasterizer();
+ if (auto* submit_list = std::get_if<SubmitListCommand>(&next.data)) {
+ Profile::Scope scope{Profile::Zone::Push};
+ scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries));
+ } else if (std::holds_alternative<GPUTickCommand>(next.data)) {
+ split.Defer([gpu = &system.GPU()] { gpu->TickWork(); });
+ } else if (auto* run = std::get_if<RunCommand>(&next.data)) {
+ split.DeferFunction(std::move(run->func));
+ } else if (const auto* flush = std::get_if<FlushRegionCommand>(&next.data)) {
+ split.Defer([rasterizer_ptr, addr = flush->addr, size = flush->size] {
+ rasterizer_ptr->FlushRegion(addr, size);
+ });
+ } else if (const auto* invalidate = std::get_if<InvalidateRegionCommand>(&next.data)) {
+ split.Defer([rasterizer_ptr, addr = invalidate->addr, size = invalidate->size] {
+ rasterizer_ptr->OnCacheInvalidation(addr, size);
+ });
+ } else {
+ ASSERT(false);
+ }
+ split.Defer([state_ptr = &state, fence = next.fence, block = next.block] {
+ SignalCommand(*state_ptr, fence, block);
+ });
+ split.Publish();
+ continue;
+ }
if (auto* submit_list = std::get_if<SubmitListCommand>(&next.data)) {
Profile::Scope scope{Profile::Zone::Push};
scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries));
@@ -89,13 +135,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten
} else {
ASSERT(false);
}
- state.signaled_fence.store(next.fence);
- if (next.block) {
- // We have to lock the write_lock to ensure that the condition_variable wait not get a
- // race between the check and the lock itself.
- std::scoped_lock lk{state.write_lock};
- state.cv.notify_all();
- }
+ SignalCommand(state, next.fence, next.block);
}
});
}
@@ -161,6 +201,7 @@ void ThreadManager::NotifyShutdown() {
thread.request_stop();
thread.join();
}
+ VideoCommon::GpuSplit::Get().Stop();
}
} // namespace VideoCommon::GPUThread
diff --git a/src/video_core/nx2_profile.h b/src/video_core/nx2_profile.h
index 0c071c85d3..0ff69f9631 100644
--- a/src/video_core/nx2_profile.h
+++ b/src/video_core/nx2_profile.h
@@ -49,6 +49,8 @@ enum class Zone : u32 {
Fence, ///< Fences made (SignalFence)
FenceSkipped, ///< Empty fences not made
Method, ///< Engine method calls (what a split GPU thread's back half would run)
+ SplitDrain, ///< Split GPU thread: the front waits for the back (gpu_split.h)
+ SplitWait, ///< Split GPU thread: the back waits for work
Count,
};
@@ -59,7 +61,7 @@ inline constexpr std::array<const char*, static_cast<size_t>(Zone::Count)> zone_
"sync descriptors", "read handles", "fill image views", "texture buffers", "update buffers",
"descriptor acquire", "bind stage buffers", "push descriptors", "render targets",
"configure draw", "dynamic state", "draw record", "texture upload", "fence",
- "fence skipped", "method",
+ "fence skipped", "method", "split drain", "split wait",
};
inline bool Enabled() {
@@ -141,8 +143,9 @@ inline void Add(Counter counter, u64 amount) {
}
}
-/// Called by the GPU thread between commands: logs and resets the totals every 10 s.
-inline void Report() {
+/// Called by the GPU thread (and the split's back thread) between commands: logs and resets
+/// the totals every 10 s.
+inline void Report(const char* thread_name = "GPU thread") {
if (!Enabled()) {
return;
}
@@ -185,7 +188,7 @@ inline void Report() {
line += fmt::format(" | acquires {:.0f} ({:.0f} waited)",
static_cast<double>(data.counters[6]) * 1e9 / window,
static_cast<double>(data.counters[7]) * 1e9 / window);
- LOG_INFO(HW_GPU, "GPU thread per second:{}", line);
+ LOG_INFO(HW_GPU, "{} per second:{}", thread_name, line);
data = {};
data.window_start = now;
}
diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp
index aa8511a5cb..e1e989c79f 100644
--- a/src/video_core/renderer_vulkan/vk_scheduler.cpp
+++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp
@@ -55,6 +55,9 @@ Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_)
Scheduler::~Scheduler() = default;
u64 Scheduler::Flush(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) {
+ if (VideoCommon::GpuSplit::check_owner) [[unlikely]] {
+ VideoCommon::GpuSplit::CheckOwner("Scheduler::Flush");
+ }
// When flushing, we only send data to the worker thread; no waiting is necessary.
const u64 signal_value = SubmitExecution(signal_semaphore, wait_semaphore);
AllocateNewContext();
@@ -62,6 +65,9 @@ u64 Scheduler::Flush(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) {
}
void Scheduler::Finish(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) {
+ if (VideoCommon::GpuSplit::check_owner) [[unlikely]] {
+ VideoCommon::GpuSplit::CheckOwner("Scheduler::Finish");
+ }
// When finishing, we need to wait for the submission to have executed on the device.
const u64 presubmit_tick = CurrentTick();
SubmitExecution(signal_semaphore, wait_semaphore);
@@ -83,6 +89,9 @@ void Scheduler::WaitWorker() {
}
void Scheduler::DispatchWork() {
+ if (VideoCommon::GpuSplit::check_owner) [[unlikely]] {
+ VideoCommon::GpuSplit::CheckOwner("Scheduler::DispatchWork");
+ }
if (chunk && !chunk->Empty()) {
// Waking the worker is a system call (~2% of the GPU thread at one dispatch per 8
// draws): skip it while the worker is still busy, it pops the queue before waiting.
diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h
index 4456e9ef83..568063ddbd 100644
--- a/src/video_core/renderer_vulkan/vk_scheduler.h
+++ b/src/video_core/renderer_vulkan/vk_scheduler.h
@@ -18,6 +18,7 @@
#include "common/common_types.h"
#include "common/polyfill_thread.h"
#include "common/settings.h"
+#include "video_core/gpu_split.h"
#include "video_core/renderer_vulkan/vk_master_semaphore.h"
#include "video_core/vulkan_common/vulkan_wrapper.h"
@@ -103,6 +104,9 @@ public:
template <typename T>
requires std::is_invocable_v<T, vk::CommandBuffer, vk::CommandBuffer>
void RecordWithUploadBuffer(T&& command) {
+ if (VideoCommon::GpuSplit::check_owner) [[unlikely]] {
+ VideoCommon::GpuSplit::CheckOwner("Scheduler::Record");
+ }
if (chunk->Record(command)) {
return;
}
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-04 03:09:10 +0000