video_core: split the GPU thread in two (EDEN_NX2_SPLIT, off by default)

The GPU thread (front) takes the CPU's commands, parses command lists
and runs the puller; engine method calls and everything behind them
(registers, draws, dispatches, the rasterizer and its caches) run in
order on a second thread (back), fed through a single-producer
single-consumer ring (gpu_split.h). Work the front did on back-owned
state (end of command list flushes, channel binds, semaphore releases,
syncpoint increments, flush/invalidate requests, presentation) is
deferred into the same ring; the front waits for the back only where it
needs a result from that state: semaphore acquires and MME DMA reads.

EDEN_NX2_SPLIT=1 runs macros on the back. EDEN_NX2_SPLIT=2 runs the 3D
and compute macros on the front against copies of the registers they
read (Maxwell3D/KeplerCompute Front*), drops state writes that keep a
register's value there, and sends the rest. EDEN_NX2_SPLIT_VERIFY=1
compares the copies with the back's registers every 4096 macro calls
(0 differences in 40,193 checks in title 1); EDEN_NX2_SPLIT_CHECK=1
logs the stack of any other thread using the Vulkan scheduler while the
back runs. Switch 2 titles, Vulkan, non-accurate GPU only.

Title 1, 3D attract loop: ~44,000 draws a second on one thread, ~47,400
with EDEN_NX2_SPLIT=1, 50,000-66,500 with EDEN_NX2_SPLIT=2.

1 files changed, 425 insertions, 0 deletions

diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp
index 997ef474f5..10527c2473 100644
--- a/src/video_core/engines/maxwell_3d.cpp
+++ b/src/video_core/engines/maxwell_3d.cpp
@@ -19,10 +19,14 @@
 #include "video_core/engines/maxwell_3d.h"
 #include "video_core/engines/maxwell_3d.h"
 #include "video_core/gpu.h"
+#include "video_core/gpu_split.h"
 #include "video_core/memory_manager.h"
 #include "video_core/nx2_profile.h"
 #include "video_core/rasterizer_interface.h"
 #include "video_core/textures/texture.h"
+#ifdef ARCHITECTURE_x86_64
+#include "video_core/macro_turing_jit.h"
+#endif
 
 namespace Tegra::Engines {
 
@@ -347,11 +351,18 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume
     case MAXWELL3D_REG_INDEX(shadow_ram_control):
         shadow_state.shadow_ram_control = static_cast<Regs::ShadowRamControl>(nonshadow_argument);
         return;
+    // With macros on the split GPU thread's front (FrontWrite), the front loads and binds them.
     case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+        if (front_macros) return;
         return GetNativeMacroEngine().ClearCode(regs.load_mme.instruction_ptr);
     case MAXWELL3D_REG_INDEX(load_mme.instruction):
+        if (front_macros) return;
         return GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr, argument);
     case MAXWELL3D_REG_INDEX(load_mme.start_address):
+        if (front_macros) {
+            ++regs.load_mme.start_address_ptr;
+            return;
+        }
         return ProcessMacroBind(argument);
     case MAXWELL3D_REG_INDEX(falcon[4]):
         return ProcessFirmwareCall4();
@@ -460,6 +471,10 @@ void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument
             VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::SameValueWrites, 1);
         }
     }
+    if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && front_macros) [[unlikely]] {
+        back_control_writes[back_control_count++ % back_control_writes.size()] = {
+            method_argument, static_cast<u32>(shadow_state.shadow_ram_control), is_last_call ? 1U : 0U};
+    }
     const u32 argument = ProcessShadowRam(method, method_argument);
     // A state register written with the value it holds changes nothing: no dirty flags (they
     // would make the next draw re-check that state), no method processing. Title 1 makes ~6
@@ -767,4 +782,414 @@ void Maxwell3D::SetHLEReplacementAttributeType(u32 bank, u32 offset,
     replace_table.emplace(key, name);
 }
 
+// Split GPU thread with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2). The front keeps
+// what macros read: the registers as the back thread will hold them when it reaches the same
+// point of the method stream. That is every write the front sends, with the shadow RAM applied,
+// plus the few changes the back makes on its own (constant buffer offset, shadow_scratch[0] on
+// falcon[4], the macro bind pointer); the front loads and binds the macro code itself.
+struct Maxwell3D::FrontState {
+    std::array<ControlWrite, 16> control_writes{};
+    u32 control_count{};
+    std::array<u32, Regs::NUM_REGS> regs{};
+    std::array<u32, Regs::NUM_REGS> shadow{};
+    Regs::ShadowRamControl control{};
+    u32 executing_macro{};
+    std::vector<u32> parameters;
+    std::vector<u32> fifo;
+    u64 calls{};
+    /// A method the draw manager handles was sent since the back last ran DrawDeferred: only
+    /// then can it have a deferred instanced draw to issue at the end of a macro.
+    bool draw_pending{true};
+};
+
+namespace {
+
+// EDEN_NX2_SPLIT_VERIFY=1: every 4096 front macro calls, wait for the back and compare the
+// front's registers with the back's (logs and resynchronizes differences).
+bool FrontVerifyEnabled() {
+    static const bool verify = std::getenv("EDEN_NX2_SPLIT_VERIFY") != nullptr;
+    return verify;
+}
+
+// Executable methods ProcessMethodCall handles itself; the rest go to the draw manager.
+bool FrontHandledOutsideDrawManager(u32 method) {
+    constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer);
+    if (method >= cb_data && method < cb_data + 16) {
+        return true;
+    }
+    switch (method) {
+    case MAXWELL3D_REG_INDEX(wait_for_idle):
+    case MAXWELL3D_REG_INDEX(shadow_ram_control):
+    case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+    case MAXWELL3D_REG_INDEX(load_mme.instruction):
+    case MAXWELL3D_REG_INDEX(load_mme.start_address):
+    case MAXWELL3D_REG_INDEX(falcon[4]):
+    case MAXWELL3D_REG_INDEX(bind_groups[0].raw_config):
+    case MAXWELL3D_REG_INDEX(bind_groups[1].raw_config):
+    case MAXWELL3D_REG_INDEX(bind_groups[2].raw_config):
+    case MAXWELL3D_REG_INDEX(bind_groups[3].raw_config):
+    case MAXWELL3D_REG_INDEX(bind_groups[4].raw_config):
+    case MAXWELL3D_REG_INDEX(report_semaphore.query):
+    case MAXWELL3D_REG_INDEX(render_enable.mode):
+    case MAXWELL3D_REG_INDEX(clear_report_value):
+    case MAXWELL3D_REG_INDEX(sync_info):
+    case MAXWELL3D_REG_INDEX(launch_dma):
+    case MAXWELL3D_REG_INDEX(inline_data):
+    case MAXWELL3D_REG_INDEX(fragment_barrier):
+    case MAXWELL3D_REG_INDEX(invalidate_texture_data_cache):
+    case MAXWELL3D_REG_INDEX(tiled_cache_barrier):
+        return true;
+    default:
+        return false;
+    }
+}
+
+// EDEN_NX2_SPLIT_SKIP=0: send state writes that keep the register's value too.
+bool FrontSkipSame() {
+    static const bool skip = [] {
+        const char* value = std::getenv("EDEN_NX2_SPLIT_SKIP");
+        return value == nullptr || value[0] != '0';
+    }();
+    return skip;
+}
+
+} // Anonymous namespace
+
+Maxwell3D::FrontState& Maxwell3D::Front(Core::System& system, VideoCommon::GpuSplit& split) {
+    if (!front) [[unlikely]] {
+        // Copy the back's state and take the macros over while it is idle (state writes it
+        // still holds in its method sink first).
+        split.Drain();
+        ConsumeSink(system);
+        front = std::make_unique<FrontState>();
+        std::memcpy(front->regs.data(), regs.reg_array.data(), sizeof(front->regs));
+        std::memcpy(front->shadow.data(), shadow_state.reg_array.data(), sizeof(front->shadow));
+        front->control = shadow_state.shadow_ram_control;
+        front->executing_macro = executing_macro;
+        front->parameters = std::move(macro_params);
+        macro_params.clear();
+        executing_macro = 0;
+        front_macros = true;
+    }
+    return *front;
+}
+
+void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 method,
+                           u32 argument, bool is_last_call, bool from_macro) {
+    namespace Profile = VideoCommon::Nx2Profile;
+    // As ProcessShadowRam does on the back.
+    u32 value = argument;
+    bool shadow_kept = true;
+    if (fs.control == Regs::ShadowRamControl::Track ||
+        fs.control == Regs::ShadowRamControl::TrackWithFilter) {
+        shadow_kept = fs.shadow[method] == argument;
+        fs.shadow[method] = argument;
+    } else if (fs.control == Regs::ShadowRamControl::Replay) {
+        value = fs.shadow[method];
+    }
+    if (Profile::Enabled()) [[unlikely]] {
+        Profile::Add(Profile::Counter::MethodWrites, 1);
+    }
+    if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && FrontVerifyEnabled()) [[unlikely]] {
+        fs.control_writes[fs.control_count++ % fs.control_writes.size()] = {
+            argument, static_cast<u32>(fs.control), from_macro ? 2U : 1U};
+    }
+    if (!execution_mask[method]) {
+        // Nothing would change on the back: CallMethod and ConsumeSinkImpl skip same-value state
+        // writes, and the shadow RAM (tracked by default) holds the value already.
+        if (fs.regs[method] == value && shadow_kept && SkipSameValueWrites() && FrontSkipSame()) {
+            if (Profile::Enabled()) [[unlikely]] {
+                Profile::Add(Profile::Counter::SameValueWrites, 1);
+            }
+            return;
+        }
+        fs.regs[method] = value;
+    } else {
+        fs.regs[method] = value;
+        constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer);
+        switch (method) {
+        case MAXWELL3D_REG_INDEX(shadow_ram_control):
+            // The back's mode is shadow_state.shadow_ram_control: the shadow copy of this very
+            // register, which a later Replay write reads back.
+            fs.control = static_cast<Regs::ShadowRamControl>(argument);
+            fs.shadow[method] = argument;
+            break;
+        case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+            GetNativeMacroEngine().ClearCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)]);
+            break;
+        case MAXWELL3D_REG_INDEX(load_mme.instruction):
+            GetNativeMacroEngine().AddCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)], value);
+            break;
+        case MAXWELL3D_REG_INDEX(load_mme.start_address): {
+            const auto positions = GetNativeMacroPositions();
+            u32& pointer = fs.regs[MAXWELL3D_REG_INDEX(load_mme.start_address_ptr)];
+            if (pointer < positions.size()) {
+                positions[pointer] = value;
+            }
+            ++pointer;
+            break;
+        }
+        case MAXWELL3D_REG_INDEX(falcon[4]):
+            fs.regs[MAXWELL3D_REG_INDEX(shadow_scratch)] = 1;
+            break;
+        default:
+            if (method >= cb_data && method < cb_data + 16) {
+                fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)] += sizeof(u32);
+            } else if (!FrontHandledOutsideDrawManager(method)) {
+                fs.draw_pending = true;
+            }
+            break;
+        }
+    }
+    if (from_macro) {
+        split.MacroWrite(this, method, argument);
+    } else {
+        split.Method(this, method, argument, is_last_call, nullptr);
+    }
+}
+
+void Maxwell3D::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method,
+                                u32 argument, bool is_last_call) {
+    auto& fs = Front(system, split);
+    if (method >= MacroRegistersStart) {
+        if (fs.executing_macro == 0) {
+            fs.executing_macro = method;
+        }
+        fs.parameters.push_back(argument);
+        if (is_last_call) {
+            FrontRunMacro(system, split, fs);
+        }
+        return;
+    }
+    FrontWrite(split, fs, method, argument, is_last_call, false);
+}
+
+void Maxwell3D::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split,
+                                     u32 method, const u32* base_start, u32 amount,
+                                     u32 methods_pending) {
+    auto& fs = Front(system, split);
+    if (method >= MacroRegistersStart) {
+        if (fs.executing_macro == 0) {
+            fs.executing_macro = method;
+        }
+        fs.parameters.insert(fs.parameters.end(), base_start, base_start + amount);
+        if (amount == methods_pending) {
+            FrontRunMacro(system, split, fs);
+        }
+        return;
+    }
+    // What CallMultiMethod does to the registers on the back, then one entry for all the words.
+    constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer);
+    const auto tracked = [&fs] {
+        return fs.control == Regs::ShadowRamControl::Track ||
+               fs.control == Regs::ShadowRamControl::TrackWithFilter;
+    };
+    if (method >= cb_data && method < cb_data + 16) {
+        fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)] += amount * sizeof(u32);
+    } else if (method == MAXWELL3D_REG_INDEX(inline_data)) {
+        // Upload data only.
+    } else if ((method == MAXWELL3D_REG_INDEX(draw_inline_index) ||
+                method == MAXWELL3D_REG_INDEX(inline_index_2x16.even) ||
+                method == MAXWELL3D_REG_INDEX(inline_index_4x8.index0)) &&
+               fs.control != Regs::ShadowRamControl::Replay) {
+        if (amount != 0) {
+            const u32 last = base_start[amount - 1];
+            if (tracked()) {
+                fs.shadow[method] = last;
+            }
+            fs.regs[method] = last;
+        }
+        fs.draw_pending = true;
+    } else {
+        // CallMethod for each word, macro uploads among them (load_mme.instruction). A word may
+        // change the shadow RAM mode for the next (a run of SET_SHADOW_RAM_CONTROL writes).
+        for (u32 i = 0; i < amount; ++i) {
+            const u32 argument = base_start[i];
+            u32 value = argument;
+            if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && FrontVerifyEnabled()) [[unlikely]] {
+                fs.control_writes[fs.control_count++ % fs.control_writes.size()] = {
+                    argument, static_cast<u32>(fs.control), 3U};
+            }
+            if (tracked()) {
+                fs.shadow[method] = argument;
+            } else if (fs.control == Regs::ShadowRamControl::Replay) {
+                value = fs.shadow[method];
+            }
+            fs.regs[method] = value;
+            if (!execution_mask[method]) {
+                continue;
+            }
+            switch (method) {
+            case MAXWELL3D_REG_INDEX(shadow_ram_control):
+                fs.control = static_cast<Regs::ShadowRamControl>(argument);
+                fs.shadow[method] = argument; // The same word on the back (see FrontWrite)
+                break;
+            case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+                GetNativeMacroEngine().ClearCode(value);
+                break;
+            case MAXWELL3D_REG_INDEX(load_mme.instruction):
+                GetNativeMacroEngine().AddCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)], value);
+                break;
+            case MAXWELL3D_REG_INDEX(load_mme.start_address): {
+                const auto positions = GetNativeMacroPositions();
+                u32& pointer = fs.regs[MAXWELL3D_REG_INDEX(load_mme.start_address_ptr)];
+                if (pointer < positions.size()) {
+                    positions[pointer] = value;
+                }
+                ++pointer;
+                break;
+            }
+            case MAXWELL3D_REG_INDEX(falcon[4]):
+                fs.regs[MAXWELL3D_REG_INDEX(shadow_scratch)] = 1;
+                break;
+            default:
+                if (!FrontHandledOutsideDrawManager(method)) {
+                    fs.draw_pending = true;
+                }
+                break;
+            }
+        }
+    }
+    split.MultiMethod(this, method, base_start, amount, methods_pending, nullptr);
+}
+
+void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split,
+                              FrontState& fs) {
+#ifdef ARCHITECTURE_x86_64
+    const u32 method = fs.executing_macro;
+    fs.executing_macro = 0;
+    auto& engine = GetNativeMacroEngine();
+    const auto positions = GetNativeMacroPositions();
+    const u32 start = positions[((method - MacroRegistersStart) >> 1) % positions.size()];
+    {
+        VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro};
+        engine.native_interpreter.active_dram = &native_data_ram;
+        if (!engine.native_jit) {
+            engine.native_jit = std::make_shared<TuringMacroJit>();
+        }
+        engine.native_jit->Prepare(engine.native_code, engine.native_code_generation);
+        fs.fifo.assign(fs.parameters.begin(), fs.parameters.end());
+        const auto result = engine.native_jit->Execute(
+            start, fs.fifo, std::span<u32>{native_data_ram},
+            [&fs](u32 address) { return address < Regs::NUM_REGS ? fs.regs[address] : 0U; },
+            [&](u32 address, u32 value) {
+                if (address >= MacroRegistersStart) {
+                    return false;
+                }
+                FrontWrite(split, fs, address, value, true, true);
+                // MME_DMA_READ (into data RAM) and MME_DMA_READ_FIFOED (into the load FIFO).
+                if (address == 0x157 || address == 0x158) {
+                    FrontDmaRead(split, fs, address, value);
+                }
+                return true;
+            });
+        if (!result.success) {
+            engine.native_execution_failed = true;
+            LOG_ERROR(HW_GPU, "Native MME (JIT, split front) failed: start={:#x}, consumed={}/{}, error={}",
+                      start, result.consumed, fs.parameters.size(), result.error);
+        }
+    }
+    fs.parameters.clear();
+    if (fs.draw_pending) {
+        fs.draw_pending = false;
+        split.Defer([this, system_ptr = &system] { FinishFrontMacro(*system_ptr); });
+    }
+    if (FrontVerifyEnabled() && (++fs.calls & 4095) == 0) [[unlikely]] {
+        FrontVerify(split, fs);
+    }
+#else
+    fs.executing_macro = 0;
+    fs.parameters.clear();
+#endif
+}
+
+void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 method,
+                             u32 count) {
+    // The words may be written by methods still queued for the back (indirect draw arguments
+    // from a culling dispatch), and the caches that may hold newer ones belong to it.
+    split.Drain();
+    const GPUVAddr source = (GPUVAddr{fs.regs[0x154] & 0x1ffffff} << 32) | fs.regs[0x155];
+    const u32 ram = fs.regs[0x156];
+    const u32 words = std::min<u32>(count, 0x1000);
+    const size_t read_bytes = size_t{words} * sizeof(u32);
+    if (VideoCommon::Nx2Profile::Enabled()) {
+        VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::DmaReadBytes, read_bytes);
+    }
+    if (memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) {
+        memory_manager.FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache);
+    }
+    for (u32 i = 0; i < words; ++i) {
+        const u32 word = memory_manager.Read<u32>(source + u64{i} * sizeof(u32));
+        if (method == 0x158) {
+            fs.fifo.push_back(word);
+        } else if (ram + i < native_data_ram.size()) {
+            native_data_ram[ram + i] = word;
+        }
+    }
+}
+
+void Maxwell3D::FrontVerify(VideoCommon::GpuSplit& split, FrontState& fs) {
+    split.Drain();
+    static u64 checks{};
+    static u64 differences{};
+    ++checks;
+    // State writes the back still holds in its method sink count as done.
+    std::array<u32, Regs::NUM_REGS> expected;
+    std::memcpy(expected.data(), regs.reg_array.data(), sizeof(expected));
+    for (const auto& [method, value] : method_sink) {
+        expected[method] = shadow_state.shadow_ram_control == Regs::ShadowRamControl::Replay
+                               ? shadow_state.reg_array[method]
+                               : value;
+    }
+    static u64 control_differences{};
+    if ((fs.control != shadow_state.shadow_ram_control ||
+         fs.shadow[MAXWELL3D_REG_INDEX(shadow_ram_control)] != shadow_state.reg_array[MAXWELL3D_REG_INDEX(shadow_ram_control)] ||
+         fs.regs[MAXWELL3D_REG_INDEX(shadow_ram_control)] != expected[MAXWELL3D_REG_INDEX(shadow_ram_control)]) &&
+        control_differences < 4) {
+        LOG_WARNING(HW_GPU, "Split shadow RAM control writes, front ({} total) / back ({} total):",
+                    fs.control_count, back_control_count);
+        for (u32 i = 0; i < 16; ++i) {
+            const auto& f = fs.control_writes[(fs.control_count + i) % 16];
+            const auto& b = back_control_writes[(back_control_count + i) % 16];
+            LOG_WARNING(HW_GPU, "  front {:#x} (mode {}, path {}) | back {:#x} (mode {}, last {})",
+                        f.argument, f.before, f.path, b.argument, b.before, b.path);
+        }
+    }
+    for (u32 i = 0; i < Regs::NUM_REGS; ++i) {
+        if (fs.regs[i] != expected[i]) {
+            if (differences++ < 64) {
+                LOG_WARNING(HW_GPU, "Split front register {:#x}: front {:#x}, back {:#x}", i,
+                            fs.regs[i], expected[i]);
+            }
+            fs.regs[i] = expected[i];
+        }
+    }
+    if (fs.control != shadow_state.shadow_ram_control) {
+        if (control_differences++ < 16) {
+            LOG_WARNING(HW_GPU, "Split front shadow RAM control {} vs back {}",
+                        static_cast<u32>(fs.control),
+                        static_cast<u32>(shadow_state.shadow_ram_control));
+        }
+        fs.control = shadow_state.shadow_ram_control;
+    }
+    for (u32 i = 0; i < Regs::NUM_REGS; ++i) {
+        if (fs.shadow[i] != shadow_state.reg_array[i]) {
+            if (control_differences++ < 16) {
+                LOG_WARNING(HW_GPU, "Split front shadow register {:#x}: front {:#x}, back {:#x}", i,
+                            fs.shadow[i], shadow_state.reg_array[i]);
+            }
+            fs.shadow[i] = shadow_state.reg_array[i];
+        }
+    }
+    if ((checks & 63) == 1) {
+        LOG_INFO(HW_GPU, "Split front register checks: {} done, {} differences", checks, differences);
+    }
+}
+
+void Maxwell3D::FinishFrontMacro(Core::System& system) {
+    // What ProcessMacro and CallMacroMethod do around the macro.
+    ConsumeSink(system);
+    draw_manager.DrawDeferred(*this);
+}
+
 } // namespace Tegra::Engines