1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#pragma once
#include <atomic>
#include <cstring>
#include <functional>
#include <memory>
#include <thread>
#include <type_traits>
#include "common/common_types.h"
namespace Core {
class System;
}
namespace Tegra::Engines {
class EngineInterface;
}
namespace VideoCommon {
// Splits the GPU thread in two (NX2_GPU_THREAD_SPLIT.md). The GPU thread (the front) takes the
// CPU's commands, parses command lists and runs the puller; engine method calls and everything
// behind them (register state, macros, draws, the rasterizer and its caches) run in order on a
// second thread (the back), fed through a single-producer single-consumer ring. Work the front
// would do on back-owned state is deferred into the same ring; the front waits for the back to
// catch up (Drain) only where it needs a result from that state, e.g. a semaphore it acquires.
// On for Switch 2 titles with the 3D and compute engines' macros on the front, against copies of
// the registers they read (Maxwell3D and KeplerCompute Front*), which sends the writes they make.
// EDEN_NX2_SPLIT=1 keeps the macros on the back; EDEN_NX2_SPLIT=0 runs everything on one thread.
class GpuSplit {
public:
static GpuSplit& Get() {
return instance;
}
/// Whether the front sends work to the back (front thread only).
[[nodiscard]] bool Active() const {
return active;
}
/// Whether the front runs the engines' macros (EDEN_NX2_SPLIT=2).
[[nodiscard]] bool FrontMacros() const {
return front_macros;
}
/// Starts the back thread once the split is wanted (EDEN_NX2_SPLIT=1, a Switch 2 title,
/// commands read without cache flushes). Called by the front between commands.
void MaybeStart(Core::System& system);
void Stop();
/// An engine method write, executed as DmaPusher would (method sink, ConsumeSink, call).
/// `segment`: the engine's current_dma_segment for the call, when the engine reads it.
void Method(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument, bool is_last,
const GPUVAddr* segment) {
SelectEngine(engine);
const size_t words = segment != nullptr ? 2 : 1;
u64* const entry = Reserve(words);
entry[0] = static_cast<u64>(Kind::Method) | (is_last ? IsLast : 0) |
(segment != nullptr ? HasSegment : 0) | (u64{method & 0xffff} << 8) |
(u64{argument} << 32);
if (segment != nullptr) {
entry[1] = *segment;
}
Commit(words);
}
void MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words,
u32 amount, u32 methods_pending, const GPUVAddr* segment);
/// A write made by a macro: CallMethod without the method sink, as the engines' macros do.
void MacroWrite(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument) {
SelectEngine(engine);
u64* const entry = Reserve(1);
entry[0] = static_cast<u64>(Kind::Method) | IsLast | Direct | (u64{method & 0xffff} << 8) |
(u64{argument} << 32);
Commit(1);
}
/// Runs `func` on the back after everything sent before it. Must be trivially copyable.
template <typename Func>
void Defer(const Func& func) {
static_assert(std::is_trivially_copyable_v<Func> && alignof(Func) <= alignof(u64));
constexpr size_t payload = (sizeof(Func) + sizeof(u64) - 1) / sizeof(u64);
u64* const entry = Reserve(2 + payload);
entry[0] = static_cast<u64>(Kind::Call) | (u64{payload} << 8);
entry[1] = reinterpret_cast<u64>(&CallThunk<Func>);
std::memcpy(entry + 2, &func, sizeof(Func));
Commit(2 + payload);
}
void DeferFunction(std::function<void()>&& func);
/// EDEN_NX2_SPLIT_CHECK=1 (debug): reports, with its stack, a thread other than the back
/// using back-owned state (the Vulkan scheduler) while the back runs entries.
static inline bool check_owner{};
static void CheckOwner(const char* where);
/// Makes what was sent visible to the back.
void Publish();
/// Waits until the back executed everything sent so far.
void Drain();
/// Compute or copy work was just sent: what can write GPU-written command lists.
void NoteWriter() {
last_writer_position = write_position;
}
/// Waits until the back executed the compute and copy work sent so far (not what came after).
void WaitForWriters();
private:
enum class Kind : u64 {
Wrap, ///< The rest of the ring is unused: continue at its start
Engine, ///< The engine the next methods go to: word 1
Method, ///< Bits 8-23 method, 32-63 argument; flags IsLast, HasSegment (word 1)
Multi, ///< Bits 8-23 method, 32-63 amount; word 1 methods pending, [segment], words
Call, ///< Bits 8-31 payload size in words; word 1 thunk, then the payload
};
static constexpr u64 IsLast = 1 << 4;
static constexpr u64 HasSegment = 1 << 5;
static constexpr u64 Direct = 1 << 6; ///< Method: call it even if not executable (macros)
static constexpr size_t RingWords = size_t{1} << 20; // 8 MiB
static constexpr size_t RingMask = RingWords - 1;
static constexpr u32 MaxMultiWords = 8192;
static constexpr size_t PublishEvery = 256;
template <typename Func>
static void CallThunk(const u64* payload) {
(*reinterpret_cast<const Func*>(payload))();
}
/// Room for an entry of `words` at the write position (never across the ring's end).
u64* Reserve(size_t words) {
const size_t index = write_position & RingMask;
if (index + words <= RingWords && write_position + words - consumed_cache <= RingWords)
[[likely]] {
return &ring[index];
}
return ReserveSlow(words);
}
u64* ReserveSlow(size_t words);
void Commit(size_t words) {
write_position += words;
if (write_position - published_local >= publish_every) {
Publish();
}
}
void WaitForSpace(size_t words);
void SelectEngine(Tegra::Engines::EngineInterface* engine) {
if (engine != front_engine) [[unlikely]] {
SwitchEngine(engine);
}
}
void SwitchEngine(Tegra::Engines::EngineInterface* engine);
void BackLoop(std::stop_token stop_token);
size_t Execute(const u64* entry);
Core::System* system{};
bool active{};
bool front_macros{};
std::unique_ptr<u64[]> ring;
// Front side.
u64 write_position{};
u64 last_writer_position{};
u64 published_local{};
u64 consumed_cache{};
/// Words between publishes: small right after a drain, while the back has nothing to do.
size_t publish_every{PublishEvery};
Tegra::Engines::EngineInterface* front_engine{};
// Back side.
Tegra::Engines::EngineInterface* back_engine{};
static GpuSplit instance;
alignas(64) std::atomic<u64> published{};
alignas(64) std::atomic<u64> consumed{};
/// A position the front waits for (WaitForWriters): the back stores `consumed` once past it.
alignas(64) std::atomic<u64> wait_target{~u64{0}};
alignas(64) std::atomic<bool> back_waiting{};
std::atomic<u32> wake{};
std::jthread back_thread;
};
} // namespace VideoCommon
|
