aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorDjihads <76681596+Djihads80@users.noreply.github.com>2026-10-30 11:33:03 +0100
committerGitHub <noreply@github.com>2026-10-30 11:33:03 +0100
commit9560dec549134cf50a7bcde3f86f87ed7658b25f (patch)
tree75493cfac29ed9d1dc350658e17c366f7e10a3c4 /src
parent666d2447639594c7cd2b184acf2117b38766ac22 (diff)
parent937bb4dd30401a339a062386c121b9714e33e38d (diff)
Merge pull request #26 from Djihads80/nx2-experimental
Nx2 experimental
Diffstat (limited to 'src')
-rw-r--r--src/common/container/unordered_map.h2
-rw-r--r--src/common/container/unordered_set.h2
-rw-r--r--src/common/container_hash.h22
-rw-r--r--src/common/cpu_features.cpp31
-rw-r--r--src/common/cpu_features.h22
-rw-r--r--src/common/range_sets.h3
-rw-r--r--src/common/range_sets.inc45
-rw-r--r--src/core/arm/dynarmic/arm_dynarmic_64.cpp29
-rw-r--r--src/core/arm/dynarmic/arm_dynarmic_64.h2
-rw-r--r--src/core/arm/nce/patcher.cpp3
-rw-r--r--src/core/core.cpp25
-rw-r--r--src/core/core_timing.cpp2
-rw-r--r--src/core/device_memory_manager.h24
-rw-r--r--src/core/device_memory_manager.inc26
-rw-r--r--src/core/hle/service/nx2_gsv.h43
-rw-r--r--src/core/hle/service/psc/time/common.h11
-rw-r--r--src/core/hle/service/sm/sm.cpp43
-rw-r--r--src/qt_common/game_list/worker.cpp21
-rw-r--r--src/shader_recompiler/frontend/ampere/lower_to_maxwell.h32
-rw-r--r--src/shader_recompiler/ir_opt/constant_propagation_pass.cpp32
-rw-r--r--src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp2
-rw-r--r--src/video_core/CMakeLists.txt2
-rw-r--r--src/video_core/buffer_cache/buffer_cache.h125
-rw-r--r--src/video_core/buffer_cache/buffer_cache_base.h25
-rw-r--r--src/video_core/control/scheduler.cpp29
-rw-r--r--src/video_core/control/scheduler.h2
-rw-r--r--src/video_core/dma_pusher.cpp86
-rw-r--r--src/video_core/dma_pusher.h1
-rw-r--r--src/video_core/engines/kepler_compute.cpp229
-rw-r--r--src/video_core/engines/kepler_compute.h28
-rw-r--r--src/video_core/engines/maxwell_3d.cpp486
-rw-r--r--src/video_core/engines/maxwell_3d.h47
-rw-r--r--src/video_core/engines/puller.cpp61
-rw-r--r--src/video_core/fence_manager.h151
-rw-r--r--src/video_core/gpu.cpp7
-rw-r--r--src/video_core/gpu_split.cpp350
-rw-r--r--src/video_core/gpu_split.h177
-rw-r--r--src/video_core/gpu_thread.cpp55
-rw-r--r--src/video_core/macro.cpp50
-rw-r--r--src/video_core/macro.h6
-rw-r--r--src/video_core/macro_memo.h30
-rw-r--r--src/video_core/macro_turing_jit.h662
-rw-r--r--src/video_core/memory_manager.h6
-rw-r--r--src/video_core/nx2_profile.h60
-rw-r--r--src/video_core/query_cache/query_cache.h10
-rw-r--r--src/video_core/query_cache/query_cache_base.h3
-rw-r--r--src/video_core/renderer_opengl/gl_graphics_pipeline.cpp4
-rw-r--r--src/video_core/renderer_vulkan/fixed_pipeline_state.cpp9
-rw-r--r--src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp4
-rw-r--r--src/video_core/renderer_vulkan/vk_pipeline_cache.cpp56
-rw-r--r--src/video_core/renderer_vulkan/vk_present_manager.cpp2
-rw-r--r--src/video_core/renderer_vulkan/vk_rasterizer.cpp25
-rw-r--r--src/video_core/renderer_vulkan/vk_scheduler.cpp23
-rw-r--r--src/video_core/renderer_vulkan/vk_scheduler.h27
-rw-r--r--src/video_core/renderer_vulkan/vk_swapchain.cpp16
-rw-r--r--src/video_core/renderer_vulkan/vk_swapchain.h3
-rw-r--r--src/video_core/shader_environment.cpp2
-rw-r--r--src/video_core/texture_cache/image_base.cpp4
-rw-r--r--src/video_core/texture_cache/texture_cache.h20
-rw-r--r--src/video_core/texture_cache/texture_cache_base.h1
-rw-r--r--src/video_core/vulkan_common/vulkan_memory_allocator.cpp2
-rw-r--r--src/yuzu/CMakeLists.txt17
-rw-r--r--src/yuzu_cmd/CMakeLists.txt10
-rw-r--r--src/yuzu_cmd/nx2_sampler.cpp4
-rw-r--r--src/yuzu_cmd/yuzu.cpp2
65 files changed, 3123 insertions, 218 deletions
diff --git a/src/common/container/unordered_map.h b/src/common/container/unordered_map.h
index 7d9e49fd80..c4d4a72663 100644
--- a/src/common/container/unordered_map.h
+++ b/src/common/container/unordered_map.h
@@ -9,7 +9,7 @@
namespace Common {
-template <class Key, class T, class Hash = std::hash<Key>, class Pred = std::equal_to<Key>,
+template <class Key, class T, class Hash = FastHash<Key>, class Pred = std::equal_to<Key>,
class Allocator = std::allocator<std::pair<const Key, T>>>
using unordered_map = boost::unordered::unordered_flat_map<Key, T, Hash, Pred, Allocator>;
diff --git a/src/common/container/unordered_set.h b/src/common/container/unordered_set.h
index 54d71406a1..690c33286f 100644
--- a/src/common/container/unordered_set.h
+++ b/src/common/container/unordered_set.h
@@ -9,7 +9,7 @@
namespace Common {
-template <class Key, class Hash = std::hash<Key>, class Pred = std::equal_to<Key>,
+template <class Key, class Hash = FastHash<Key>, class Pred = std::equal_to<Key>,
class Allocator = std::allocator<Key>>
using unordered_set = boost::unordered::unordered_flat_set<Key, Hash, Pred, Allocator>;
diff --git a/src/common/container_hash.h b/src/common/container_hash.h
index d7477f7e4f..f06f69ad8b 100644
--- a/src/common/container_hash.h
+++ b/src/common/container_hash.h
@@ -19,6 +19,28 @@
namespace Common {
+/// The default hash of Common::unordered_map/set. std::hash of an integer or a pointer is a
+/// byte-wise FNV-1a on MSVC, and boost's flat containers mix a hash that does not say it
+/// avalanches anyway: pass those keys through (the GPU thread's page maps looked keys up
+/// tens of thousands of times a second with it). Other keys keep std::hash.
+template <class Key>
+struct FastHash : std::hash<Key> {};
+
+template <class Key>
+ requires std::is_integral_v<Key>
+struct FastHash<Key> {
+ std::size_t operator()(Key key) const noexcept {
+ return static_cast<std::size_t>(key);
+ }
+};
+
+template <class T>
+struct FastHash<T*> {
+ std::size_t operator()(T* pointer) const noexcept {
+ return reinterpret_cast<std::size_t>(pointer);
+ }
+};
+
namespace detail {
template <typename T>
diff --git a/src/common/cpu_features.cpp b/src/common/cpu_features.cpp
index 52fd003c44..93eecb1dc5 100644
--- a/src/common/cpu_features.cpp
+++ b/src/common/cpu_features.cpp
@@ -278,8 +278,16 @@ std::chrono::milliseconds WallClock::GetTimeMS() const {
s64 WallClock::GetCNTPCT() const {
if (!invariant)
- return GetUptime() * NsToCNTPCTRatio::num / NsToCNTPCTRatio::den;
- return MultiplyHigh(GetUptime(), cntpct_rdtsc_factor);
+ return MultiplyAndDivide64(GetUptime(), GetGuestCNTFRQ(), NsRatio::den);
+ return MultiplyHigh(GetUptime(), cntpct_rdtsc_factor.load(std::memory_order_relaxed));
+}
+
+void WallClock::SetGuestCNTFRQ(u64 frequency) {
+ guest_cntfrq.store(frequency, std::memory_order_relaxed);
+ if (invariant) {
+ cntpct_rdtsc_factor.store(GetFixedPoint64Factor(frequency, rdtsc_frequency),
+ std::memory_order_relaxed);
+ }
}
s64 WallClock::GetGPUTick() const {
@@ -336,7 +344,7 @@ namespace {
} // namespace
WallClock::WallClock(bool invariant_, u64 rdtsc_frequency_) noexcept {
- const u64 host_cntfrq = std::max<u64>(GetHostCNTFRQ(), 1);
+ host_cntfrq = std::max<u64>(GetHostCNTFRQ(), 1);
ns_cntfrq_factor = GetFixedPointFactor(NsRatio::den, host_cntfrq);
us_cntfrq_factor = GetFixedPointFactor(UsRatio::den, host_cntfrq);
ms_cntfrq_factor = GetFixedPointFactor(MsRatio::den, host_cntfrq);
@@ -361,6 +369,11 @@ s64 WallClock::GetCNTPCT() const {
return MultiplyHigh(GetUptime(), guest_cntfrq_factor);
}
+void WallClock::SetGuestCNTFRQ(u64 frequency) {
+ guest_cntfrq.store(frequency, std::memory_order_relaxed);
+ guest_cntfrq_factor = GetFixedPointFactor(frequency, host_cntfrq);
+}
+
s64 WallClock::GetGPUTick() const {
return MultiplyHigh(GetUptime(), gputick_cntfrq_factor);
}
@@ -399,7 +412,11 @@ std::chrono::milliseconds WallClock::GetTimeMS() const {
}
s64 WallClock::GetCNTPCT() const {
- return GetUptime() * NsToCNTPCTRatio::num / NsToCNTPCTRatio::den;
+ return MultiplyAndDivide64(GetUptime(), GetGuestCNTFRQ(), NsRatio::den);
+}
+
+void WallClock::SetGuestCNTFRQ(u64 frequency) {
+ guest_cntfrq.store(frequency, std::memory_order_relaxed);
}
s64 WallClock::GetGPUTick() const {
@@ -419,9 +436,13 @@ u64 WallClock::NsToTicks(std::chrono::nanoseconds ns) const {
}
#endif
+u64 WallClock::CPUTickToGuestCNTPCT(u64 cpu_tick) const {
+ return MultiplyAndDivide64(cpu_tick, GetGuestCNTFRQ(), CPUTickFreq);
+}
+
// Wall clock MUST be initialized AFTER g_cpu_caps
// C++ only guarantees ctor init in the order they appear in TU
-const WallClock g_wall_clock = [] {
+WallClock g_wall_clock = [] {
#if defined(ARCHITECTURE_x86_64)
auto const& caps = Common::g_cpu_caps;
return WallClock(caps.invariant_tsc && caps.tsc_frequency > std::nano::den, caps.tsc_frequency);
diff --git a/src/common/cpu_features.h b/src/common/cpu_features.h
index 61aa2767ab..31be2dc956 100644
--- a/src/common/cpu_features.h
+++ b/src/common/cpu_features.h
@@ -7,6 +7,7 @@
#pragma once
+#include <atomic>
#include <optional>
#include <string_view>
#include <chrono>
@@ -20,11 +21,24 @@ namespace Common {
class WallClock {
public:
static constexpr u64 CNTFRQ = 19'200'000; // CNTPCT_EL0 Frequency = 19.2 MHz
+ static constexpr u64 Switch2CNTFRQ = 31'250'000; // Switch 2 CNTPCT_EL0 Frequency = 31.25 MHz
static constexpr u64 GPUTickFreq = 614'400'000; // GM20B GPU Tick Frequency = 614.4 MHz
static constexpr u64 CPUTickFreq = 1'020'000'000; // T210/4 A57 CPU Tick Frequency = 1020.0 MHz
explicit WallClock(bool invariant, u64 rdtsc_frequency_) noexcept;
+ /// Sets the guest counter's frequency: CNTFRQ, or Switch2CNTFRQ for Switch 2 titles (their SDK
+ /// converts ticks with a constant 31.25 MHz). Called at boot, before guest code runs.
+ void SetGuestCNTFRQ(u64 frequency);
+
+ /// @returns The guest counter's frequency (CNTFRQ_EL0).
+ [[nodiscard]] u64 GetGuestCNTFRQ() const {
+ return guest_cntfrq.load(std::memory_order_relaxed);
+ }
+
+ /// @returns Emulated CPU ticks (single core timing) as guest CNTPCT ticks.
+ [[nodiscard]] u64 CPUTickToGuestCNTPCT(u64 cpu_tick) const;
+
/// @returns The time in nanoseconds since the construction of this clock.
std::chrono::nanoseconds GetTimeNS() const;
@@ -34,7 +48,7 @@ public:
/// @returns The time in milliseconds since the construction of this clock.
std::chrono::milliseconds GetTimeMS() const;
- /// @returns The guest CNTPCT ticks since the construction of this clock.
+ /// @returns The guest CNTPCT ticks (at GetGuestCNTFRQ()) since the construction of this clock.
s64 GetCNTPCT() const;
/// @returns The guest GPU ticks since the construction of this clock.
@@ -91,6 +105,7 @@ public:
using CPUTickToCNTPCTRatio = std::ratio<CNTFRQ, CPUTickFreq>;
using CPUTickToGPUTickRatio = std::ratio<GPUTickFreq, CPUTickFreq>;
+ std::atomic<u64> guest_cntfrq{CNTFRQ};
#if defined(ARCHITECTURE_x86_64)
u64 rdtsc_frequency;
u64 ns_rdtsc_factor;
@@ -98,7 +113,7 @@ public:
u64 ms_rdtsc_factor;
u64 rdtsc_ns_integer;
u64 rdtsc_ns_factor;
- u64 cntpct_rdtsc_factor;
+ std::atomic<u64> cntpct_rdtsc_factor;
u64 gputick_rdtsc_factor;
bool invariant;
#elif defined(HAS_NCE)
@@ -106,6 +121,7 @@ public:
[[nodiscard]] inline FactorType GetGuestCNTFRQFactor() const {
return guest_cntfrq_factor;
}
+ u64 host_cntfrq;
FactorType ns_cntfrq_factor;
FactorType us_cntfrq_factor;
FactorType ms_cntfrq_factor;
@@ -183,6 +199,6 @@ std::optional<int> GetProcessorCount();
/// @brief Global cpu caps
extern const CPUCaps g_cpu_caps;
/// @brief Global wall clock
-extern const WallClock g_wall_clock;
+extern WallClock g_wall_clock;
} // namespace Common
diff --git a/src/common/range_sets.h b/src/common/range_sets.h
index f8fcee4837..e5cb53b058 100644
--- a/src/common/range_sets.h
+++ b/src/common/range_sets.h
@@ -26,6 +26,9 @@ public:
void Clear();
bool Empty() const;
+ /// Whether any range overlaps [base_address, base_address + size): one lookup.
+ bool Intersects(AddressType base_address, size_t size) const;
+
template <typename Func>
void ForEach(Func&& func) const;
diff --git a/src/common/range_sets.inc b/src/common/range_sets.inc
index 3edd8c8a43..aeda7bc351 100644
--- a/src/common/range_sets.inc
+++ b/src/common/range_sets.inc
@@ -116,28 +116,25 @@ struct OverlapRangeSet<AddressType>::OverlapRangeSetImpl {
}
AddressType end_address = base_address + static_cast<AddressType>(size);
IntervalType interval{base_address, end_address};
- bool any_removals = false;
m_split_ranges_set += std::make_pair(interval, -amount);
- do {
- any_removals = false;
- auto it = m_split_ranges_set.lower_bound(interval);
- if (it == m_split_ranges_set.end()) {
- return;
+ // One pass, erasing the segments that dropped to zero or below in address order (the
+ // search restarted after every erase: quadratic in the segments of the range, and the
+ // GPU fencing thread does this for each downloaded buffer range while holding the buffer
+ // cache's lock). Erasing a node leaves the other iterators valid.
+ auto it = m_split_ranges_set.lower_bound(interval);
+ const auto end_it = m_split_ranges_set.upper_bound(interval);
+ while (it != end_it) {
+ if (it->second > 0) {
+ ++it;
+ continue;
}
- auto end_it = m_split_ranges_set.upper_bound(interval);
- for (; it != end_it; it++) {
- if (it->second <= 0) {
- if constexpr (has_on_delete) {
- if (it->second == 0) {
- on_delete(it->first.lower(), it->first.upper());
- }
- }
- any_removals = true;
- m_split_ranges_set.erase(it);
- break;
+ if constexpr (has_on_delete) {
+ if (it->second == 0) {
+ on_delete(it->first.lower(), it->first.upper());
}
}
- } while (any_removals);
+ m_split_ranges_set.erase(it++);
+ }
}
template <typename Func>
@@ -224,6 +221,18 @@ bool RangeSet<AddressType>::Empty() const {
}
template <typename AddressType>
+bool RangeSet<AddressType>::Intersects(AddressType base_address, size_t size) const {
+ const auto& set = m_impl->m_ranges_set;
+ if (set.empty() || size == 0) {
+ return false;
+ }
+ const AddressType end_address = base_address + static_cast<AddressType>(size);
+ // The first range not entirely below the query; it overlaps when it starts before its end.
+ const auto it = set.lower_bound(typename RangeSetImpl::IntervalType{base_address, end_address});
+ return it != set.end() && it->lower() < end_address;
+}
+
+template <typename AddressType>
template <typename Func>
void RangeSet<AddressType>::ForEach(Func&& func) const {
m_impl->ForEach(std::move(func));
diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp
index c2bb05afba..e65e38dd9e 100644
--- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp
+++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp
@@ -5,6 +5,10 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <atomic>
+#include <cstdlib>
+#if defined(_M_X64) || defined(__x86_64__)
+#include <immintrin.h>
+#endif
#include "common/settings.h"
#include "core/arm/dynarmic/arm_dynarmic.h"
#include "core/arm/dynarmic/arm_dynarmic_64.h"
@@ -166,7 +170,26 @@ u64 DynarmicCallbacks64::GetTicksRemaining() {
}
u64 DynarmicCallbacks64::GetCNTPCT() {
- return m_parent.m_system.CoreTiming().GetClockTicks();
+ const u64 ticks = m_parent.m_system.CoreTiming().GetClockTicks();
+ // Guest code that waits by polling the counter (title 1's main thread reads it ~17 million
+ // times a second while waiting for the GPU) keeps a host core busy at full speed, and the
+ // GPU thread is often on the other half of that SMT core. Two reads within ~1.7 us of each
+ // other are such a loop: pause, which hands the core's resources to the sibling thread for a
+ // moment. The guest sees the same counter. EDEN_NX2_CNTPCT_PAUSE=0 turns it off.
+ static const bool pause_spins = [] {
+ const char* value = std::getenv("EDEN_NX2_CNTPCT_PAUSE");
+ return value == nullptr || value[0] != '0';
+ }();
+ if (pause_spins && ticks - m_last_cntpct < m_spin_ticks) {
+#if defined(_M_X64) || defined(__x86_64__)
+ _mm_pause();
+ _mm_pause();
+#elif defined(__aarch64__)
+ asm volatile("yield");
+#endif
+ }
+ m_last_cntpct = ticks;
+ return ticks;
}
bool DynarmicCallbacks64::CheckMemoryAccess(u64 addr, u64 size, Kernel::DebugWatchpointType type) {
@@ -246,7 +269,9 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s
config.tpidr_el0 = &m_cb->m_tpidr_el0;
config.dczid_el0 = 4;
config.ctr_el0 = 0x8444c004;
- config.cntfrq_el0 = Hardware::CNTFREQ;
+ // 31.25 MHz for Switch 2 titles (Common::WallClock::SetGuestCNTFRQ, set at boot).
+ config.cntfrq_el0 = static_cast<u32>(Common::g_wall_clock.GetGuestCNTFRQ());
+ m_cb->m_spin_ticks = config.cntfrq_el0 / 600'000;
// Unpredictable instructions
config.define_unpredictable_behaviour = true;
diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.h b/src/core/arm/dynarmic/arm_dynarmic_64.h
index 5ea5dde8a6..66b07474c1 100644
--- a/src/core/arm/dynarmic/arm_dynarmic_64.h
+++ b/src/core/arm/dynarmic/arm_dynarmic_64.h
@@ -65,6 +65,8 @@ public:
Core::Memory::Memory& m_memory;
u64 m_tpidrro_el0{};
u64 m_tpidr_el0{};
+ u64 m_last_cntpct{}; ///< The previous counter read (GetCNTPCT spin detection)
+ u64 m_spin_ticks{32}; ///< Reads closer than this (~1.7 us) count as a polling loop
Kernel::KProcess* m_process{};
const bool m_debugger_enabled{};
const bool m_check_memory_access{};
diff --git a/src/core/arm/nce/patcher.cpp b/src/core/arm/nce/patcher.cpp
index f21d7cacdc..72c3497f03 100644
--- a/src/core/arm/nce/patcher.cpp
+++ b/src/core/arm/nce/patcher.cpp
@@ -627,7 +627,7 @@ void Patcher::WriteMsrHandler(ModuleDestLabel module_dest, oaknut::XReg src_reg,
}
void Patcher::WriteCntfrqHandler(ModuleDestLabel module_dest, oaknut::XReg dest_reg, oaknut::VectorCodeGenerator& cg) {
- cg.MOV(dest_reg, Common::WallClock::CNTFRQ);
+ cg.MOV(dest_reg, Common::g_wall_clock.GetGuestCNTFRQ());
// Jump back to the instruction after the emulated MRS.
if (&cg == &c_pre)
@@ -642,6 +642,7 @@ void Patcher::WriteCntpctHandler(ModuleDestLabel module_dest, oaknut::XReg dest_
#else
static Common::WallClock clock(true, 1);
#endif
+ clock.SetGuestCNTFRQ(Common::g_wall_clock.GetGuestCNTFRQ());
const auto factor = clock.GetGuestCNTFRQFactor();
const auto raw_factor = std::bit_cast<std::array<u64, 2>>(factor);
diff --git a/src/core/core.cpp b/src/core/core.cpp
index f3d224b1ce..3728ccf8f7 100644
--- a/src/core/core.cpp
+++ b/src/core/core.cpp
@@ -2,11 +2,13 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include <array>
+#include <cstdlib>
#include <atomic>
#include <memory>
#include <utility>
#include "game_settings.h"
+#include "common/cpu_features.h"
#include "audio_core/audio_core.h"
#include "common/fs/fs.h"
#include "common/logging.h"
@@ -313,16 +315,33 @@ struct System::Impl {
fs_controller.InitTempStorage();
}
+ const bool switch2_program =
+ virtual_filesystem != nullptr &&
+ IsSwitch2Program(GetGameFileFromPath(virtual_filesystem, filepath));
+
// Switch 2 titles size their heap from the total memory and need the 12 GiB layout. Set it
// as a per-game value, so RestoreGlobalState brings back the configured layout afterwards.
- if (virtual_filesystem != nullptr &&
- Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_12Gb &&
- IsSwitch2Program(GetGameFileFromPath(virtual_filesystem, filepath))) {
+ if (switch2_program &&
+ Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_12Gb) {
LOG_INFO(Core, "Switch 2 program: using the 12 GiB memory layout");
Settings::values.memory_layout_mode.SetGlobal(false);
Settings::values.memory_layout_mode.SetValue(Settings::MemoryLayout::Memory_12Gb);
}
+ // The Switch 2 counter runs at 31.25 MHz and its SDK converts ticks with that constant
+ // (nn::os::GetSystemTickFrequency, TickManager::ConvertToTimeSpan/ConvertToTick). At 19.2 MHz
+ // every tick-based time in the game was off by 1.63x: elapsed times read short and a frame
+ // deadline in ticks waited ~27 ms. EDEN_NX2_CNTFRQ=<Hz> overrides it (A/B tests).
+ static const u64 forced_cntfrq = [] {
+ const char* value = std::getenv("EDEN_NX2_CNTFRQ");
+ return value != nullptr ? std::strtoull(value, nullptr, 10) : 0ULL;
+ }();
+ const u64 cntfrq = forced_cntfrq != 0 ? forced_cntfrq
+ : switch2_program ? Common::WallClock::Switch2CNTFRQ
+ : Common::WallClock::CNTFRQ;
+ Common::g_wall_clock.SetGuestCNTFRQ(cntfrq);
+ LOG_INFO(Core, "Guest counter frequency: {} Hz", cntfrq);
+
InitializeKernel(system);
if (params.applet_type == Service::AM::AppletType::Application) {
diff --git a/src/core/core_timing.cpp b/src/core/core_timing.cpp
index b85bd30a58..dfc3f6f2a6 100644
--- a/src/core/core_timing.cpp
+++ b/src/core/core_timing.cpp
@@ -224,7 +224,7 @@ void CoreTiming::ResetTicks() {
}
u64 CoreTiming::GetClockTicks() const {
- u64 fres = is_multicore ? Common::g_wall_clock.GetCNTPCT() : Common::WallClock::CPUTickToCNTPCT(cpu_ticks);
+ u64 fres = is_multicore ? Common::g_wall_clock.GetCNTPCT() : Common::g_wall_clock.CPUTickToGuestCNTPCT(cpu_ticks);
if (const u32 target = CpuClockTargetMhz(Settings::values.cpu_clock.GetValue());
target != CPU_CLOCK_BASE_MHZ) {
fres = fres * target / CPU_CLOCK_BASE_MHZ;
diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h
index 9ed0b6f5e5..46cda14e78 100644
--- a/src/core/device_memory_manager.h
+++ b/src/core/device_memory_manager.h
@@ -132,6 +132,27 @@ private:
u8* host_ptr{};
};
+ /// A thread's last four page translations of one manager. Several threads read through
+ /// them (the GPU thread, the fencing thread, service threads): one cache shared by all was
+ /// cleared by Map on a service thread between the GPU thread's null check of an entry and
+ /// its use, and the GPU thread copied from address 0 (title 1 crashed every few minutes).
+ struct TranslationCache {
+ const void* owner{};
+ u64 generation{};
+ std::array<TranslationEntry, 4> slots{};
+ u32 cursor{};
+ };
+ static inline thread_local TranslationCache t_cache{};
+
+ /// This thread's translations, emptied when Map or Unmap ran since they were made.
+ TranslationCache& Translations() {
+ const u64 current = translation_generation.load(std::memory_order_acquire);
+ if (t_cache.owner != this || t_cache.generation != current) {
+ t_cache = TranslationCache{.owner = this, .generation = current};
+ }
+ return t_cache;
+ }
+
// Internal helper that performs the update assuming the caller already holds the necessary lock.
void UpdatePagesCachedCountNoLock(DAddr addr, size_t size, s32 delta);
@@ -204,8 +225,7 @@ private:
tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit);
}
- std::array<TranslationEntry, 4> t_slot{};
- u32 cache_cursor = 0;
+ std::atomic<u64> translation_generation{1}; ///< Bumped by Map and Unmap
using CounterType = u8;
using CounterAtomicType = std::atomic_uint8_t;
static constexpr size_t subentries = 8 / sizeof(CounterType);
diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc
index 97c2706abb..d70444ef8c 100644
--- a/src/core/device_memory_manager.inc
+++ b/src/core/device_memory_manager.inc
@@ -234,7 +234,7 @@ void DeviceMemoryManager<Traits>::Map(DAddr address, VAddr virtual_address, size
}
impl->multi_dev_address.Register(new_dev, start_id);
}
- t_slot = {};
+ translation_generation.fetch_add(1, std::memory_order_release);
if (track) {
TrackContinuityImpl(address, virtual_address, size, asid);
}
@@ -268,7 +268,7 @@ void DeviceMemoryManager<Traits>::Unmap(DAddr address, size_t size) {
base_dev = new_start | MULTI_FLAG;
}
}
- t_slot = {};
+ translation_generation.fetch_add(1, std::memory_order_release);
}
template <typename Traits>
void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtual_address,
@@ -404,9 +404,10 @@ void DeviceMemoryManager<Traits>::ReadBlock(DAddr address, void* dest_pointer, s
const std::size_t page_offset = address & Memory::YUZU_PAGEMASK;
if (size <= Memory::YUZU_PAGESIZE - page_offset) {
const DAddr guest_page = address & ~static_cast<DAddr>(Memory::YUZU_PAGEMASK);
- for (size_t i = 0; i < 4; ++i) {
- if (t_slot[i].guest_page == guest_page && t_slot[i].host_ptr != nullptr) {
- std::memcpy(dest_pointer, t_slot[i].host_ptr + page_offset, size);
+ TranslationCache& cache = Translations();
+ for (const TranslationEntry& entry : cache.slots) {
+ if (entry.guest_page == guest_page && entry.host_ptr != nullptr) {
+ std::memcpy(dest_pointer, entry.host_ptr + page_offset, size);
return;
}
}
@@ -415,8 +416,8 @@ void DeviceMemoryManager<Traits>::ReadBlock(DAddr address, void* dest_pointer, s
const auto phys_addr = tracked_entries[page_index].compressed_physical_ptr;
if (phys_addr != 0) {
auto* const mem_ptr = GetPointerFromRaw<u8>((PAddr(phys_addr - 1) << Memory::YUZU_PAGEBITS));
- t_slot[cache_cursor % t_slot.size()] = TranslationEntry{.guest_page = guest_page, .host_ptr = mem_ptr};
- cache_cursor = (cache_cursor + 1) & 3U;
+ cache.slots[cache.cursor] = TranslationEntry{.guest_page = guest_page, .host_ptr = mem_ptr};
+ cache.cursor = (cache.cursor + 1) & 3U;
std::memcpy(dest_pointer, mem_ptr + page_offset, size);
return;
}
@@ -462,9 +463,10 @@ void DeviceMemoryManager<Traits>::ReadBlockUnsafe(DAddr address, void* dest_poin
const std::size_t page_offset = address & Memory::YUZU_PAGEMASK;
if (size <= Memory::YUZU_PAGESIZE - page_offset) {
const DAddr guest_page = address & ~static_cast<DAddr>(Memory::YUZU_PAGEMASK);
- for (size_t i = 0; i < 4; ++i) {
- if (t_slot[i].guest_page == guest_page && t_slot[i].host_ptr != nullptr) {
- std::memcpy(dest_pointer, t_slot[i].host_ptr + page_offset, size);
+ TranslationCache& cache = Translations();
+ for (const TranslationEntry& entry : cache.slots) {
+ if (entry.guest_page == guest_page && entry.host_ptr != nullptr) {
+ std::memcpy(dest_pointer, entry.host_ptr + page_offset, size);
return;
}
}
@@ -473,8 +475,8 @@ void DeviceMemoryManager<Traits>::ReadBlockUnsafe(DAddr address, void* dest_poin
const auto phys_addr = tracked_entries[page_index].compressed_physical_ptr;
if (phys_addr != 0) {
auto* const mem_ptr = GetPointerFromRaw<u8>((PAddr(phys_addr - 1) << Memory::YUZU_PAGEBITS));
- t_slot[cache_cursor % t_slot.size()] = TranslationEntry{.guest_page = guest_page, .host_ptr = mem_ptr};
- cache_cursor = (cache_cursor + 1) & 3U;
+ cache.slots[cache.cursor] = TranslationEntry{.guest_page = guest_page, .host_ptr = mem_ptr};
+ cache.cursor = (cache.cursor + 1) & 3U;
std::memcpy(dest_pointer, mem_ptr + page_offset, size);
return;
}
diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h
index 5eab9d420f..3f1f25b6bf 100644
--- a/src/core/hle/service/nx2_gsv.h
+++ b/src/core/hle/service/nx2_gsv.h
@@ -210,7 +210,7 @@ private:
const u32 syncpoint = ~cmd_buf[3];
const u64 value = u64{cmd_buf[4]} | (u64{cmd_buf[5]} << 32);
const u32 mode = cmd_buf[6];
- LOG_INFO(Service, "nx2:syncpt TIPC 201 slot={:#x} syncpoint={} value={:#x} mode={}", slot,
+ LOG_DEBUG(Service, "nx2:syncpt TIPC 201 slot={:#x} syncpoint={} value={:#x} mode={}", slot,
syncpoint, value, mode);
auto*& event = m_events[slot];
if (event == nullptr) {
@@ -249,7 +249,7 @@ private:
void CancelSlotEvent(HLERequestContext& ctx) {
const u32 slot = ctx.CommandBuffer()[2];
- LOG_INFO(Service, "nx2:syncpt TIPC 202 slot={:#x}", slot);
+ LOG_DEBUG(Service, "nx2:syncpt TIPC 202 slot={:#x}", slot);
if (auto it = m_requests.find(slot); it != m_requests.end()) {
Cancel(it->second);
}
@@ -402,7 +402,7 @@ private:
const auto* b = ctx.CommandBuffer();
const u64 start = u64{b[2]} | (u64{b[3]} << 32);
const u64 last = u64{b[4]} | (u64{b[5]} << 32);
- LOG_INFO(Service, "nx2:gpu sub 101 start={:#x} last={:#x} [6]={:#x} [7]={:#x}", start,
+ LOG_DEBUG(Service, "nx2:gpu sub 101 start={:#x} last={:#x} [6]={:#x} [7]={:#x}", start,
last, b[6], b[7]);
if (last > start) {
Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Register(start, last - start + 1);
@@ -410,7 +410,7 @@ private:
// read the first (see video_core/nx2.h).
if (last - start + 1 == 0x2000) {
Tegra::Nx2AddSemaphoreRegion(start);
- LOG_INFO(Service, "nx2:gpu semaphore page {:#x} mirrored to {:#x}", start + 0x1000,
+ LOG_DEBUG(Service, "nx2:gpu semaphore page {:#x} mirrored to {:#x}", start + 0x1000,
start);
}
}
@@ -425,7 +425,7 @@ private:
template <size_t Slot>
void SubSlotJob(HLERequestContext& ctx) {
const auto* w = ctx.CommandBuffer();
- LOG_INFO(Service, "(STUBBED) nx2:gpu {} words {:08x} slot={}", ctx.GetCommand(),
+ LOG_DEBUG(Service, "(STUBBED) nx2:gpu {} words {:08x} slot={}", ctx.GetCommand(),
fmt::join(std::span<const u32>{w + 2, 18}, " "), w[Slot]);
EventFor(w[Slot])->Signal(system.Kernel());
IPC::ResponseBuilder rb{ctx, 2};
@@ -443,7 +443,7 @@ private:
const u64 c = u64{cmd_buf[8]} | (u64{cmd_buf[9]} << 32);
// Words 3-4: first GPU address, 5-6: last, 7: memory handle, 8-9: offset into it.
const u32 handle = cmd_buf[7];
- LOG_INFO(Service, "nx2:gpu 211 map [{:#x}, {:#x}] handle={} offset={:#x} slot={}", a, b,
+ LOG_DEBUG(Service, "nx2:gpu 211 map [{:#x}, {:#x}] handle={} offset={:#x} slot={}", a, b,
handle, c, slot);
if (const auto it = m_memory.find(handle); it != m_memory.end() && b > a) {
// Also at the memory's own address: an earlier 213 may have unmapped the mapping 101
@@ -528,7 +528,7 @@ private:
const u64 start = u64{w[3]} | (u64{w[4]} << 32);
const u64 last = u64{w[5]} | (u64{w[6]} << 32);
const u32 slot = w[11];
- LOG_INFO(Service, "nx2:gpu 214 words {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}",
+ LOG_DEBUG(Service, "nx2:gpu 214 words {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}",
w[1], w[2], w[3], w[4], w[5], w[6], w[7], w[8], w[9], w[10], w[11], w[12]);
if (last > start && ((last + 1 - start) % Core::Memory::YUZU_PAGESIZE) == 0) {
const u64 size = last + 1 - start;
@@ -554,7 +554,7 @@ private:
const auto* w = ctx.CommandBuffer();
const u64 first = u64{w[3]} | (u64{w[4]} << 32);
const u64 last = u64{w[5]} | (u64{w[6]} << 32);
- LOG_INFO(Service, "nx2:gpu 215 destroy sparse region [{:#x}, {:#x}] words {:#x} {:#x} {:#x} {:#x}",
+ LOG_DEBUG(Service, "nx2:gpu 215 destroy sparse region [{:#x}, {:#x}] words {:#x} {:#x} {:#x} {:#x}",
first, last, w[1], w[2], w[7], w[8]);
FreeReleasedRegions();
if (last > first) {
@@ -879,7 +879,10 @@ private:
// game is blocked here while Eden's GPU thread catches up (fence) and composites.
{
using us = std::chrono::microseconds;
- const auto to_us = [](auto d) { return std::chrono::duration_cast<us>(d).count(); };
+ // long long like the stats: microseconds::rep is long on Linux (std::max needs one type).
+ const auto to_us = [](auto d) -> long long {
+ return std::chrono::duration_cast<us>(d).count();
+ };
m_stats.fence_us += to_us(waited - entered);
m_stats.composite_us += to_us(composited - waited);
m_stats.pace_us += to_us(paced - composited);
@@ -1014,10 +1017,18 @@ private:
}
u32 crop_width = layout.width;
u32 crop_height = layout.height;
- if (system.GetApplicationProcessProgramID() == 0x0400C3F00006E000ULL &&
- layout.width == 3840 && layout.height == 2160) {
- crop_width = 2560;
- crop_height = 1440;
+ // Mario Kart World draws into the top-left corner of its present images: 2560x1440 docked
+ // (in 3840x2160 images), 1920x1080 handheld. It keeps its docked images after a switch to
+ // handheld, and showing all of them put the frame in a corner with stale docked frames
+ // around it.
+ if (system.GetApplicationProcessProgramID() == 0x0400C3F00006E000ULL) {
+ if (!Settings::IsDockedMode()) {
+ crop_width = (std::min)(crop_width, 1920U);
+ crop_height = (std::min)(crop_height, 1080U);
+ } else if (layout.width == 3840 && layout.height == 2160) {
+ crop_width = 2560;
+ crop_height = 1440;
+ }
}
static const u32 env_crop_w = [] {
const char* v = std::getenv("EDEN_NX2_CROP_WIDTH");
@@ -1031,7 +1042,10 @@ private:
crop_width = env_crop_w;
crop_height = env_crop_h;
}
- if (m_presents <= 4) {
+ const u64 shown = (u64{layout.width} << 48) | (u64{layout.height} << 32) |
+ (u64{crop_width} << 16) | crop_height;
+ if (shown != m_shown_layout) {
+ m_shown_layout = shown;
LOG_INFO(Service, "nx2:vi2 composite image {} layout {}x{}, crop {}x{}",
image_id, layout.width, layout.height, crop_width, crop_height);
}
@@ -1214,6 +1228,7 @@ private:
std::shared_ptr<VsyncState> m_vsync = std::make_shared<VsyncState>();
std::shared_ptr<Core::Timing::EventType> m_vsync_timer;
u64 m_presents{};
+ u64 m_shown_layout{}; // image and crop size last logged
std::shared_ptr<std::atomic<bool>> m_capture_pending = std::make_shared<std::atomic<bool>>(false);
};
diff --git a/src/core/hle/service/psc/time/common.h b/src/core/hle/service/psc/time/common.h
index 2183fa72b8..6f7d2cb88b 100644
--- a/src/core/hle/service/psc/time/common.h
+++ b/src/core/hle/service/psc/time/common.h
@@ -134,12 +134,13 @@ struct OperationEvent : public Common::IntrusiveListBaseNode<OperationEvent> {
Kernel::KEvent* m_event{};
};
-constexpr inline std::chrono::nanoseconds ConvertToTimeSpan(s64 ticks) {
+inline std::chrono::nanoseconds ConvertToTimeSpan(s64 ticks) {
constexpr auto one_second_ns{
std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::seconds(1)).count()};
- constexpr s64 max{Common::WallClock::CNTFRQ *
- ((std::numeric_limits<s64>::max)() / one_second_ns)};
+ // The guest counter's frequency: 31.25 MHz for Switch 2 titles.
+ const s64 frequency{static_cast<s64>(Common::g_wall_clock.GetGuestCNTFRQ())};
+ const s64 max{frequency * ((std::numeric_limits<s64>::max)() / one_second_ns)};
if (ticks > max) {
return std::chrono::nanoseconds((std::numeric_limits<s64>::max)());
@@ -147,8 +148,8 @@ constexpr inline std::chrono::nanoseconds ConvertToTimeSpan(s64 ticks) {
return std::chrono::nanoseconds((std::numeric_limits<s64>::min)());
}
- auto a{ticks / Common::WallClock::CNTFRQ * one_second_ns};
- auto b{((ticks % Common::WallClock::CNTFRQ) * one_second_ns) / Common::WallClock::CNTFRQ};
+ auto a{ticks / frequency * one_second_ns};
+ auto b{((ticks % frequency) * one_second_ns) / frequency};
return std::chrono::nanoseconds(a + b);
}
diff --git a/src/core/hle/service/sm/sm.cpp b/src/core/hle/service/sm/sm.cpp
index c75c5437df..60f0dd89cb 100644
--- a/src/core/hle/service/sm/sm.cpp
+++ b/src/core/hle/service/sm/sm.cpp
@@ -11,11 +11,19 @@
#include "core/hle/kernel/k_client_port.h"
#include "core/hle/kernel/k_client_session.h"
#include "core/hle/kernel/k_port.h"
+#include "core/hle/kernel/k_process.h"
+#include "core/hle/kernel/k_resource_limit.h"
#include "core/hle/kernel/k_scoped_resource_reservation.h"
#include "core/hle/kernel/k_server_port.h"
+#include "core/hle/kernel/k_server_session.h"
+#include "core/hle/kernel/k_session.h"
+#include "core/hle/kernel/k_thread.h"
+#include "core/hle/kernel/kernel.h"
#include "core/hle/result.h"
#include "core/hle/service/cmif_types.h"
+#include "core/hle/service/hle_ipc.h"
#include "core/hle/service/ipc_helpers.h"
+#include "core/hle/service/nx2_stub_interface.h"
#include "core/hle/service/server_manager.h"
#include "core/hle/service/sm/sm.h"
#include "core/hle/service/sm/sm_controller.h"
@@ -170,6 +178,30 @@ static std::string PopServiceName(IPC::RequestParser& rp) {
return result;
}
+namespace {
+
+// A session served by a Switch 2 stub interface named after the missing service (see
+// nx2_stub_interface.h), set up like a session handed out by a service.
+Kernel::KClientSession* CreateStubSession(Core::System& system, HLERequestContext& ctx,
+ const std::string& name) {
+ auto& kernel = system.Kernel();
+ ASSERT(Kernel::GetCurrentProcess(kernel).GetResourceLimit()->Reserve(
+ kernel, Kernel::LimitableResource::SessionCountMax, 1));
+
+ auto* session = Kernel::KSession::Create(kernel);
+ session->Initialize(kernel, nullptr, 0);
+ Kernel::KSession::Register(kernel, session);
+
+ auto manager =
+ std::make_shared<SessionRequestManager>(kernel, ctx.GetManager()->GetServerManager());
+ manager->SetSessionHandler(std::make_shared<INx2StubInterface>(
+ system, INx2StubInterface::InternName(fmt::format("{}{}", Nx2StubPrefix, name))));
+ ctx.GetManager()->GetServerManager().RegisterSession(&session->GetServerSession(), manager);
+ return &session->GetClientSession();
+}
+
+} // namespace
+
Result SM::GetServiceImpl(Kernel::KClientSession** out_client_session, HLERequestContext& ctx) {
if (!ctx.GetManager()->GetIsInitializedForSm()) {
return Service::SM::ResultInvalidClient;
@@ -187,6 +219,17 @@ Result SM::GetServiceImpl(Kernel::KClientSession** out_client_session, HLEReques
}
if (port_result != ResultSuccess) {
+ // Switch 2 programs also ask for services Eden does not have at all (newer Mario Kart
+ // World updates open chat:u, GameChat, while booting). Nothing ever registers those, so
+ // waiting would hang them for good: hand out a placeholder whose commands are all
+ // auto-stubbed, like their unknown commands on existing services.
+ const auto* const caller = ctx.GetThread().GetOwnerProcess();
+ if (caller != nullptr && caller->IsSwitch2()) {
+ LOG_WARNING(Service_SM, "(STUBBED) Switch 2 service {} does not exist, returning a stub",
+ name);
+ *out_client_session = CreateStubSession(system, ctx, name);
+ return ResultSuccess;
+ }
LOG_INFO(Service_SM, "Waiting for service {} to become available", name);
ctx.SetIsDeferred();
return Service::SM::ResultNotRegistered;
diff --git a/src/qt_common/game_list/worker.cpp b/src/qt_common/game_list/worker.cpp
index 6b6f93447d..70577cb270 100644
--- a/src/qt_common/game_list/worker.cpp
+++ b/src/qt_common/game_list/worker.cpp
@@ -167,6 +167,19 @@ QString FormatGameName(const std::string& physical_name) {
return physical_name_as_qstring;
}
+// Loose ExeFS dumps (<game>/exefs/main, updates in <game>/updates/exefs/main) have no control
+// data, so they are named after their game folder.
+QString LooseExeFSName(const std::string& physical_name) {
+ QDir dir = QFileInfo(QString::fromStdString(physical_name)).dir();
+ if (dir.dirName().compare(QStringLiteral("exefs"), Qt::CaseInsensitive) != 0 || !dir.cdUp()) {
+ return {};
+ }
+ if (dir.dirName().compare(QStringLiteral("updates"), Qt::CaseInsensitive) == 0 && dir.cdUp()) {
+ return QObject::tr("%1 (update)").arg(dir.dirName());
+ }
+ return dir.dirName();
+}
+
QString FormatPatchNameVersions(const FileSys::PatchManager& patch_manager,
Loader::AppLoader& loader, bool updatable = true) {
QString out;
@@ -392,7 +405,13 @@ void GameListWorker::ScanFileSystem(ScanTarget target, const std::string& dir_pa
[[maybe_unused]] const auto res1 = app_loader->ReadIcon(icon);
std::string name = " ";
- [[maybe_unused]] const auto res3 = app_loader->ReadTitle(name);
+ const auto res3 = app_loader->ReadTitle(name);
+ if ((res3 != Loader::ResultStatus::Success || name.empty() || name == " ") &&
+ IsExtractedNCAMain(physical_name)) {
+ if (const QString loose = LooseExeFSName(physical_name); !loose.isEmpty()) {
+ name = loose.toStdString();
+ }
+ }
const FileSys::PatchManager patch{id, system.GetFileSystemController(),
system.GetContentProvider()};
diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h
index 9e5dd0313f..0bed72eaec 100644
--- a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h
+++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h
@@ -758,20 +758,35 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
(u64{insn.Bits(64, 8)} << 20) | (u64{maxwell} << 28));
break;
}
- case 0xb6c: { // TXD ARRAY_2D, four channels and consecutive split destinations.
+ case 0xb6c: { // TXD 2D or ARRAY_2D, four channels; the two destination pairs anywhere.
const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14);
const u32 coordinates = insn.Bits(24, 8), derivatives = insn.Bits(32, 8);
+ // Ampere dimensions: 1 = 2D, 5 = ARRAY_2D; Maxwell TXD's are 2 and 3. The coordinates
+ // (x, y[, layer]) and the four gradients sit in the same registers for both.
+ const u32 dimension = insn.Bits(61, 3);
+ const u32 high = insn.Bits(64, 8); // destination of the third and fourth channels
if ((insn.words[2] & ~0xffU) != 0x001e0f00 || insn.Bits(59, 2) ||
- insn.Bits(61, 3) != 5 || binding >= 18 || offset > 0x1fff ||
- (dest & 1) || dest >= 248 || insn.Bits(64, 8) != dest + 2 ||
+ (dimension != 1 && dimension != 5) || binding >= 18 || offset > 0x1fff ||
+ (dest & 1) || dest >= 248 || (high & 1) || (high >= 248 && high != 255) ||
coordinates >= 248 || derivatives >= 247)
- return fail(index, "native array gradient texture mode");
+ return fail(index, "native gradient texture mode");
if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
return fail(index, "native texture descriptor buffer conflict");
result.required_texture_buffer = binding;
- emit(base(0xde00000000000000ULL) | (u64{coordinates} << 8) |
- (u64{derivatives} << 20) | (u64{3} << 28) | (u64{15} << 31) |
+ // Maxwell writes four consecutive registers: when the second pair is elsewhere (Splatoon
+ // Raiders: R12/R18, R18/R44), write them to scratch and move each pair into place.
+ const bool consecutive = high == dest + 2;
+ const u64 target = consecutive ? dest : 248;
+ emit(0xde00000000000000ULL | predicate | target | (u64{coordinates} << 8) |
+ (u64{derivatives} << 20) | (u64{dimension == 5 ? 3U : 2U} << 28) | (u64{15} << 31) |
(u64{offset} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51));
+ if (!consecutive) {
+ for (u32 i = 0; i < 2; ++i) {
+ emit(0x5c98000000000000ULL | predicate | (dest + i) | (u64{15} << 39) | (u64{248 + i} << 20));
+ if (high != 255)
+ emit(0x5c98000000000000ULL | predicate | (high + i) | (u64{15} << 39) | (u64{250 + i} << 20));
+ }
+ }
break;
}
case 0x989: { // SHFL, immediate lane and register clamp/segmentation mask, any mode.
@@ -1706,7 +1721,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
case 0x220: case 0x820: case 0xa20: case 0xc20: { // FMUL
const u32 scale = insn.Bits(84, 3);
if (insn.Bits(76, 1) || scale == 0 || insn.Bits(81, 3) ||
- insn.Bits(87, 4) || insn.Bits(91, 1) != (opcode == 0xc20) ||
+ insn.Bits(87, 4) || insn.Bits(91, 1) != static_cast<u32>(opcode == 0xc20) ||
(scale != 4 && (!insn.Bits(80, 1) || insn.Bits(78, 2))))
return fail(index, "native FMUL scale/mode");
const u32 a = operand(0), b = operand(1);
@@ -1715,7 +1730,8 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
break;
}
case 0x221: case 0x421: case 0x621: case 0x821: case 0xa21: case 0xe21: { // FADD
- if (insn.Bits(81, 10) || insn.Bits(91, 1) != (opcode == 0xe21)) return fail(index, "native FADD mode");
+ if (insn.Bits(81, 10) || insn.Bits(91, 1) != static_cast<u32>(opcode == 0xe21))
+ return fail(index, "native FADD mode");
const u32 a = operand(0), b = operand(insn.Bits(9, 3) == 2 || insn.Bits(9, 3) == 3 || insn.Bits(9, 3) == 7 ? 2 : 1);
emit(base(0x5c58000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
(u64{insn.Bits(78, 2)} << 39) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(77, 1)} << 50));
diff --git a/src/shader_recompiler/ir_opt/constant_propagation_pass.cpp b/src/shader_recompiler/ir_opt/constant_propagation_pass.cpp
index 6285fef16f..0c0d4fb1a2 100644
--- a/src/shader_recompiler/ir_opt/constant_propagation_pass.cpp
+++ b/src/shader_recompiler/ir_opt/constant_propagation_pass.cpp
@@ -953,6 +953,36 @@ void FoldDriverConstBuffer(Environment& env, IR::Block& block, IR::Inst& inst, u
}
}
+void FoldSMax32(IR::Block& block, IR::Inst& inst) {
+ for (size_t index = 0; index < 2; ++index) {
+ const IR::Value value{inst.Arg(index).Resolve()};
+ const IR::Inst* const negation{inst.Arg(1 - index).TryInstRecursive()};
+ if (!negation || negation->GetOpcode() != IR::Opcode::INeg32 ||
+ negation->Arg(0).Resolve() != value) {
+ continue;
+ }
+ const IR::Inst* const select{value.TryInstRecursive()};
+ if (!select || select->GetOpcode() != IR::Opcode::SelectU32) {
+ continue;
+ }
+ const IR::Value true_value{select->Arg(1).Resolve()};
+ const IR::Value false_value{select->Arg(2).Resolve()};
+ const auto is_mask = [](const IR::Value& candidate) {
+ return candidate.IsImmediate() &&
+ (candidate.U32() == 0 || candidate.U32() == 1 ||
+ candidate.U32() == 0xffffffffU);
+ };
+ if (!is_mask(true_value) || !is_mask(false_value)) {
+ continue;
+ }
+ // max(x, -x) is 0 or 1 for boolean and predicate-mask constants.
+ IR::IREmitter ir{block, IR::Block::InstructionList::s_iterator_to(inst)};
+ inst.ReplaceUsesWith(ir.Select(IR::U1{select->Arg(0)},
+ ir.Imm32(true_value.U32() != 0 ? 1U : 0U),
+ ir.Imm32(false_value.U32() != 0 ? 1U : 0U)));
+ return;
+ }
+}
void ConstantPropagation(Environment& env, IR::Block& block, IR::Inst& inst) {
switch (inst.GetOpcode()) {
case IR::Opcode::GetRegister:
@@ -963,6 +993,8 @@ void ConstantPropagation(Environment& env, IR::Block& block, IR::Inst& inst) {
return FoldAdd<u32>(block, inst);
case IR::Opcode::ISub32:
return FoldISub32(inst);
+ case IR::Opcode::SMax32:
+ return FoldSMax32(block, inst);
case IR::Opcode::IMul32:
FoldWhenAllImmediates(inst, [](u32 a, u32 b) { return a * b; });
return;
diff --git a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp
index bdb4494a15..6aee241dc2 100644
--- a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp
+++ b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp
@@ -414,7 +414,7 @@ void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info)
LOG_WARNING(Shader, "Storage buffer failed to track, using global memory fallbacks");
return;
}
- LOG_WARNING(Shader, "Storage buffer tracked without bias, index {} offset {}",
+ LOG_DEBUG(Shader, "Storage buffer tracked without bias, index {} offset {}",
storage_buffer->index, storage_buffer->offset);
}
// Collect storage buffer and the instruction
diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt
index 163caa32d0..7f83e0d3c2 100644
--- a/src/video_core/CMakeLists.txt
+++ b/src/video_core/CMakeLists.txt
@@ -97,6 +97,8 @@ add_library(video_core STATIC
fence_manager.h
gpu.cpp
gpu.h
+ gpu_split.cpp
+ gpu_split.h
gpu_thread.cpp
gpu_thread.h
guest_memory.h
diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index ad2e200438..42d7ba066d 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -17,6 +17,7 @@
#include "video_core/buffer_cache/buffer_cache_base.h"
#include "video_core/guest_memory.h"
#include "video_core/host1x/gpu_device_memory_manager.h"
+#include "video_core/nx2_profile.h"
#include "video_core/texture_cache/util.h"
namespace VideoCommon {
@@ -57,21 +58,39 @@ template <class P>
BufferCache<P>::~BufferCache() = default;
template <class P>
-void BufferCache<P>::RunGarbageCollector() {
- const bool aggressive_gc = total_used_memory >= critical_memory;
+void BufferCache<P>::RunGarbageCollector(bool aggressive_gc) {
const u64 ticks_to_destroy = aggressive_gc ? 60 : 120;
int num_iterations = aggressive_gc ? 64 : 32;
- const auto clean_up = [this, &num_iterations](BufferId buffer_id) {
+ // Their GPU-written data is downloaded first, all with one wait for the GPU: one wait per
+ // buffer stalled the GPU thread every frame on title 1 (an 8 GB GPU near its budget, no
+ // per-fence downloads keeping the buffers clean), ~2.5% of its time.
+ boost::container::small_vector<BufferId, 64> evicted;
+ lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, [&](BufferId buffer_id) {
if (num_iterations == 0) {
return true;
}
--num_iterations;
- auto& buffer = slot_buffers[buffer_id];
- DownloadBufferMemory(buffer);
- DeleteBuffer(buffer_id);
+ // Below the critical threshold, buffers holding GPU-written data stay: evicting one
+ // means downloading it first. The thresholds compare the whole device's use (mostly
+ // textures) with fixed headroom, so on an 8 GB GPU the collector ran every frame once a
+ // title 1 course was loaded, and the downloads were ~2.7% of the GPU thread.
+ const Buffer& buffer = slot_buffers[buffer_id];
+ if (!aggressive_gc && gpu_modified_ranges.Intersects(buffer.CpuAddr(), buffer.SizeBytes())) {
+ return false;
+ }
+ evicted.push_back(buffer_id);
return false;
- };
- lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, clean_up);
+ });
+ if constexpr (USE_MEMORY_MAPS) {
+ DownloadBuffersMemory(evicted);
+ } else {
+ for (const BufferId buffer_id : evicted) {
+ DownloadBufferMemory(slot_buffers[buffer_id]);
+ }
+ }
+ for (const BufferId buffer_id : evicted) {
+ DeleteBuffer(buffer_id);
+ }
}
template <class P>
@@ -117,8 +136,17 @@ void BufferCache<P>::TickFrame() {
if (runtime.CanReportMemoryUsage()) {
total_used_memory = runtime.GetDeviceMemoryUsage();
}
- if (total_used_memory >= minimum_memory) {
- RunGarbageCollector();
+ if (gpu_memory != nullptr && gpu_memory->UsesNativeTiling()) {
+ // Switch 2 titles fill the device with textures (their collector watches the budget);
+ // title 1's buffers were ~200 MiB of an 8 GB GPU's ~6.8 GB in use, yet this collector
+ // ran aggressively every frame on the device-wide figure, evicting up to 64 buffers (and
+ // downloading the GPU-written ones, ~5% of the GPU thread) that were recreated later.
+ // Collect when the buffers themselves are large.
+ if (buffer_bytes >= 1_GiB) {
+ RunGarbageCollector(buffer_bytes >= 2_GiB);
+ }
+ } else if (total_used_memory >= minimum_memory) {
+ RunGarbageCollector(total_used_memory >= critical_memory);
}
++frame_tick;
delayed_destruction_ring.Tick();
@@ -251,7 +279,9 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am
const DAddr diff = base_address - *cpu_src_address;
const DAddr new_base_address = *cpu_dest_address + diff;
tmp_intervals.push_back({new_base_address, size});
- uncommitted_gpu_modified_ranges.Add(new_base_address, size);
+ if (AsyncBufferDownloads()) {
+ uncommitted_gpu_modified_ranges.Add(new_base_address, size);
+ }
};
gpu_modified_ranges.ForEachInRange(*cpu_src_address, amount, mirror);
// This subtraction in this order is important for overlapping copies.
@@ -666,6 +696,8 @@ void BufferCache<P>::CommitAsyncFlushesHigh() {
async_buffers.emplace_back(std::optional<Async_Buffer>{});
return;
}
+ Nx2Profile::Add(Nx2Profile::Counter::DownloadBytes, total_size_bytes);
+ Nx2Profile::Add(Nx2Profile::Counter::DownloadCopies, downloads.size());
auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes, true);
boost::container::small_vector<BufferCopy, 4> normalized_copies;
runtime.PreCopyBarrier();
@@ -676,6 +708,19 @@ void BufferCache<P>::CommitAsyncFlushesHigh() {
Buffer& buffer = slot_buffers[buffer_id];
second_copy.src_offset = static_cast<size_t>(buffer.CpuAddr()) + copy.src_offset;
const DAddr orig_device_addr = static_cast<DAddr>(second_copy.src_offset);
+ if (Nx2Profile::Enabled()) {
+ u64 on_dma_pages = 0;
+ const auto& pages = Nx2Profile::DmaReadPages();
+ for (DAddr page = orig_device_addr >> 12; page <= (orig_device_addr + copy.size - 1) >> 12;
+ ++page) {
+ if (pages.contains(page)) {
+ const DAddr begin = (std::max)(orig_device_addr, page << 12);
+ const DAddr end = (std::min)(orig_device_addr + copy.size, (page + 1) << 12);
+ on_dma_pages += end - begin;
+ }
+ }
+ Nx2Profile::Add(Nx2Profile::Counter::DownloadBytesOnDmaPages, on_dma_pages);
+ }
async_downloads.Add(orig_device_addr, copy.size);
buffer.MarkUsage(copy.src_offset, copy.size);
runtime.CopyBuffer(download_staging.buffer, buffer, copies, false);
@@ -728,9 +773,9 @@ void BufferCache<P>::PopAsyncBuffers() {
template <class P>
bool BufferCache<P>::IsRegionGpuModified(DAddr addr, size_t size) {
- bool is_dirty = false;
- gpu_modified_ranges.ForEachInRange(addr, size, [&](DAddr, DAddr) { is_dirty = true; });
- return is_dirty;
+ // One lookup instead of visiting every range inside (~4% of the GPU thread for title 1's
+ // inline uploads once GPU-written ranges are no longer cleared by downloads).
+ return gpu_modified_ranges.Intersects(addr, size);
}
template <class P>
@@ -1558,7 +1603,9 @@ void BufferCache<P>::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3
}
memory_tracker.MarkRegionAsGpuModified(device_addr, size);
gpu_modified_ranges.Add(device_addr, size);
- uncommitted_gpu_modified_ranges.Add(device_addr, size);
+ if (AsyncBufferDownloads()) {
+ uncommitted_gpu_modified_ranges.Add(device_addr, size);
+ }
}
template <class P>
@@ -1739,9 +1786,11 @@ void BufferCache<P>::ChangeRegister(BufferId buffer_id) {
Buffer& buffer = slot_buffers[buffer_id];
const auto size = buffer.SizeBytes();
if (insert) {
+ buffer_bytes += Common::AlignUp(size, 1024);
total_used_memory += Common::AlignUp(size, 1024);
buffer.setLRUID(lru_cache.Insert(buffer_id, frame_tick));
} else {
+ buffer_bytes -= Common::AlignUp(size, 1024);
total_used_memory -= Common::AlignUp(size, 1024);
lru_cache.Free(buffer.getLRUID());
}
@@ -1932,6 +1981,52 @@ void BufferCache<P>::DownloadBufferMemory(Buffer& buffer) {
}
template <class P>
+void BufferCache<P>::DownloadBuffersMemory(std::span<const BufferId> buffer_ids) {
+ struct Download {
+ BufferId buffer_id;
+ BufferCopy copy;
+ };
+ std::vector<Download> downloads;
+ u64 total_size_bytes = 0;
+ for (const BufferId buffer_id : buffer_ids) {
+ Buffer& buffer = slot_buffers[buffer_id];
+ const DAddr buffer_addr = buffer.CpuAddr();
+ memory_tracker.ForEachDownloadRangeAndClear(
+ buffer_addr, buffer.SizeBytes(), [&](u64 device_addr_out, u64 range_size) {
+ gpu_modified_ranges.ForEachInRange(device_addr_out, range_size,
+ [&](DAddr start, DAddr end) {
+ downloads.push_back({buffer_id, BufferCopy{
+ .src_offset = start - buffer_addr,
+ .dst_offset = total_size_bytes,
+ .size = end - start,
+ }});
+ total_size_bytes += Common::AlignUp(end - start, 64);
+ });
+ ClearDownload(device_addr_out, range_size);
+ gpu_modified_ranges.Subtract(device_addr_out, range_size);
+ });
+ }
+ if (total_size_bytes == 0) {
+ return;
+ }
+ auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes);
+ for (Download& download : downloads) {
+ download.copy.dst_offset += download_staging.offset;
+ Buffer& buffer = slot_buffers[download.buffer_id];
+ buffer.MarkUsage(download.copy.src_offset, download.copy.size);
+ runtime.CopyBuffer(download_staging.buffer, buffer, std::span{&download.copy, 1}, true);
+ }
+ runtime.Finish();
+ const u8* const mapped_memory = download_staging.mapped_span.data();
+ for (const Download& download : downloads) {
+ const DAddr device_addr = slot_buffers[download.buffer_id].CpuAddr() + download.copy.src_offset;
+ device_memory.WriteBlockUnsafe(
+ device_addr, mapped_memory + (download.copy.dst_offset - download_staging.offset),
+ download.copy.size);
+ }
+}
+
+template <class P>
void BufferCache<P>::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 size) {
boost::container::small_vector<BufferCopy, 1> copies;
u64 total_size_bytes = 0;
diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h
index 57011752cc..45a1836341 100644
--- a/src/video_core/buffer_cache/buffer_cache_base.h
+++ b/src/video_core/buffer_cache/buffer_cache_base.h
@@ -7,6 +7,7 @@
#pragma once
#include <algorithm>
+#include <cstdlib>
#include <array>
#include <bit>
#include <functional>
@@ -377,7 +378,7 @@ private:
((device_addr + size) & ~Core::DEVICE_PAGEMASK);
}
- void RunGarbageCollector();
+ void RunGarbageCollector(bool aggressive_gc);
void BindHostIndexBuffer();
@@ -467,6 +468,9 @@ private:
void DownloadBufferMemory(Buffer& buffer_id, DAddr device_addr, u64 size);
+ /// Downloads the GPU-written data of several buffers with one wait for the GPU.
+ void DownloadBuffersMemory(std::span<const BufferId> buffer_ids);
+
void DeleteBuffer(BufferId buffer_id, bool do_not_mark = false);
[[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index,
@@ -514,6 +518,24 @@ private:
MemoryTracker memory_tracker;
Common::RangeSet<DAddr> uncommitted_gpu_modified_ranges;
+
+ /// Whether GPU-written buffer ranges are copied back to guest memory at the next fence;
+ /// otherwise only when something flushes them (as upstream yuzu below High GPU accuracy).
+ /// Not for Switch 2 titles: title 1 had 250-330 MB/s copied back (a fence per compute
+ /// dispatch, the fencing thread holding the buffer cache while it wrote them), while its CPU
+ /// read GPU-written memory at most 64 times in 200 s (counted with reactive flushing) and
+ /// the native MME's DMA reads never touched those pages. Without them a fence costs ~1 us
+ /// instead of ~5 and draw setup ~7% less. EDEN_NX2_BUFFER_DOWNLOADS=0/1 forces it.
+ bool AsyncBufferDownloads() const {
+ static const int forced = [] {
+ const char* value = std::getenv("EDEN_NX2_BUFFER_DOWNLOADS");
+ return value == nullptr ? -1 : (value[0] == '0' ? 0 : 1);
+ }();
+ if (forced >= 0) {
+ return forced == 1;
+ }
+ return gpu_memory == nullptr || !gpu_memory->UsesNativeTiling();
+ }
Common::RangeSet<DAddr> gpu_modified_ranges;
std::deque<Common::RangeSet<DAddr>> committed_gpu_modified_ranges;
@@ -538,6 +560,7 @@ private:
std::vector<MultiRangeSegment> compute_segments;
u64 frame_tick = 0;
u64 total_used_memory = 0;
+ u64 buffer_bytes = 0; ///< The buffers' own size (total_used_memory may be the device's)
u64 minimum_memory = 0;
u64 critical_memory = 0;
BufferId inline_buffer_id;
diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp
index 24f1db1f7c..dc42906752 100644
--- a/src/video_core/control/scheduler.cpp
+++ b/src/video_core/control/scheduler.cpp
@@ -9,12 +9,28 @@
#include <vector>
#include "common/assert.h"
+#include "core/core.h"
#include "video_core/control/channel_state.h"
#include "video_core/control/scheduler.h"
#include "video_core/gpu.h"
+#include "video_core/gpu_split.h"
namespace Tegra::Control {
+void Scheduler::BindChannel(GPU& gpu, s32 bind_id) {
+ // With the GPU thread split the rasterizer's channel switches with the methods around it, on
+ // the back thread (deferred outside the lock: the back takes it to bind).
+ if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) {
+ split.Defer([this, gpu_ptr = &gpu, bind_id] {
+ std::scoped_lock lk{scheduling_guard};
+ gpu_ptr->BindChannel(bind_id);
+ });
+ return;
+ }
+ std::scoped_lock lk{scheduling_guard};
+ gpu.BindChannel(bind_id);
+}
+
void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) {
std::shared_ptr<ChannelState> channel_state;
{
@@ -22,8 +38,8 @@ void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) {
auto it = channels.find(channel);
ASSERT(it != channels.end());
channel_state = it->second;
- gpu.BindChannel(channel_state->bind_id);
}
+ BindChannel(gpu, channel_state->bind_id);
// Process commands outside the lock to reduce contention.
// Multiple channels can prepare their commands in parallel.
channel_state->payload->dma_pusher.Push(std::move(entries));
@@ -44,13 +60,16 @@ void Scheduler::ResumePending(GPU& gpu) {
progress = false;
for (const auto& state : pending_channels) {
auto& dma_pusher = state->payload->dma_pusher;
+ // Powered off (shutting down), DispatchCalls returns at once and the acquire stays
+ // pending: resuming it again counted as progress forever and the GPU thread never
+ // reached its stop request (closing the game hung).
+ if (!dma_pusher.system.IsPoweredOn()) {
+ return;
+ }
if (!dma_pusher.HasPendingAcquire() || !dma_pusher.CanResumeAcquire()) {
continue;
}
- {
- std::scoped_lock lk{scheduling_guard};
- gpu.BindChannel(state->bind_id);
- }
+ BindChannel(gpu, state->bind_id);
dma_pusher.DispatchCalls();
progress = true;
}
diff --git a/src/video_core/control/scheduler.h b/src/video_core/control/scheduler.h
index 3a3dcc58cf..9a0f1b919f 100644
--- a/src/video_core/control/scheduler.h
+++ b/src/video_core/control/scheduler.h
@@ -34,6 +34,8 @@ public:
void DeclareChannel(std::shared_ptr<ChannelState> new_channel);
private:
+ void BindChannel(GPU& gpu, s32 bind_id);
+
::Common::unordered_map<s32, std::shared_ptr<ChannelState>> channels;
std::mutex scheduling_guard;
std::vector<std::shared_ptr<ChannelState>> pending_channels; ///< Suspended on an acquire
diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp
index 3cc5afe967..62f7e6bfdb 100644
--- a/src/video_core/dma_pusher.cpp
+++ b/src/video_core/dma_pusher.cpp
@@ -12,10 +12,13 @@
#include "common/settings.h"
#include "core/core.h"
#include "core/memory.h"
+#include "video_core/nx2_profile.h"
#include "video_core/dma_pusher.h"
#include "video_core/control/channel_state.h"
+#include "video_core/engines/kepler_compute.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/gpu.h"
+#include "video_core/gpu_split.h"
#include "video_core/guest_memory.h"
#include "video_core/memory_manager.h"
#include "video_core/nx2.h"
@@ -44,6 +47,14 @@ void DmaPusher::DispatchCalls() {
break;
}
}
+ if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) {
+ split.Defer([gpu = &system.GPU()] {
+ gpu->FlushCommands();
+ gpu->OnCommandListEnd();
+ });
+ split.Publish();
+ return;
+ }
system.GPU().FlushCommands();
system.GPU().OnCommandListEnd();
}
@@ -97,7 +108,16 @@ bool DmaPusher::Step() {
const bool macro_payload = engine == Engines::EngineTypes::Maxwell3D && dma_state.method >= MacroRegistersStart;
if (kepler_payload || macro_payload) {
const size_t words = std::min<size_t>(dma_state.method_count, header.size);
- subchannels[dma_state.subchannel]->current_dirty = memory_manager.IsMemoryDirty(dma_state.dma_get, words * sizeof(u32));
+ auto* const engine_state = subchannels[dma_state.subchannel];
+ if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) {
+ // The caches behind IsMemoryDirty belong to the split's back thread.
+ split.Defer([engine_state, memory = &memory_manager, address = GPUVAddr{dma_state.dma_get},
+ bytes = words * sizeof(u32)] {
+ engine_state->current_dirty = memory->IsMemoryDirty(address, bytes);
+ });
+ } else {
+ engine_state->current_dirty = memory_manager.IsMemoryDirty(dma_state.dma_get, words * sizeof(u32));
+ }
}
}
if (command_resume_offset == 0) {
@@ -115,9 +135,9 @@ bool DmaPusher::Step() {
u64{header.allow_flush}, split_method_end, parked_methods.size());
std::array<u32, 8> next_words{};
std::array<u32, 4> prev_words{};
- memory_manager.ReadBlock(header.addr, next_words.data(),
+ memory_manager.ReadBlockUnsafe(header.addr, next_words.data(),
std::min<size_t>(header.size, 8) * sizeof(u32));
- memory_manager.ReadBlock(split_method_end - sizeof(prev_words), prev_words.data(),
+ memory_manager.ReadBlockUnsafe(split_method_end - sizeof(prev_words), prev_words.data(),
sizeof(prev_words));
LOG_DEBUG(HW_GPU, " previous entry ends {:08x}; next entry starts {:08x}",
fmt::join(prev_words, " "), fmt::join(next_words, " "));
@@ -175,6 +195,7 @@ bool DmaPusher::Step() {
}
if (signal_sync) {
+ VideoCommon::GpuSplit::Get().Drain();
rasterizer->SignalFence([this]() {
std::scoped_lock lk(sync_mutex);
synced = true;
@@ -315,10 +336,18 @@ void DmaPusher::WatchArgument(u32 method, u32 argument) const {
}
}
+bool DmaPusher::WantsSegment() const {
+ // Only macro parameters and compute uploads read the engine's current_dma_segment.
+ const auto engine = subchannel_type[dma_state.subchannel];
+ return engine == Engines::EngineTypes::KeplerCompute ||
+ (engine == Engines::EngineTypes::Maxwell3D && dma_state.method >= MacroRegistersStart);
+}
+
void DmaPusher::CallMethod(u32 argument) {
if (g_semaphore_watch != 0) [[unlikely]] {
WatchArgument(dma_state.method, argument);
}
+ auto& split = VideoCommon::GpuSplit::Get();
if (dma_state.method < non_puller_methods) {
puller.CallPullerMethod(*this, Engines::Puller::MethodCall{
dma_state.method,
@@ -332,9 +361,34 @@ void DmaPusher::CallMethod(u32 argument) {
ReportUnboundSubchannel();
return;
}
+ if (split.Active()) {
+ const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset;
+ if (split.FrontMacros()) {
+ const auto type = subchannel_type[dma_state.subchannel];
+ if (type == Engines::EngineTypes::Maxwell3D) {
+ auto* const maxwell3d = static_cast<Engines::Maxwell3D*>(subchannel);
+ if (maxwell3d->GetNativeMacroEngine().native_turing) {
+ maxwell3d->FrontCallMethod(system, split, dma_state.method, argument,
+ dma_state.is_last_call);
+ return;
+ }
+ } else if (type == Engines::EngineTypes::KeplerCompute) {
+ auto* const compute = static_cast<Engines::KeplerCompute*>(subchannel);
+ if (compute->FrontRunsMacros()) {
+ compute->FrontCallMethod(system, split, dma_state.method, argument,
+ dma_state.is_last_call, &segment);
+ return;
+ }
+ }
+ }
+ split.Method(subchannel, dma_state.method, argument, dma_state.is_last_call,
+ WantsSegment() ? &segment : nullptr);
+ return;
+ }
if (!subchannel->execution_mask[dma_state.method]) {
subchannel->method_sink.emplace_back(dma_state.method, argument);
} else {
+ VideoCommon::Nx2Profile::Scope method_scope{VideoCommon::Nx2Profile::Zone::Method};
subchannel->ConsumeSink(system);
subchannel->current_dma_segment = dma_state.dma_get + dma_state.dma_word_offset;
subchannel->CallMethod(system, dma_state.method, argument, dma_state.is_last_call);
@@ -348,6 +402,7 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) {
WatchArgument(dma_state.method + (dma_state.non_incrementing ? 0 : i), base_start[i]);
}
}
+ auto& split = VideoCommon::GpuSplit::Get();
if (dma_state.method < non_puller_methods) {
puller.CallMultiMethod(*this, dma_state.method, dma_state.subchannel, base_start, num_methods, dma_state.method_count);
} else {
@@ -356,6 +411,31 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) {
ReportUnboundSubchannel();
return;
}
+ if (split.Active()) {
+ const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset;
+ if (split.FrontMacros()) {
+ const auto type = subchannel_type[dma_state.subchannel];
+ if (type == Engines::EngineTypes::Maxwell3D) {
+ auto* const maxwell3d = static_cast<Engines::Maxwell3D*>(subchannel);
+ if (maxwell3d->GetNativeMacroEngine().native_turing) {
+ maxwell3d->FrontCallMultiMethod(system, split, dma_state.method, base_start,
+ num_methods, dma_state.method_count);
+ return;
+ }
+ } else if (type == Engines::EngineTypes::KeplerCompute) {
+ auto* const compute = static_cast<Engines::KeplerCompute*>(subchannel);
+ if (compute->FrontRunsMacros()) {
+ compute->FrontCallMultiMethod(system, split, dma_state.method, base_start,
+ num_methods, dma_state.method_count, &segment);
+ return;
+ }
+ }
+ }
+ split.MultiMethod(subchannel, dma_state.method, base_start, num_methods,
+ dma_state.method_count, WantsSegment() ? &segment : nullptr);
+ return;
+ }
+ VideoCommon::Nx2Profile::Scope method_scope{VideoCommon::Nx2Profile::Zone::Method};
subchannel->ConsumeSink(system);
subchannel->current_dma_segment = dma_state.dma_get + dma_state.dma_word_offset;
subchannel->CallMultiMethod(system, dma_state.method, base_start, num_methods, dma_state.method_count);
diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h
index bbcc345c84..82d29edb4b 100644
--- a/src/video_core/dma_pusher.h
+++ b/src/video_core/dma_pusher.h
@@ -172,6 +172,7 @@ private:
void CallMethod(u32 argument);
void CallMultiMethod(const u32* base_start, u32 num_methods);
+ [[nodiscard]] bool WantsSegment() const;
void ReportUnboundSubchannel();
void WatchArgument(u32 method, u32 argument) const;
diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp
index 7792bc9b5d..ab2ba10162 100644
--- a/src/video_core/engines/kepler_compute.cpp
+++ b/src/video_core/engines/kepler_compute.cpp
@@ -17,12 +17,17 @@
#include "common/logging.h"
#include "shader_recompiler/exception.h"
#include "core/core.h"
+#include "video_core/gpu_split.h"
+#include "video_core/nx2_profile.h"
#include "video_core/nx2.h"
#include "video_core/engines/fermi_2d.h"
#include "video_core/engines/kepler_compute.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/memory_manager.h"
#include "video_core/native_query_trace.h"
+#ifdef ARCHITECTURE_x86_64
+#include "video_core/macro_turing_jit.h"
+#endif
#include "video_core/rasterizer_interface.h"
#include "video_core/textures/decoders.h"
@@ -43,6 +48,20 @@ KeplerCompute::KeplerCompute(MemoryManager& memory_manager_)
execution_mask[KEPLER_COMPUTE_REG_INDEX(exec_upload)] = true;
execution_mask[KEPLER_COMPUTE_REG_INDEX(data_upload)] = true;
execution_mask[KEPLER_COMPUTE_REG_INDEX(launch)] = true;
+ // Everything ProcessNativeMethod acts on besides storing the value; the QMD and 0x5a
+ // trace range too while native draw diagnostics trace it.
+ native_plain_methods.set();
+ for (const u32 method : {0x45U, 0x46U, 0x47U, 0x48U, 0xaeU, 0xafU, 0xb0U, 0x107U, 0x6c3U,
+ static_cast<u32>(KEPLER_COMPUTE_REG_INDEX(exec_upload)),
+ static_cast<u32>(KEPLER_COMPUTE_REG_INDEX(data_upload))}) {
+ native_plain_methods[method] = false;
+ }
+ if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr) {
+ native_plain_methods[0x5a] = false;
+ for (u32 method = 0xc6; method <= 0x108; ++method) {
+ native_plain_methods[method] = false;
+ }
+ }
}
KeplerCompute::~KeplerCompute() = default;
@@ -84,8 +103,8 @@ void KeplerCompute::ProcessNativeReportSemaphore(u32 argument) {
rasterizer->SignalFence([this, address, payload, reduction] {
const u32 old_value = memory_manager.Read<u32>(address);
const u32 value = reduction == 0 ? old_value + payload
- : reduction == 1 ? std::min(old_value, payload)
- : reduction == 2 ? std::max(old_value, payload)
+ : reduction == 1 ? (std::min)(old_value, payload)
+ : reduction == 2 ? (std::max)(old_value, payload)
: reduction == 3 ? (old_value >= payload ? 0 : old_value + 1)
: reduction == 4 ? (old_value == 0 || old_value > payload ? payload : old_value - 1)
: reduction == 5 ? (old_value & payload)
@@ -102,7 +121,7 @@ void KeplerCompute::ProcessNativeReportSemaphore(u32 argument) {
}
bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call) {
- if (native_context) {
+ if (native_context && !front_macros) {
if (method == 0x45 || method == 0x47) native_compute_uploaded = true;
native_macro = native_compute_uploaded ? &native_context->compute_engine : &native_context->engine;
native_macro_positions = native_compute_uploaded ? std::span<u32>{native_context->compute_positions} : std::span<u32>{native_context->positions};
@@ -154,9 +173,36 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar
if (watched && log_emits) LOG_INFO(HW_GPU, "Semaphore watch: compute macro emit {:#x} = {:#x}", address, value);
if (diagnostic_writes++ < 4096) LOG_DEBUG(HW_GPU, "Native compute MME emit method={:#x}, value={:#x}", address, value);
if (address >= 0xe00) return false;
+ VideoCommon::Nx2Profile::Scope method_scope{VideoCommon::Nx2Profile::Zone::Method};
CallMethod(system, address, value, true);
return true;
};
+#ifdef ARCHITECTURE_x86_64
+ // Compiled macro code, as for 3D macros (macro_turing_jit.h); EDEN_NX2_MME_JIT=0 off.
+ static const bool use_jit = [] {
+ const char* value = std::getenv("EDEN_NX2_MME_JIT");
+ return value == nullptr || value[0] != '0';
+ }();
+ if (use_jit && !watched) {
+ if (!native_macro->native_jit) {
+ native_macro->native_jit = std::make_shared<TuringMacroJit>();
+ }
+ native_macro->native_jit->Prepare(native_macro->native_code,
+ native_macro->native_code_generation);
+ const auto result = native_macro->native_jit->Execute(
+ start, native_parameters,
+ std::span<u32>{*native_macro->native_interpreter.active_dram},
+ [&](u32 address) { return address < native_registers.size() ? native_registers[address] : 0U; },
+ emit);
+ if (!result.success) {
+ native_macro->native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Native MME (JIT) failed: compute start={:#x}, consumed={}/{}, error={}",
+ start, result.consumed, native_parameters.size(), result.error);
+ }
+ native_parameters.clear();
+ return true;
+ }
+#endif
// Repeated calls replay their writes, as for 3D macros (macro_memo.h).
static const bool use_memo = [] {
const char* value = std::getenv("EDEN_NX2_MME_MEMO");
@@ -263,6 +309,9 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar
}
return true;
}
+ if (front_macros && method >= 0x45 && method <= 0x48) {
+ return true; // Loaded and bound by the split GPU thread's front (FrontApply).
+ }
switch (method) {
case 0x45: native_macro->ClearCode(argument); return true;
case 0x46: native_macro->AddCode(0, argument); return true;
@@ -334,7 +383,7 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar
// releases also advance: a plain write would move it backwards and leave
// GEQ acquires waiting forever.
rasterizer->SignalFence([this, address, payload] {
- const u32 value = std::max(memory_manager.Read<u32>(address), payload);
+ const u32 value = (std::max)(memory_manager.Read<u32>(address), payload);
memory_manager.Write<u32>(address, value);
if (const auto device = memory_manager.GpuToCpuAddress(address)) {
Tegra::Nx2SemaphoreWaiters::Get().OnWrite(*device, value);
@@ -390,7 +439,19 @@ bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 ar
}
void KeplerCompute::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) {
- if (native_mode && ProcessNativeMethod(system, method, method_argument, is_last_call)) return;
+ if (native_mode) {
+ // Most of the compute class's register writes (QMD fields replayed by its macros for
+ // each of thousands of dispatches a second) only store the value: ~4% of title 1's GPU
+ // thread went through ProcessNativeMethod's checks for them.
+ if (method < native_plain_methods.size() && native_plain_methods[method]) {
+ native_registers[method] = method_argument;
+ if (method < Regs::NUM_REGS) {
+ regs.reg_array[method] = method_argument;
+ }
+ return;
+ }
+ if (ProcessNativeMethod(system, method, method_argument, is_last_call)) return;
+ }
if (method >= Regs::NUM_REGS) {
LOG_ERROR(HW_GPU, "Compute register out of range: {:#x}", method);
return;
@@ -697,4 +758,162 @@ Texture::TSCEntry KeplerCompute::GetTSCEntry(u32 tsc_index) const {
return tsc_entry;
}
+// Split GPU thread with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2): the front
+// runs compute macros against its own copy of the native registers (which hold the last value
+// written to each, on the back too) and loads and binds the compute macro code itself.
+struct KeplerCompute::FrontState {
+ std::array<u32, 0x1000> registers{};
+ std::optional<u32> executing_macro;
+ std::vector<u32> parameters;
+ bool compute_uploaded{};
+ u32 bind_pointer{};
+ MacroEngine* macro{}; ///< Without a native context: the one macro engine
+};
+
+KeplerCompute::FrontState& KeplerCompute::Front(VideoCommon::GpuSplit& split) {
+ if (!front) [[unlikely]] {
+ // Copy the back's state and take the macros over while it is idle.
+ split.Drain();
+ front = std::make_unique<FrontState>();
+ front->registers = native_registers;
+ front->executing_macro = native_executing_macro;
+ front->parameters = std::move(native_parameters);
+ native_parameters.clear();
+ native_executing_macro.reset();
+ front->compute_uploaded = native_compute_uploaded;
+ front->bind_pointer = native_bind_pointer;
+ front->macro = native_macro;
+ front_macros = true;
+ }
+ return *front;
+}
+
+void KeplerCompute::FrontApply(FrontState& fs, u32 method, u32 argument) {
+ // As ProcessNativeMethod does with the macro code and bindings.
+ if (method == 0x45 || method == 0x47) {
+ fs.compute_uploaded = true;
+ }
+ fs.registers[method] = argument;
+ switch (method) {
+ case 0x45:
+ FrontMacroEngine(fs).ClearCode(argument);
+ break;
+ case 0x46:
+ FrontMacroEngine(fs).AddCode(0, argument);
+ break;
+ case 0x47:
+ fs.bind_pointer = argument;
+ break;
+ case 0x48: {
+ const auto positions = FrontMacroPositions(fs);
+ if (fs.bind_pointer < positions.size()) {
+ positions[fs.bind_pointer++] = argument;
+ } else {
+ LOG_ERROR(HW_GPU, "Native compute macro bind out of bounds");
+ }
+ break;
+ }
+ default:
+ break;
+ }
+}
+
+MacroEngine& KeplerCompute::FrontMacroEngine(FrontState& fs) {
+ if (native_context) {
+ return fs.compute_uploaded ? native_context->compute_engine : native_context->engine;
+ }
+ return *fs.macro;
+}
+
+std::span<u32> KeplerCompute::FrontMacroPositions(FrontState& fs) {
+ if (native_context) {
+ return fs.compute_uploaded ? std::span<u32>{native_context->compute_positions}
+ : std::span<u32>{native_context->positions};
+ }
+ return native_macro_positions;
+}
+
+void KeplerCompute::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split,
+ u32 method, u32 argument, bool is_last_call,
+ const GPUVAddr* segment) {
+ auto& fs = Front(split);
+ if (method >= fs.registers.size()) {
+ split.Method(this, method, argument, is_last_call, segment);
+ return;
+ }
+ if (method >= 0xe00) {
+ if (!fs.executing_macro) {
+ if (method & 1) {
+ LOG_ERROR(HW_GPU, "Native compute macro starts on argument method");
+ return;
+ }
+ fs.executing_macro = method;
+ }
+ fs.parameters.push_back(argument);
+ if (is_last_call) {
+ FrontRunMacro(system, split, fs);
+ }
+ return;
+ }
+ FrontApply(fs, method, argument);
+ split.Method(this, method, argument, is_last_call, segment);
+}
+
+void KeplerCompute::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split,
+ u32 method, const u32* base_start, u32 amount,
+ u32 methods_pending, const GPUVAddr* segment) {
+ auto& fs = Front(split);
+ if (method >= 0xe00 && method < fs.registers.size()) {
+ for (u32 i = 0; i < amount; ++i) {
+ FrontCallMethod(system, split, method, base_start[i], methods_pending - i <= 1, segment);
+ }
+ return;
+ }
+ if (method < fs.registers.size()) {
+ for (u32 i = 0; i < amount; ++i) {
+ FrontApply(fs, method, base_start[i]);
+ }
+ }
+ split.MultiMethod(this, method, base_start, amount, methods_pending, segment);
+}
+
+void KeplerCompute::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split,
+ FrontState& fs) {
+#ifdef ARCHITECTURE_x86_64
+ const auto positions = FrontMacroPositions(fs);
+ const u32 start = positions[((*fs.executing_macro - 0xe00) / 2) % positions.size()];
+ fs.executing_macro.reset();
+ auto& macro = FrontMacroEngine(fs);
+ if (native_data_ram) {
+ macro.native_interpreter.active_dram = native_data_ram;
+ }
+ VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro};
+ if (!macro.native_jit) {
+ macro.native_jit = std::make_shared<TuringMacroJit>();
+ }
+ macro.native_jit->Prepare(macro.native_code, macro.native_code_generation);
+ const auto result = macro.native_jit->Execute(
+ start, fs.parameters, std::span<u32>{*macro.native_interpreter.active_dram},
+ [&fs](u32 address) {
+ return address < fs.registers.size() ? fs.registers[address] : 0U;
+ },
+ [&](u32 address, u32 value) {
+ if (address >= 0xe00) {
+ return false;
+ }
+ FrontApply(fs, address, value);
+ split.MacroWrite(this, address, value);
+ return true;
+ });
+ if (!result.success) {
+ macro.native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Native MME (JIT, split front) failed: compute start={:#x}, consumed={}, error={}",
+ start, result.consumed, result.error);
+ }
+#else
+ fs.executing_macro.reset();
+#endif
+ fs.parameters.clear();
+}
+
} // namespace Tegra::Engines
diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h
index 5ebf8edf38..b525636dde 100644
--- a/src/video_core/engines/kepler_compute.h
+++ b/src/video_core/engines/kepler_compute.h
@@ -9,7 +9,9 @@
#include <array>
#include <atomic>
#include <cstddef>
+#include <memory>
#include <optional>
+#include <span>
#include <vector>
#include "common/bit_field.h"
#include "common/common_funcs.h"
@@ -24,6 +26,10 @@ namespace Core {
class System;
}
+namespace VideoCommon {
+class GpuSplit;
+}
+
namespace Tegra {
class MemoryManager;
}
@@ -237,7 +243,26 @@ public:
bool IsNativeMode() const { return native_mode; }
+ /// GPU thread split with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2), called by
+ /// the front instead of CallMethod for native compute engines: runs their macros and macro
+ /// uploads against a copy of the registers and sends the register writes to the back.
+ bool FrontRunsMacros() const { return native_mode; }
+ void FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method,
+ u32 argument, bool is_last_call, const GPUVAddr* segment);
+ void FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method,
+ const u32* base_start, u32 amount, u32 methods_pending,
+ const GPUVAddr* segment);
+
private:
+ struct FrontState;
+ FrontState& Front(VideoCommon::GpuSplit& split);
+ void FrontApply(FrontState& state, u32 method, u32 argument);
+ MacroEngine& FrontMacroEngine(FrontState& state);
+ std::span<u32> FrontMacroPositions(FrontState& state);
+ void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state);
+ std::unique_ptr<FrontState> front;
+ bool front_macros{}; ///< The front owns the macro code and bindings (the back skips them)
+
bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor);
void ProcessNativeReportSemaphore(u32 argument);
void DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd);
@@ -252,6 +277,9 @@ private:
bool native_compute_uploaded{};
MacroEngine* native_macro{};
std::array<u32, 0x1000> native_registers{};
+ /// Native methods that only store their value (no macro, QMD, upload, semaphore or code
+ /// side effect): CallMethod writes those directly.
+ std::bitset<0xe00> native_plain_methods;
std::span<u32> native_macro_positions;
std::vector<u32> native_parameters;
std::vector<u32> native_memo_parameters; ///< The current call's parameters, for the memo
diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp
index 60efef9b8e..6adfcf89b8 100644
--- a/src/video_core/engines/maxwell_3d.cpp
+++ b/src/video_core/engines/maxwell_3d.cpp
@@ -4,6 +4,8 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <algorithm>
+#include <array>
#include <cstring>
#include <limits>
#include <optional>
@@ -17,9 +19,14 @@
#include "video_core/engines/maxwell_3d.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/gpu.h"
+#include "video_core/gpu_split.h"
#include "video_core/memory_manager.h"
+#include "video_core/nx2_profile.h"
#include "video_core/rasterizer_interface.h"
#include "video_core/textures/texture.h"
+#ifdef ARCHITECTURE_x86_64
+#include "video_core/macro_turing_jit.h"
+#endif
namespace Tegra::Engines {
@@ -208,11 +215,29 @@ void Maxwell3D::ProcessMacro(Core::System& system, u32 method, const u32* base_s
executing_macro = method;
}
- macro_params.insert(macro_params.end(), base_start, base_start + amount);
macro_segments.emplace_back(current_dma_segment, amount);
current_macro_dirty |= current_dirty;
current_dirty = false;
+ // Nearly every call arrives whole in one method run (title 1 makes ~1,000,000 a second):
+ // copy it to the stack instead of growing the parameter vector. Still a copy: the command
+ // list is often guest memory read in place, which the guest may reuse while the macro runs
+ // (and the macro memo stores the parameters after it ran). Only accurate GPU mode rewrites
+ // the parameters in place (RefreshParameters), so it keeps the vector.
+ // EDEN_NX2_MACRO_COPY=1: always use the vector (for comparisons).
+ static const bool always_copy = std::getenv("EDEN_NX2_MACRO_COPY") != nullptr;
+ if (is_last_call && macro_params.empty() && amount <= 64 && !always_copy &&
+ !Settings::IsGPULevelHigh()) {
+ std::array<u32, 64> parameters;
+ std::copy_n(base_start, amount, parameters.begin());
+ ConsumeSink(system);
+ CallMacroMethod(system, executing_macro, std::span<const u32>{parameters.data(), amount});
+ macro_segments.clear();
+ current_macro_dirty = false;
+ return;
+ }
+ macro_params.insert(macro_params.end(), base_start, base_start + amount);
+
// Call the macro when there are no more parameters in the command buffer
if (is_last_call) {
ConsumeSink(system);
@@ -257,7 +282,7 @@ u32 Maxwell3D::GetMaxCurrentVertices() {
const u64 stream_size = GetNativeMacroEngine().native_turing
? regs.NativeVertexStreamSize(index)
: gpu_addr_end - gpu_addr_begin;
- const u32 address_size = static_cast<u32>(std::min<u64>(stream_size, std::numeric_limits<u32>::max()));
+ const u32 address_size = static_cast<u32>((std::min<u64>)(stream_size, (std::numeric_limits<u32>::max)()));
num_vertices = (std::max)(
num_vertices, address_size / (std::max)(attribute.SizeInBytes(), array.stride.Value()));
break;
@@ -287,17 +312,25 @@ u32 Maxwell3D::ProcessShadowRam(u32 method, u32 argument) {
void Maxwell3D::ConsumeSinkImpl(Core::System& system) {
const auto control = shadow_state.shadow_ram_control;
+ // Sunk methods are state writes only (not in execution_mask): skip unchanged values, as
+ // CallMethod does.
+ const bool skip_same = SkipSameValueWrites();
+ const auto process = [&](u32 method, u32 value) {
+ if (!skip_same || regs.reg_array[method] != value) {
+ ProcessDirtyRegisters(method, value);
+ }
+ };
if (control == Regs::ShadowRamControl::Track || control == Regs::ShadowRamControl::TrackWithFilter) {
for (auto [method, value] : method_sink) {
shadow_state.reg_array[method] = value;
- ProcessDirtyRegisters(method, value);
+ process(method, value);
}
} else if (control == Regs::ShadowRamControl::Replay) {
for (auto [method, value] : method_sink)
- ProcessDirtyRegisters(method, shadow_state.reg_array[method]);
+ process(method, shadow_state.reg_array[method]);
} else {
for (auto [method, value] : method_sink)
- ProcessDirtyRegisters(method, value);
+ process(method, value);
}
method_sink.clear();
}
@@ -318,11 +351,18 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume
case MAXWELL3D_REG_INDEX(shadow_ram_control):
shadow_state.shadow_ram_control = static_cast<Regs::ShadowRamControl>(nonshadow_argument);
return;
+ // With macros on the split GPU thread's front (FrontWrite), the front loads and binds them.
case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+ if (front_macros) return;
return GetNativeMacroEngine().ClearCode(regs.load_mme.instruction_ptr);
case MAXWELL3D_REG_INDEX(load_mme.instruction):
+ if (front_macros) return;
return GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr, argument);
case MAXWELL3D_REG_INDEX(load_mme.start_address):
+ if (front_macros) {
+ ++regs.load_mme.start_address_ptr;
+ return;
+ }
return ProcessMacroBind(argument);
case MAXWELL3D_REG_INDEX(falcon[4]):
return ProcessFirmwareCall4();
@@ -379,7 +419,7 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume
}
}
-void Maxwell3D::CallMacroMethod(Core::System& system, u32 method, const std::vector<u32>& parameters) {
+void Maxwell3D::CallMacroMethod(Core::System& system, u32 method, std::span<const u32> parameters) {
// Reset the current macro.
executing_macro = 0;
@@ -425,7 +465,29 @@ void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument
}
ASSERT(method < Regs::NUM_REGS && "Invalid Maxwell3D register, increase the size of the Regs structure");
+ if (VideoCommon::Nx2Profile::Enabled()) [[unlikely]] {
+ VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::MethodWrites, 1);
+ if (regs.reg_array[method] == method_argument) {
+ VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::SameValueWrites, 1);
+ }
+ }
+ if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && front_macros) [[unlikely]] {
+ back_control_writes[back_control_count++ % back_control_writes.size()] = {
+ method_argument, static_cast<u32>(shadow_state.shadow_ram_control), is_last_call ? 1U : 0U};
+ }
const u32 argument = ProcessShadowRam(method, method_argument);
+ // A state register written with the value it holds changes nothing: no dirty flags (they
+ // would make the next draw re-check that state), no method processing. Title 1 makes ~6
+ // million 3D register writes a second, ~60% of them same-value, most from macro replays.
+ // Methods with side effects (execution_mask: draws, clears, constant buffer data and
+ // binds, semaphores, ...) always run. EDEN_NX2_SKIP_SAME=0 turns it off.
+ if (!execution_mask[method]) {
+ // ProcessMethodCall (and the draw manager behind it) acts only on executable methods.
+ if (!SkipSameValueWrites() || regs.reg_array[method] != argument) {
+ ProcessDirtyRegisters(method, argument);
+ }
+ return;
+ }
ProcessDirtyRegisters(method, argument);
ProcessMethodCall(method, argument, method_argument, is_last_call);
}
@@ -720,4 +782,416 @@ void Maxwell3D::SetHLEReplacementAttributeType(u32 bank, u32 offset,
replace_table.emplace(key, name);
}
+// Split GPU thread with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2). The front keeps
+// what macros read: the registers as the back thread will hold them when it reaches the same
+// point of the method stream. That is every write the front sends, with the shadow RAM applied,
+// plus the few changes the back makes on its own (constant buffer offset, shadow_scratch[0] on
+// falcon[4], the macro bind pointer); the front loads and binds the macro code itself.
+struct Maxwell3D::FrontState {
+ std::array<ControlWrite, 16> control_writes{};
+ u32 control_count{};
+ std::array<u32, Regs::NUM_REGS> regs{};
+ std::array<u32, Regs::NUM_REGS> shadow{};
+ Regs::ShadowRamControl control{};
+ u32 executing_macro{};
+ std::vector<u32> parameters;
+ std::vector<u32> fifo;
+ u64 calls{};
+ /// A method the draw manager handles was sent since the back last ran DrawDeferred: only
+ /// then can it have a deferred instanced draw to issue at the end of a macro.
+ bool draw_pending{true};
+ bool skip_same{}; ///< SkipSameValueWrites() && FrontSkipSame(), read once
+};
+
+namespace {
+
+// EDEN_NX2_SPLIT_VERIFY=1: every 4096 front macro calls, wait for the back and compare the
+// front's registers with the back's (logs and resynchronizes differences).
+bool FrontVerifyEnabled() {
+ static const bool verify = std::getenv("EDEN_NX2_SPLIT_VERIFY") != nullptr;
+ return verify;
+}
+
+// Executable methods ProcessMethodCall handles itself; the rest go to the draw manager.
+bool FrontHandledOutsideDrawManager(u32 method) {
+ constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer);
+ if (method >= cb_data && method < cb_data + 16) {
+ return true;
+ }
+ switch (method) {
+ case MAXWELL3D_REG_INDEX(wait_for_idle):
+ case MAXWELL3D_REG_INDEX(shadow_ram_control):
+ case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+ case MAXWELL3D_REG_INDEX(load_mme.instruction):
+ case MAXWELL3D_REG_INDEX(load_mme.start_address):
+ case MAXWELL3D_REG_INDEX(falcon[4]):
+ case MAXWELL3D_REG_INDEX(bind_groups[0].raw_config):
+ case MAXWELL3D_REG_INDEX(bind_groups[1].raw_config):
+ case MAXWELL3D_REG_INDEX(bind_groups[2].raw_config):
+ case MAXWELL3D_REG_INDEX(bind_groups[3].raw_config):
+ case MAXWELL3D_REG_INDEX(bind_groups[4].raw_config):
+ case MAXWELL3D_REG_INDEX(report_semaphore.query):
+ case MAXWELL3D_REG_INDEX(render_enable.mode):
+ case MAXWELL3D_REG_INDEX(clear_report_value):
+ case MAXWELL3D_REG_INDEX(sync_info):
+ case MAXWELL3D_REG_INDEX(launch_dma):
+ case MAXWELL3D_REG_INDEX(inline_data):
+ case MAXWELL3D_REG_INDEX(fragment_barrier):
+ case MAXWELL3D_REG_INDEX(invalidate_texture_data_cache):
+ case MAXWELL3D_REG_INDEX(tiled_cache_barrier):
+ return true;
+ default:
+ return false;
+ }
+}
+
+// EDEN_NX2_SPLIT_SKIP=0: send state writes that keep the register's value too.
+bool FrontSkipSame() {
+ static const bool skip = [] {
+ const char* value = std::getenv("EDEN_NX2_SPLIT_SKIP");
+ return value == nullptr || value[0] != '0';
+ }();
+ return skip;
+}
+
+} // Anonymous namespace
+
+Maxwell3D::FrontState& Maxwell3D::Front(Core::System& system, VideoCommon::GpuSplit& split) {
+ if (!front) [[unlikely]] {
+ // Copy the back's state and take the macros over while it is idle (state writes it
+ // still holds in its method sink first).
+ split.Drain();
+ ConsumeSink(system);
+ front = std::make_unique<FrontState>();
+ std::memcpy(front->regs.data(), regs.reg_array.data(), sizeof(front->regs));
+ std::memcpy(front->shadow.data(), shadow_state.reg_array.data(), sizeof(front->shadow));
+ front->control = shadow_state.shadow_ram_control;
+ front->skip_same = SkipSameValueWrites() && FrontSkipSame();
+ front->executing_macro = executing_macro;
+ front->parameters = std::move(macro_params);
+ macro_params.clear();
+ executing_macro = 0;
+ front_macros = true;
+ }
+ return *front;
+}
+
+void Maxwell3D::FrontWrite(VideoCommon::GpuSplit& split, FrontState& fs, u32 method,
+ u32 argument, bool is_last_call, bool from_macro) {
+ namespace Profile = VideoCommon::Nx2Profile;
+ // As ProcessShadowRam does on the back.
+ u32 value = argument;
+ bool shadow_kept = true;
+ if (fs.control == Regs::ShadowRamControl::Track ||
+ fs.control == Regs::ShadowRamControl::TrackWithFilter) {
+ shadow_kept = fs.shadow[method] == argument;
+ fs.shadow[method] = argument;
+ } else if (fs.control == Regs::ShadowRamControl::Replay) {
+ value = fs.shadow[method];
+ }
+ if (Profile::Enabled()) [[unlikely]] {
+ Profile::Add(Profile::Counter::MethodWrites, 1);
+ }
+ if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && FrontVerifyEnabled()) [[unlikely]] {
+ fs.control_writes[fs.control_count++ % fs.control_writes.size()] = {
+ argument, static_cast<u32>(fs.control), from_macro ? 2U : 1U};
+ }
+ if (!execution_mask[method]) {
+ // Nothing would change on the back: CallMethod and ConsumeSinkImpl skip same-value state
+ // writes, and the shadow RAM (tracked by default) holds the value already.
+ if (fs.regs[method] == value && shadow_kept && fs.skip_same) {
+ if (Profile::Enabled()) [[unlikely]] {
+ Profile::Add(Profile::Counter::SameValueWrites, 1);
+ }
+ return;
+ }
+ fs.regs[method] = value;
+ } else {
+ fs.regs[method] = value;
+ constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer);
+ switch (method) {
+ case MAXWELL3D_REG_INDEX(shadow_ram_control):
+ // The back's mode is shadow_state.shadow_ram_control: the shadow copy of this very
+ // register, which a later Replay write reads back.
+ fs.control = static_cast<Regs::ShadowRamControl>(argument);
+ fs.shadow[method] = argument;
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+ GetNativeMacroEngine().ClearCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)]);
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.instruction):
+ GetNativeMacroEngine().AddCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)], value);
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.start_address): {
+ const auto positions = GetNativeMacroPositions();
+ u32& pointer = fs.regs[MAXWELL3D_REG_INDEX(load_mme.start_address_ptr)];
+ if (pointer < positions.size()) {
+ positions[pointer] = value;
+ }
+ ++pointer;
+ break;
+ }
+ case MAXWELL3D_REG_INDEX(falcon[4]):
+ fs.regs[MAXWELL3D_REG_INDEX(shadow_scratch)] = 1;
+ break;
+ default:
+ if (method >= cb_data && method < cb_data + 16) {
+ fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)] += sizeof(u32);
+ } else if (!FrontHandledOutsideDrawManager(method)) {
+ fs.draw_pending = true;
+ }
+ break;
+ }
+ }
+ if (from_macro) {
+ split.MacroWrite(this, method, argument);
+ } else {
+ split.Method(this, method, argument, is_last_call, nullptr);
+ }
+}
+
+void Maxwell3D::FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method,
+ u32 argument, bool is_last_call) {
+ auto& fs = Front(system, split);
+ if (method >= MacroRegistersStart) {
+ if (fs.executing_macro == 0) {
+ fs.executing_macro = method;
+ }
+ fs.parameters.push_back(argument);
+ if (is_last_call) {
+ FrontRunMacro(system, split, fs);
+ }
+ return;
+ }
+ FrontWrite(split, fs, method, argument, is_last_call, false);
+}
+
+void Maxwell3D::FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split,
+ u32 method, const u32* base_start, u32 amount,
+ u32 methods_pending) {
+ auto& fs = Front(system, split);
+ if (method >= MacroRegistersStart) {
+ if (fs.executing_macro == 0) {
+ fs.executing_macro = method;
+ }
+ fs.parameters.insert(fs.parameters.end(), base_start, base_start + amount);
+ if (amount == methods_pending) {
+ FrontRunMacro(system, split, fs);
+ }
+ return;
+ }
+ // What CallMultiMethod does to the registers on the back, then one entry for all the words.
+ constexpr u32 cb_data = MAXWELL3D_REG_INDEX(const_buffer.buffer);
+ const auto tracked = [&fs] {
+ return fs.control == Regs::ShadowRamControl::Track ||
+ fs.control == Regs::ShadowRamControl::TrackWithFilter;
+ };
+ if (method >= cb_data && method < cb_data + 16) {
+ fs.regs[MAXWELL3D_REG_INDEX(const_buffer.offset)] += amount * sizeof(u32);
+ } else if (method == MAXWELL3D_REG_INDEX(inline_data)) {
+ // Upload data only.
+ } else if ((method == MAXWELL3D_REG_INDEX(draw_inline_index) ||
+ method == MAXWELL3D_REG_INDEX(inline_index_2x16.even) ||
+ method == MAXWELL3D_REG_INDEX(inline_index_4x8.index0)) &&
+ fs.control != Regs::ShadowRamControl::Replay) {
+ if (amount != 0) {
+ const u32 last = base_start[amount - 1];
+ if (tracked()) {
+ fs.shadow[method] = last;
+ }
+ fs.regs[method] = last;
+ }
+ fs.draw_pending = true;
+ } else {
+ // CallMethod for each word, macro uploads among them (load_mme.instruction). A word may
+ // change the shadow RAM mode for the next (a run of SET_SHADOW_RAM_CONTROL writes).
+ for (u32 i = 0; i < amount; ++i) {
+ const u32 argument = base_start[i];
+ u32 value = argument;
+ if (method == MAXWELL3D_REG_INDEX(shadow_ram_control) && FrontVerifyEnabled()) [[unlikely]] {
+ fs.control_writes[fs.control_count++ % fs.control_writes.size()] = {
+ argument, static_cast<u32>(fs.control), 3U};
+ }
+ if (tracked()) {
+ fs.shadow[method] = argument;
+ } else if (fs.control == Regs::ShadowRamControl::Replay) {
+ value = fs.shadow[method];
+ }
+ fs.regs[method] = value;
+ if (!execution_mask[method]) {
+ continue;
+ }
+ switch (method) {
+ case MAXWELL3D_REG_INDEX(shadow_ram_control):
+ fs.control = static_cast<Regs::ShadowRamControl>(argument);
+ fs.shadow[method] = argument; // The same word on the back (see FrontWrite)
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
+ GetNativeMacroEngine().ClearCode(value);
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.instruction):
+ GetNativeMacroEngine().AddCode(fs.regs[MAXWELL3D_REG_INDEX(load_mme.instruction_ptr)], value);
+ break;
+ case MAXWELL3D_REG_INDEX(load_mme.start_address): {
+ const auto positions = GetNativeMacroPositions();
+ u32& pointer = fs.regs[MAXWELL3D_REG_INDEX(load_mme.start_address_ptr)];
+ if (pointer < positions.size()) {
+ positions[pointer] = value;
+ }
+ ++pointer;
+ break;
+ }
+ case MAXWELL3D_REG_INDEX(falcon[4]):
+ fs.regs[MAXWELL3D_REG_INDEX(shadow_scratch)] = 1;
+ break;
+ default:
+ if (!FrontHandledOutsideDrawManager(method)) {
+ fs.draw_pending = true;
+ }
+ break;
+ }
+ }
+ }
+ split.MultiMethod(this, method, base_start, amount, methods_pending, nullptr);
+}
+
+void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split,
+ FrontState& fs) {
+#ifdef ARCHITECTURE_x86_64
+ const u32 method = fs.executing_macro;
+ fs.executing_macro = 0;
+ auto& engine = GetNativeMacroEngine();
+ const auto positions = GetNativeMacroPositions();
+ const u32 start = positions[((method - MacroRegistersStart) >> 1) % positions.size()];
+ {
+ VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro};
+ engine.native_interpreter.active_dram = &native_data_ram;
+ if (!engine.native_jit) {
+ engine.native_jit = std::make_shared<TuringMacroJit>();
+ }
+ engine.native_jit->Prepare(engine.native_code, engine.native_code_generation);
+ fs.fifo.assign(fs.parameters.begin(), fs.parameters.end());
+ const auto result = engine.native_jit->Execute(
+ start, fs.fifo, std::span<u32>{native_data_ram},
+ [&fs](u32 address) { return address < Regs::NUM_REGS ? fs.regs[address] : 0U; },
+ [&](u32 address, u32 value) {
+ if (address >= MacroRegistersStart) {
+ return false;
+ }
+ FrontWrite(split, fs, address, value, true, true);
+ // MME_DMA_READ (into data RAM) and MME_DMA_READ_FIFOED (into the load FIFO).
+ if (address == 0x157 || address == 0x158) {
+ FrontDmaRead(split, fs, address, value);
+ }
+ return true;
+ });
+ if (!result.success) {
+ engine.native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Native MME (JIT, split front) failed: start={:#x}, consumed={}/{}, error={}",
+ start, result.consumed, fs.parameters.size(), result.error);
+ }
+ }
+ fs.parameters.clear();
+ if (fs.draw_pending) {
+ fs.draw_pending = false;
+ split.Defer([this, system_ptr = &system] { FinishFrontMacro(*system_ptr); });
+ }
+ if (FrontVerifyEnabled() && (++fs.calls & 4095) == 0) [[unlikely]] {
+ FrontVerify(split, fs);
+ }
+#else
+ fs.executing_macro = 0;
+ fs.parameters.clear();
+#endif
+}
+
+void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 method,
+ u32 count) {
+ // The words may be written by methods still queued for the back (indirect draw arguments
+ // from a culling dispatch), and the caches that may hold newer ones belong to it.
+ split.Drain();
+ const GPUVAddr source = (GPUVAddr{fs.regs[0x154] & 0x1ffffff} << 32) | fs.regs[0x155];
+ const u32 ram = fs.regs[0x156];
+ const u32 words = std::min<u32>(count, 0x1000);
+ const size_t read_bytes = size_t{words} * sizeof(u32);
+ if (VideoCommon::Nx2Profile::Enabled()) {
+ VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::DmaReadBytes, read_bytes);
+ }
+ if (memory_manager.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) {
+ memory_manager.FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache);
+ }
+ for (u32 i = 0; i < words; ++i) {
+ const u32 word = memory_manager.Read<u32>(source + u64{i} * sizeof(u32));
+ if (method == 0x158) {
+ fs.fifo.push_back(word);
+ } else if (ram + i < native_data_ram.size()) {
+ native_data_ram[ram + i] = word;
+ }
+ }
+}
+
+void Maxwell3D::FrontVerify(VideoCommon::GpuSplit& split, FrontState& fs) {
+ split.Drain();
+ static u64 checks{};
+ static u64 differences{};
+ ++checks;
+ // State writes the back still holds in its method sink count as done.
+ std::array<u32, Regs::NUM_REGS> expected;
+ std::memcpy(expected.data(), regs.reg_array.data(), sizeof(expected));
+ for (const auto& [method, value] : method_sink) {
+ expected[method] = shadow_state.shadow_ram_control == Regs::ShadowRamControl::Replay
+ ? shadow_state.reg_array[method]
+ : value;
+ }
+ static u64 control_differences{};
+ if ((fs.control != shadow_state.shadow_ram_control ||
+ fs.shadow[MAXWELL3D_REG_INDEX(shadow_ram_control)] != shadow_state.reg_array[MAXWELL3D_REG_INDEX(shadow_ram_control)] ||
+ fs.regs[MAXWELL3D_REG_INDEX(shadow_ram_control)] != expected[MAXWELL3D_REG_INDEX(shadow_ram_control)]) &&
+ control_differences < 4) {
+ LOG_WARNING(HW_GPU, "Split shadow RAM control writes, front ({} total) / back ({} total):",
+ fs.control_count, back_control_count);
+ for (u32 i = 0; i < 16; ++i) {
+ const auto& f = fs.control_writes[(fs.control_count + i) % 16];
+ const auto& b = back_control_writes[(back_control_count + i) % 16];
+ LOG_WARNING(HW_GPU, " front {:#x} (mode {}, path {}) | back {:#x} (mode {}, last {})",
+ f.argument, f.before, f.path, b.argument, b.before, b.path);
+ }
+ }
+ for (u32 i = 0; i < Regs::NUM_REGS; ++i) {
+ if (fs.regs[i] != expected[i]) {
+ if (differences++ < 64) {
+ LOG_WARNING(HW_GPU, "Split front register {:#x}: front {:#x}, back {:#x}", i,
+ fs.regs[i], expected[i]);
+ }
+ fs.regs[i] = expected[i];
+ }
+ }
+ if (fs.control != shadow_state.shadow_ram_control) {
+ if (control_differences++ < 16) {
+ LOG_WARNING(HW_GPU, "Split front shadow RAM control {} vs back {}",
+ static_cast<u32>(fs.control),
+ static_cast<u32>(shadow_state.shadow_ram_control));
+ }
+ fs.control = shadow_state.shadow_ram_control;
+ }
+ for (u32 i = 0; i < Regs::NUM_REGS; ++i) {
+ if (fs.shadow[i] != shadow_state.reg_array[i]) {
+ if (control_differences++ < 16) {
+ LOG_WARNING(HW_GPU, "Split front shadow register {:#x}: front {:#x}, back {:#x}", i,
+ fs.shadow[i], shadow_state.reg_array[i]);
+ }
+ fs.shadow[i] = shadow_state.reg_array[i];
+ }
+ }
+ if ((checks & 63) == 1) {
+ LOG_INFO(HW_GPU, "Split front register checks: {} done, {} differences", checks, differences);
+ }
+}
+
+void Maxwell3D::FinishFrontMacro(Core::System& system) {
+ // What ProcessMacro and CallMacroMethod do around the macro.
+ ConsumeSink(system);
+ draw_manager.DrawDeferred(*this);
+}
+
} // namespace Tegra::Engines
diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h
index 82e716b822..0121f1a01c 100644
--- a/src/video_core/engines/maxwell_3d.h
+++ b/src/video_core/engines/maxwell_3d.h
@@ -9,8 +9,10 @@
#include <array>
#include <bitset>
#include <cmath>
+#include <cstdlib>
#include <limits>
#include <optional>
+#include <span>
#include <type_traits>
#include <vector>
@@ -35,6 +37,10 @@ namespace Tegra {
class MemoryManager;
}
+namespace VideoCommon {
+class GpuSplit;
+}
+
namespace VideoCore {
class RasterizerInterface;
}
@@ -3161,6 +3167,14 @@ public:
/// Write multiple values to the register identified by method.
void CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) override;
+ /// GPU thread split with macros on its front half (gpu_split.h, EDEN_NX2_SPLIT=2), called by
+ /// the front instead of CallMethod: runs macros and macro uploads against a copy of the
+ /// registers and sends the register writes to the back thread.
+ void FrontCallMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method,
+ u32 argument, bool is_last_call);
+ void FrontCallMultiMethod(Core::System& system, VideoCommon::GpuSplit& split, u32 method,
+ const u32* base_start, u32 amount, u32 methods_pending);
+
bool ShouldExecute() const {
return execute_on;
}
@@ -3247,7 +3261,16 @@ private:
* @param method Method to call
* @param parameters Arguments to the method call
*/
- void CallMacroMethod(Core::System& system, u32 method, const std::vector<u32>& parameters);
+ void CallMacroMethod(Core::System& system, u32 method, std::span<const u32> parameters);
+
+ /// Whether state register writes that keep the register's value are dropped (CallMethod).
+ static bool SkipSameValueWrites() {
+ static const bool skip = [] {
+ const char* value = std::getenv("EDEN_NX2_SKIP_SAME");
+ return value == nullptr || value[0] != '0';
+ }();
+ return skip;
+ }
/// Handles writes to the macro uploading register.
void ProcessMacroUpload(u32 data);
@@ -3313,6 +3336,28 @@ private:
std::vector<std::pair<GPUVAddr, size_t>> macro_segments;
bool current_macro_dirty{};
+
+ // Split GPU thread, macros on the front (FrontCallMethod): the front's state, and whether
+ // the front owns the macro code and bindings (the back thread then leaves them alone).
+ struct FrontState;
+ FrontState& Front(Core::System& system, VideoCommon::GpuSplit& split);
+ void FrontWrite(VideoCommon::GpuSplit& split, FrontState& state, u32 method, u32 argument,
+ bool is_last_call, bool from_macro);
+ void FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split, FrontState& state);
+ void FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& state, u32 method, u32 count);
+ void FrontVerify(VideoCommon::GpuSplit& split, FrontState& state);
+ void FinishFrontMacro(Core::System& system);
+ std::unique_ptr<FrontState> front;
+ bool front_macros{};
+ // EDEN_NX2_SPLIT_VERIFY: the last SET_SHADOW_RAM_CONTROL writes each side saw (argument,
+ // mode before, path), printed when the sides disagree.
+ struct ControlWrite {
+ u32 argument;
+ u32 before;
+ u32 path;
+ };
+ std::array<ControlWrite, 16> back_control_writes{};
+ u32 back_control_count{};
};
#define ASSERT_REG_POSITION(field_name, position) \
diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp
index b607f20161..0bb1809aa3 100644
--- a/src/video_core/engines/puller.cpp
+++ b/src/video_core/engines/puller.cpp
@@ -11,6 +11,7 @@
#include "common/logging.h"
#include "common/settings.h"
#include "core/core.h"
+#include "video_core/nx2_profile.h"
#include "video_core/control/channel_state.h"
#include "video_core/dma_pusher.h"
#include "video_core/engines/fermi_2d.h"
@@ -20,6 +21,7 @@
#include "video_core/engines/maxwell_dma.h"
#include "video_core/engines/puller.h"
#include "video_core/gpu.h"
+#include "video_core/gpu_split.h"
#include "video_core/memory_manager.h"
#include "video_core/nx2.h"
#include "video_core/rasterizer_interface.h"
@@ -42,6 +44,21 @@ u32 ReadSemaphoreWord(DmaPusher& dma_pusher, GPUVAddr address) {
return dma_pusher.memory_manager.Read<u32>(address);
}
+// With the GPU thread split (gpu_split.h) the rasterizer belongs to the back thread: work that
+// only feeds it runs there in order; reads of what it wrote first wait for it (DrainBack).
+template <typename Func>
+void OnBack(const Func& func) {
+ if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) {
+ split.Defer(func);
+ } else {
+ func();
+ }
+}
+
+void DrainBack() {
+ VideoCommon::GpuSplit::Get().Drain();
+}
+
} // Anonymous namespace
bool Puller::PendingAcquire::Matches(u32 current) const {
@@ -61,6 +78,9 @@ bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) {
if (!pending_acquire) {
return true;
}
+ // The release may still be queued for the split GPU thread's back half (and the caches
+ // ReadSemaphoreWord asks are its state).
+ DrainBack();
const u32 current = ReadSemaphoreWord(dma_pusher, pending_acquire->address);
if (Nx2SemaphoreWaiters::Get().Latched(this)) {
if (static std::atomic<u32> rescued{};
@@ -160,10 +180,12 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) {
case Puller::FenceOperation::Acquire:
// UNIMPLEMENTED_MSG("Channel Scheduling pending.");
// WaitFence(regs.fence_action.syncpoint_id, regs.fence_value);
- dma_pusher.rasterizer->ReleaseFences();
+ OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->ReleaseFences(); });
break;
case Puller::FenceOperation::Increment:
- dma_pusher.rasterizer->SignalSyncPoint(regs.fence_action.syncpoint_id);
+ OnBack([rasterizer = dma_pusher.rasterizer, id = static_cast<u32>(regs.fence_action.syncpoint_id.Value())] {
+ rasterizer->SignalSyncPoint(id);
+ });
break;
default:
UNIMPLEMENTED_MSG("Unimplemented operation {}", regs.fence_action.op.Value());
@@ -195,12 +217,16 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
const auto properties = g_nx2_ampere && (regs.semaphore_trigger & (1U << 24)) != 0
? VideoCommon::QueryPropertiesFlags::IsAFence
: VideoCommon::QueryPropertiesFlags::HasTimeout;
- dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0);
- if (const GPUVAddr mirror = g_nx2_ampere ? Nx2SemaphoreMirror(sequence_address) : 0;
- mirror != 0) {
+ const GPUVAddr mirror = g_nx2_ampere ? Nx2SemaphoreMirror(sequence_address) : 0;
+ if (mirror != 0) {
LOG_DEBUG(HW_GPU, "Switch 2 semaphore {:#x}={:#x} mirrored to {:#x}", sequence_address, payload, mirror);
- dma_pusher.rasterizer->Query(mirror, VideoCommon::QueryType::Payload, properties, payload, 0);
}
+ OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, mirror, properties, payload] {
+ rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0);
+ if (mirror != 0) {
+ rasterizer->Query(mirror, VideoCommon::QueryType::Payload, properties, payload, 0);
+ }
+ });
} else if (g_nx2_ampere) {
// Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has
// written the commands this channel runs next: suspend the channel until the value is
@@ -217,6 +243,8 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
LOG_ERROR(HW_GPU, "Invalid semaphore operation {}", static_cast<u32>(op));
return;
}
+ VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireChecks, 1);
+ DrainBack();
if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) {
return;
}
@@ -229,10 +257,12 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
dma_pusher.rasterizer->ReleaseFences();
if (!waiters.Latched(this) && !acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) {
pending_acquire = acquire;
+ VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireWaits, 1);
} else {
waiters.Remove(this);
}
} else {
+ DrainBack();
do {
const u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress());
regs.acquire_source = true;
@@ -266,10 +296,14 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
void Puller::ProcessSemaphoreRelease(DmaPusher& dma_pusher) {
const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()};
const u32 payload = regs.semaphore_release;
- dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+ OnBack([rasterizer = dma_pusher.rasterizer, sequence_address, payload] {
+ rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload,
+ VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+ });
}
void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) {
+ DrainBack();
u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress());
const auto value = regs.semaphore_acquire;
while (word != value) {
@@ -291,11 +325,14 @@ void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) {
const u32 payload = regs.reg_array[0x19];
const u32 operation = argument & 7;
if (operation == 1) { // RELEASE
- dma_pusher.rasterizer->Query(address, VideoCommon::QueryType::Payload,
- VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+ OnBack([rasterizer = dma_pusher.rasterizer, address, payload] {
+ rasterizer->Query(address, VideoCommon::QueryType::Payload,
+ VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+ });
return;
}
// Acquires: check once after flushing pending fences; never block the GPU thread on them.
+ DrainBack();
dma_pusher.rasterizer->ReleaseFences();
const u32 word = dma_pusher.memory_manager.Read<u32>(address);
const bool ready = operation == 0 ? word == payload
@@ -336,13 +373,13 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
case BufferMethods::WrcacheFlush:
break;
case BufferMethods::RefCnt:
- dma_pusher.rasterizer->SignalReference();
+ OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->SignalReference(); });
break;
case BufferMethods::SyncpointOperation:
ProcessFenceActionMethod(dma_pusher);
break;
case BufferMethods::WaitForIdle:
- dma_pusher.rasterizer->WaitForIdle();
+ OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->WaitForIdle(); });
break;
case BufferMethods::SemaphoreOperation: {
ProcessSemaphoreTriggerMethod(dma_pusher);
@@ -358,7 +395,7 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
}
case BufferMethods::MemOpB: {
// Implement this better.
- dma_pusher.rasterizer->InvalidateGPUCache();
+ OnBack([rasterizer = dma_pusher.rasterizer] { rasterizer->InvalidateGPUCache(); });
break;
}
case BufferMethods::MemOpC:
diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h
index eb72529da0..61c9844cf2 100644
--- a/src/video_core/fence_manager.h
+++ b/src/video_core/fence_manager.h
@@ -8,11 +8,13 @@
#include <algorithm>
#include <condition_variable>
+#include <cstdlib>
#include <cstring>
#include <deque>
#include <functional>
#include <mutex>
#include <thread>
+#include <vector>
#include <queue>
#include "common/common_types.h"
@@ -22,6 +24,7 @@
#include "video_core/gpu.h"
#include "video_core/host1x/host1x.h"
#include "video_core/host1x/syncpoint_manager.h"
+#include "video_core/nx2_profile.h"
#include "video_core/rasterizer_interface.h"
namespace VideoCommon {
@@ -76,12 +79,21 @@ public:
}
void SignalFence(std::function<void()>&& func) {
+ Nx2Profile::Scope profile_scope{Nx2Profile::Zone::Fence};
const bool delay_fence = Settings::IsGPUFenceBehaviorDefault() ? Settings::IsGPULevelHigh() : Settings::IsGPUFenceBehaviorBalanced() || Settings::IsGPUFenceBehaviorAccurate();
- const bool should_flush = ShouldFlush();
if constexpr (!can_async_check) {
TryReleasePendingFences<false>();
}
- const u8 committed_flushes = CommitAsyncFlushes();
+ auto [should_flush, committed_flushes] = CommitCacheFlushes();
+ if (!delay_fence && committed_flushes == 0 && SkipEmptyFence()) {
+ Nx2Profile::Scope skipped_scope{Nx2Profile::Zone::FenceSkipped};
+ func();
+ rasterizer.InvalidateGPUCache();
+ return;
+ }
+ skipped_fences = 0;
+ should_flush |= query_cache.HasUncommittedFlushes();
+ query_cache.CommitAsyncFlushes();
TFence new_fence = CreateFence(!should_flush);
if constexpr (can_async_check) {
guard.lock();
@@ -207,9 +219,29 @@ private:
Common::SetCurrentThreadPriority(Common::ThreadPriority::High);
Common::SetCurrentThreadToPerformanceCores();
- TFence current_fence;
- std::deque<std::function<void()>> current_operations;
- u8 current_flushes{};
+ // Every fence the host GPU has already passed is released together: Switch 2 titles
+ // signal thousands a second and the host GPU completes them in bursts, so each cache's
+ // lock is taken once per burst instead of once per fence (the GPU thread, which holds
+ // them for nearly every command, spun, slept and woke this thread for each one).
+ // EDEN_NX2_FENCE_BATCH=0: one fence at a time (for comparisons).
+ static const size_t MaxBatch = [] {
+ const char* value = std::getenv("EDEN_NX2_FENCE_BATCH");
+ return value != nullptr && value[0] == '0' ? size_t{1} : size_t{64};
+ }();
+ struct Released {
+ TFence fence;
+ std::deque<std::function<void()>> operations;
+ u8 flushes;
+ };
+ std::vector<Released> batch;
+ batch.reserve(MaxBatch);
+ const auto take_front = [&] {
+ batch.push_back({std::move(fences.front()), std::move(pending_operations.front()),
+ pending_flushes.front()});
+ fences.pop();
+ pending_operations.pop_front();
+ pending_flushes.pop_front();
+ };
while (!stop_token.stop_requested()) {
{
std::unique_lock lock(guard);
@@ -217,24 +249,50 @@ private:
if (stop_token.stop_requested()) [[unlikely]] {
return;
}
- current_fence = std::move(fences.front());
- current_operations = std::move(pending_operations.front());
- current_flushes = pending_flushes.front();
- fences.pop();
- pending_operations.pop_front();
- pending_flushes.pop_front();
+ take_front();
}
- if (!current_fence->IsStubbed()) {
- WaitFence(current_fence);
+ if (!batch.front().fence->IsStubbed()) {
+ WaitFence(batch.front().fence);
}
- PopAsyncFlushes(current_flushes);
- for (auto& operation : current_operations) {
- operation();
+ {
+ std::unique_lock lock(guard);
+ while (batch.size() < MaxBatch && !fences.empty() &&
+ (fences.front()->IsStubbed() || IsFenceSignaled(fences.front()))) {
+ take_front();
+ }
+ }
+ size_t texture_pops = 0;
+ size_t buffer_pops = 0;
+ for (const Released& released : batch) {
+ texture_pops += (released.flushes & TextureFlushes) != 0 ? 1 : 0;
+ buffer_pops += (released.flushes & BufferFlushes) != 0 ? 1 : 0;
+ }
+ if (texture_pops != 0) {
+ std::scoped_lock lock{texture_cache.mutex};
+ for (size_t i = 0; i < texture_pops; ++i) {
+ texture_cache.PopAsyncFlushes();
+ }
+ }
+ if (buffer_pops != 0) {
+ std::scoped_lock lock{buffer_cache.mutex};
+ for (size_t i = 0; i < buffer_pops; ++i) {
+ buffer_cache.PopAsyncFlushes();
+ }
+ }
+ // Queries and operations keep their per-fence order.
+ for (Released& released : batch) {
+ query_cache.PopAsyncFlushes();
+ for (auto& operation : released.operations) {
+ operation();
+ }
}
{
std::unique_lock lock(ring_guard);
- delayed_destruction_ring.Push(std::move(current_fence));
+ for (Released& released : batch) {
+ delayed_destruction_ring.Push(std::move(released.fence));
+ }
}
+ batch.clear();
}
}
@@ -244,12 +302,6 @@ private:
query_cache.ShouldWaitAsyncFlushes();
}
- bool ShouldFlush() const {
- std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
- return texture_cache.HasUncommittedFlushes() || buffer_cache.HasUncommittedFlushes() ||
- query_cache.HasUncommittedFlushes();
- }
-
static constexpr u8 TextureFlushes = 1;
static constexpr u8 BufferFlushes = 2;
@@ -266,35 +318,66 @@ private:
query_cache.PopAsyncFlushes();
}
- /// Returns which caches committed downloads for this fence (TextureFlushes, BufferFlushes).
- /// Switch 2 titles signal a fence per compute dispatch (thousands a second), mostly with
- /// nothing or only buffer data to download: the fencing thread then takes only the lock of
- /// the cache with work, instead of holding both while it copies (the GPU thread's draws and
- /// uploads waited for it).
- u8 CommitAsyncFlushes() {
- u8 committed = 0;
+ struct CacheFlushes {
+ bool should_flush; ///< Downloads to wait for: the fence must reach the host GPU
+ u8 committed; ///< TextureFlushes, BufferFlushes: the caches that committed downloads
+ };
+
+ /// Commits the texture and buffer caches' downloads for a new fence (the query cache's are
+ /// committed once a fence is made). Switch 2 titles signal a fence per compute dispatch
+ /// (thousands a second), mostly with nothing or only buffer data to download: each cache's
+ /// lock is taken once, alone, and the fencing thread later takes only the lock of the cache
+ /// with work, instead of holding both while it copies (the GPU thread's draws and uploads
+ /// waited for it).
+ CacheFlushes CommitCacheFlushes() {
+ CacheFlushes flushes{};
{
std::scoped_lock lock{texture_cache.mutex};
+ flushes.should_flush = texture_cache.HasUncommittedFlushes();
if (texture_cache.HasAsyncFlushWork()) {
texture_cache.CommitAsyncFlushes();
- committed |= TextureFlushes;
+ flushes.committed |= TextureFlushes;
}
}
{
std::scoped_lock lock{buffer_cache.mutex};
if (buffer_cache.HasUncommittedFlushes()) {
buffer_cache.CommitAsyncFlushes();
- committed |= BufferFlushes;
+ flushes.should_flush = true;
+ flushes.committed |= BufferFlushes;
}
}
- query_cache.CommitAsyncFlushes();
- return committed;
+ return flushes;
+ }
+
+ /// Whether a fence with no downloads would do nothing, so need not be made: with fast GPU
+ /// fences its callback runs at once anyway, and most of the per-dispatch fences of Switch 2
+ /// titles have no queries or queued operations either. Making, queueing and retiring them
+ /// cost the GPU thread ~5% and woke the fencing thread for each. One in 64 is still made, so
+ /// the operations fences carry along (query slot cleanup) keep running.
+ bool SkipEmptyFence() {
+ if constexpr (requires { query_cache.HasPendingFenceWork(); }) {
+ if (skipped_fences >= 64 || query_cache.HasPendingFenceWork()) {
+ return false;
+ }
+ {
+ std::scoped_lock lock{uncommitted_guard};
+ if (!uncommitted_operations.empty()) {
+ return false;
+ }
+ }
+ ++skipped_fences;
+ return true;
+ } else {
+ return false;
+ }
}
std::queue<TFence> fences;
std::deque<std::function<void()>> uncommitted_operations;
std::deque<std::deque<std::function<void()>>> pending_operations;
std::deque<u8> pending_flushes; ///< Per fence: the caches that committed downloads
+ u32 skipped_fences{}; ///< Empty fences skipped since the last one made (GPU thread only)
std::mutex guard;
std::mutex ring_guard;
diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp
index 6441b9511a..53f8f063f7 100644
--- a/src/video_core/gpu.cpp
+++ b/src/video_core/gpu.cpp
@@ -221,6 +221,13 @@ struct GPU::Impl {
return raster_area;
}
raster_area.preemtive = true;
+ // A guest CPU read of GPU-written memory (reactive flushing): count them, they decide
+ // whether buffer data must be copied back at every fence (EDEN_NX2_BUFFER_DOWNLOADS).
+ if (static std::atomic<u64> reads{}; (reads.fetch_add(1, std::memory_order_relaxed) & 63) == 0) {
+ LOG_INFO(HW_GPU, "CPU read of GPU-written memory [{:#x}, {:#x}) ({} so far)",
+ raster_area.start_address, raster_area.end_address,
+ reads.load(std::memory_order_relaxed));
+ }
const u64 fence = RequestSyncOperation([this, &raster_area]() {
renderer->ReadRasterizer()->FlushRegion(raster_area.start_address, raster_area.end_address - raster_area.start_address);
});
diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp
new file mode 100644
index 0000000000..f913ca82ce
--- /dev/null
+++ b/src/video_core/gpu_split.cpp
@@ -0,0 +1,350 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <algorithm>
+#include <cstdlib>
+#include <thread>
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+#include <cstdio>
+
+#include "common/logging.h"
+#include "common/settings.h"
+#include "common/thread.h"
+#include "core/core.h"
+#include "video_core/engines/engine_interface.h"
+#include "video_core/gpu_split.h"
+#include "video_core/nx2.h"
+#include "video_core/nx2_profile.h"
+
+namespace VideoCommon {
+
+namespace {
+
+void Pause() {
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
+ _mm_pause();
+#elif defined(__x86_64__) || defined(__i386__)
+ __builtin_ia32_pause();
+#else
+ std::this_thread::yield();
+#endif
+}
+
+std::atomic<u32> g_back_thread_id{};
+std::atomic<bool> g_back_busy{};
+
+} // Anonymous namespace
+
+void GpuSplit::CheckOwner(const char* where) {
+#ifdef _WIN32
+ if (!g_back_busy.load(std::memory_order_acquire)) {
+ return;
+ }
+ const DWORD thread = GetCurrentThreadId();
+ if (thread == g_back_thread_id.load(std::memory_order_relaxed)) {
+ return;
+ }
+ static std::atomic<u32> reports{};
+ if (reports.fetch_add(1) >= 12) {
+ return;
+ }
+ std::fprintf(stderr, "GPU split owner check: %s on thread %lu while the back runs\n", where,
+ thread);
+ void* frames[40];
+ const USHORT count = RtlCaptureStackBackTrace(1, 40, frames, nullptr);
+ for (USHORT i = 0; i < count; ++i) {
+ HMODULE module{};
+ char name[MAX_PATH] = "?";
+ if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
+ GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+ static_cast<LPCSTR>(frames[i]), &module)) {
+ GetModuleFileNameA(module, name, sizeof(name));
+ }
+ const char* base_name = std::strrchr(name, '\\');
+ std::fprintf(stderr, " frame %s+%#llx\n", base_name ? base_name + 1 : name,
+ static_cast<unsigned long long>(reinterpret_cast<uintptr_t>(frames[i]) -
+ reinterpret_cast<uintptr_t>(module)));
+ }
+ std::fflush(stderr);
+#else
+ (void)where;
+#endif
+}
+
+GpuSplit GpuSplit::instance;
+
+void GpuSplit::MaybeStart(Core::System& system_) {
+ static const char mode = [] {
+ const char* value = std::getenv("EDEN_NX2_SPLIT");
+ return value != nullptr ? value[0] : '2';
+ }();
+ const bool wanted = mode == '1' || mode == '2';
+ if (active || !wanted || !Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) {
+ return;
+ }
+ // The front reads command lists straight from guest memory; a read that flushes the caches
+ // first (safe DMA, accurate GPU) would touch back-owned state.
+ const bool safe_reads = Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh()
+ : Settings::IsDMALevelSafe();
+ if (safe_reads || Settings::IsGPULevelHigh() ||
+ Settings::values.renderer_backend.GetValue() != Settings::RendererBackend::Vulkan) {
+ return;
+ }
+ system = &system_;
+ check_owner = std::getenv("EDEN_NX2_SPLIT_CHECK") != nullptr;
+ ring = std::make_unique<u64[]>(RingWords);
+ back_thread = std::jthread([this](std::stop_token stop_token) { BackLoop(stop_token); });
+ active = true;
+#ifdef ARCHITECTURE_x86_64
+ // Front macros run compiled (macro_turing_jit.h) only.
+ const char* jit = std::getenv("EDEN_NX2_MME_JIT");
+ front_macros = mode == '2' && (jit == nullptr || jit[0] != '0');
+#endif
+ LOG_INFO(HW_GPU, "GPU thread split: engine methods run on a second thread{}",
+ front_macros ? ", macros on the first" : "");
+}
+
+void GpuSplit::Stop() {
+ if (!back_thread.joinable()) {
+ return;
+ }
+ back_thread.request_stop();
+ wake.fetch_add(1, std::memory_order_seq_cst);
+ wake.notify_one();
+ back_thread.join();
+ active = false;
+ // The split is static and outlives the emulation session. The next session's back thread
+ // starts reading at 0: with the old positions kept it skipped through the new, empty ring a
+ // wrap at a time and never reached the old write position (the next game's first GPU work
+ // never ran, it waited on its syncpoint forever), and the engines it held were freed.
+ ring.reset();
+ write_position = 0;
+ published_local = 0;
+ consumed_cache = 0;
+ publish_every = PublishEvery;
+ front_engine = nullptr;
+ back_engine = nullptr;
+ front_macros = false;
+ published.store(0, std::memory_order_relaxed);
+ consumed.store(0, std::memory_order_relaxed);
+ back_waiting.store(false, std::memory_order_relaxed);
+}
+
+u64* GpuSplit::ReserveSlow(size_t words) {
+ size_t index = write_position & RingMask;
+ if (index + words > RingWords) {
+ // Entries never wrap: skip to the start of the ring.
+ const size_t rest = RingWords - index;
+ WaitForSpace(rest);
+ ring[index] = static_cast<u64>(Kind::Wrap);
+ write_position += rest;
+ index = 0;
+ }
+ WaitForSpace(words);
+ return &ring[index];
+}
+
+void GpuSplit::WaitForSpace(size_t words) {
+ if (write_position + words - consumed_cache <= RingWords) {
+ return;
+ }
+ Publish();
+ Nx2Profile::Scope scope{Nx2Profile::Zone::SplitDrain};
+ for (u32 spins = 0;; ++spins) {
+ consumed_cache = consumed.load(std::memory_order_acquire);
+ if (write_position + words - consumed_cache <= RingWords) {
+ return;
+ }
+ if (spins < 4096) {
+ Pause();
+ } else {
+ std::this_thread::yield();
+ }
+ }
+}
+
+void GpuSplit::Publish() {
+ if (published_local == write_position) {
+ return;
+ }
+ published_local = write_position;
+ publish_every = std::min<size_t>(publish_every * 2, PublishEvery);
+ published.store(write_position, std::memory_order_seq_cst);
+ if (back_waiting.load(std::memory_order_seq_cst)) {
+ wake.fetch_add(1, std::memory_order_seq_cst);
+ wake.notify_one();
+ }
+}
+
+void GpuSplit::Drain() {
+ consumed_cache = consumed.load(std::memory_order_acquire);
+ if (consumed_cache == write_position) {
+ return;
+ }
+ Publish();
+ Nx2Profile::Scope scope{Nx2Profile::Zone::SplitDrain};
+ for (u32 spins = 0;; ++spins) {
+ consumed_cache = consumed.load(std::memory_order_acquire);
+ if (consumed_cache == write_position) {
+ publish_every = 8;
+ return;
+ }
+ if (spins < 16384) {
+ Pause();
+ } else {
+ std::this_thread::yield();
+ }
+ }
+}
+
+void GpuSplit::SwitchEngine(Tegra::Engines::EngineInterface* engine) {
+ front_engine = engine;
+ u64* const entry = Reserve(2);
+ entry[0] = static_cast<u64>(Kind::Engine);
+ entry[1] = reinterpret_cast<u64>(engine);
+ Commit(2);
+}
+
+void GpuSplit::MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words,
+ u32 amount, u32 methods_pending, const GPUVAddr* segment) {
+ SelectEngine(engine);
+ // Long runs (constant buffer and inline uploads) go in pieces, as when a method's data
+ // continues in the next GPFIFO entry.
+ while (amount > 0) {
+ const u32 piece = std::min(amount, MaxMultiWords);
+ const size_t head = segment != nullptr ? 3 : 2;
+ const size_t size = head + (piece + 1) / 2;
+ u64* const entry = Reserve(size);
+ entry[0] = static_cast<u64>(Kind::Multi) | (segment != nullptr ? HasSegment : 0) |
+ (u64{method & 0xffff} << 8) | (u64{piece} << 32);
+ entry[1] = methods_pending;
+ if (segment != nullptr) {
+ entry[2] = *segment;
+ }
+ entry[size - 1] = 0;
+ std::memcpy(entry + head, words, size_t{piece} * sizeof(u32));
+ Commit(size);
+ words += piece;
+ amount -= piece;
+ methods_pending -= piece;
+ }
+}
+
+void GpuSplit::DeferFunction(std::function<void()>&& func) {
+ auto* const heap = new std::function<void()>(std::move(func));
+ Defer([heap] {
+ (*heap)();
+ delete heap;
+ });
+}
+
+size_t GpuSplit::Execute(const u64* entry) {
+ const u64 head = entry[0];
+ switch (static_cast<Kind>(head & 0xf)) {
+ case Kind::Wrap:
+ return 0;
+ case Kind::Engine:
+ back_engine = reinterpret_cast<Tegra::Engines::EngineInterface*>(entry[1]);
+ return 2;
+ case Kind::Method: {
+ const u32 method = static_cast<u32>((head >> 8) & 0xffff);
+ const u32 argument = static_cast<u32>(head >> 32);
+ auto* const engine = back_engine;
+ if ((head & Direct) != 0) {
+ Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method};
+ engine->ConsumeSink(*system);
+ engine->CallMethod(*system, method, argument, true);
+ } else if (!engine->execution_mask[method]) {
+ engine->method_sink.emplace_back(method, argument);
+ } else {
+ Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method};
+ engine->ConsumeSink(*system);
+ if ((head & HasSegment) != 0) {
+ engine->current_dma_segment = entry[1];
+ }
+ engine->CallMethod(*system, method, argument, (head & IsLast) != 0);
+ }
+ return (head & HasSegment) != 0 ? 2 : 1;
+ }
+ case Kind::Multi: {
+ const u32 method = static_cast<u32>((head >> 8) & 0xffff);
+ const u32 amount = static_cast<u32>(head >> 32);
+ const size_t data = (head & HasSegment) != 0 ? 3 : 2;
+ auto* const engine = back_engine;
+ Nx2Profile::Scope method_scope{Nx2Profile::Zone::Method};
+ engine->ConsumeSink(*system);
+ if ((head & HasSegment) != 0) {
+ engine->current_dma_segment = entry[2];
+ }
+ engine->CallMultiMethod(*system, method, reinterpret_cast<const u32*>(entry + data),
+ amount, static_cast<u32>(entry[1]));
+ return data + (amount + 1) / 2;
+ }
+ case Kind::Call: {
+ const size_t payload = static_cast<size_t>((head >> 8) & 0xffffff);
+ reinterpret_cast<void (*)(const u64*)>(entry[1])(entry + 2);
+ return 2 + payload;
+ }
+ }
+ return 1;
+}
+
+void GpuSplit::BackLoop(std::stop_token stop_token) {
+ Common::SetCurrentThreadName("GPUBack");
+ Common::SetCurrentThreadPriority(Common::ThreadPriority::Critical);
+ Common::SetCurrentThreadToPerformanceCores();
+ system->RegisterHostThread();
+#ifdef _WIN32
+ g_back_thread_id.store(GetCurrentThreadId(), std::memory_order_relaxed);
+#endif
+ u64 read_position = 0;
+ while (!stop_token.stop_requested()) {
+ Nx2Profile::Report("GPU back thread");
+ u64 available = published.load(std::memory_order_acquire);
+ if (available == read_position) {
+ Nx2Profile::Scope scope{Nx2Profile::Zone::SplitWait};
+ // The front usually sends more within microseconds: spin briefly before sleeping.
+ for (u32 spins = 0; spins < 8192 && available == read_position; ++spins) {
+ Pause();
+ available = published.load(std::memory_order_acquire);
+ }
+ if (available == read_position) {
+ // Publish bumps `wake` when it sees back_waiting: read it before the last look
+ // at `published`, so a publish in between ends the wait at once.
+ back_waiting.store(true, std::memory_order_seq_cst);
+ const u32 seen = wake.load(std::memory_order_seq_cst);
+ available = published.load(std::memory_order_seq_cst);
+ if (available == read_position && !stop_token.stop_requested()) {
+ wake.wait(seen, std::memory_order_seq_cst);
+ }
+ back_waiting.store(false, std::memory_order_relaxed);
+ continue;
+ }
+ }
+ u64 last_store = read_position;
+ g_back_busy.store(true, std::memory_order_relaxed);
+ while (read_position != available) {
+ const size_t index = read_position & RingMask;
+ const size_t size = Execute(&ring[index]);
+ read_position += size != 0 ? size : RingWords - index;
+ // Hand ring space back now and then while working through a long batch.
+ if (read_position - last_store >= 4096 && read_position != available) {
+ consumed.store(read_position, std::memory_order_release);
+ last_store = read_position;
+ }
+ }
+ g_back_busy.store(false, std::memory_order_release);
+ consumed.store(read_position, std::memory_order_release);
+ }
+}
+
+} // namespace VideoCommon
diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h
new file mode 100644
index 0000000000..d6f91e4141
--- /dev/null
+++ b/src/video_core/gpu_split.h
@@ -0,0 +1,177 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <atomic>
+#include <cstring>
+#include <functional>
+#include <memory>
+#include <thread>
+#include <type_traits>
+
+#include "common/common_types.h"
+
+namespace Core {
+class System;
+}
+
+namespace Tegra::Engines {
+class EngineInterface;
+}
+
+namespace VideoCommon {
+
+// Splits the GPU thread in two (NX2_GPU_THREAD_SPLIT.md). The GPU thread (the front) takes the
+// CPU's commands, parses command lists and runs the puller; engine method calls and everything
+// behind them (register state, macros, draws, the rasterizer and its caches) run in order on a
+// second thread (the back), fed through a single-producer single-consumer ring. Work the front
+// would do on back-owned state is deferred into the same ring; the front waits for the back to
+// catch up (Drain) only where it needs a result from that state, e.g. a semaphore it acquires.
+// On for Switch 2 titles with the 3D and compute engines' macros on the front, against copies of
+// the registers they read (Maxwell3D and KeplerCompute Front*), which sends the writes they make.
+// EDEN_NX2_SPLIT=1 keeps the macros on the back; EDEN_NX2_SPLIT=0 runs everything on one thread.
+class GpuSplit {
+public:
+ static GpuSplit& Get() {
+ return instance;
+ }
+
+ /// Whether the front sends work to the back (front thread only).
+ [[nodiscard]] bool Active() const {
+ return active;
+ }
+ /// Whether the front runs the engines' macros (EDEN_NX2_SPLIT=2).
+ [[nodiscard]] bool FrontMacros() const {
+ return front_macros;
+ }
+
+ /// Starts the back thread once the split is wanted (EDEN_NX2_SPLIT=1, a Switch 2 title,
+ /// commands read without cache flushes). Called by the front between commands.
+ void MaybeStart(Core::System& system);
+ void Stop();
+
+ /// An engine method write, executed as DmaPusher would (method sink, ConsumeSink, call).
+ /// `segment`: the engine's current_dma_segment for the call, when the engine reads it.
+ void Method(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument, bool is_last,
+ const GPUVAddr* segment) {
+ SelectEngine(engine);
+ const size_t words = segment != nullptr ? 2 : 1;
+ u64* const entry = Reserve(words);
+ entry[0] = static_cast<u64>(Kind::Method) | (is_last ? IsLast : 0) |
+ (segment != nullptr ? HasSegment : 0) | (u64{method & 0xffff} << 8) |
+ (u64{argument} << 32);
+ if (segment != nullptr) {
+ entry[1] = *segment;
+ }
+ Commit(words);
+ }
+ void MultiMethod(Tegra::Engines::EngineInterface* engine, u32 method, const u32* words,
+ u32 amount, u32 methods_pending, const GPUVAddr* segment);
+ /// A write made by a macro: CallMethod without the method sink, as the engines' macros do.
+ void MacroWrite(Tegra::Engines::EngineInterface* engine, u32 method, u32 argument) {
+ SelectEngine(engine);
+ u64* const entry = Reserve(1);
+ entry[0] = static_cast<u64>(Kind::Method) | IsLast | Direct | (u64{method & 0xffff} << 8) |
+ (u64{argument} << 32);
+ Commit(1);
+ }
+
+ /// Runs `func` on the back after everything sent before it. Must be trivially copyable.
+ template <typename Func>
+ void Defer(const Func& func) {
+ static_assert(std::is_trivially_copyable_v<Func> && alignof(Func) <= alignof(u64));
+ constexpr size_t payload = (sizeof(Func) + sizeof(u64) - 1) / sizeof(u64);
+ u64* const entry = Reserve(2 + payload);
+ entry[0] = static_cast<u64>(Kind::Call) | (u64{payload} << 8);
+ entry[1] = reinterpret_cast<u64>(&CallThunk<Func>);
+ std::memcpy(entry + 2, &func, sizeof(Func));
+ Commit(2 + payload);
+ }
+ void DeferFunction(std::function<void()>&& func);
+
+ /// EDEN_NX2_SPLIT_CHECK=1 (debug): reports, with its stack, a thread other than the back
+ /// using back-owned state (the Vulkan scheduler) while the back runs entries.
+ static inline bool check_owner{};
+ static void CheckOwner(const char* where);
+
+ /// Makes what was sent visible to the back.
+ void Publish();
+ /// Waits until the back executed everything sent so far.
+ void Drain();
+
+private:
+ enum class Kind : u64 {
+ Wrap, ///< The rest of the ring is unused: continue at its start
+ Engine, ///< The engine the next methods go to: word 1
+ Method, ///< Bits 8-23 method, 32-63 argument; flags IsLast, HasSegment (word 1)
+ Multi, ///< Bits 8-23 method, 32-63 amount; word 1 methods pending, [segment], words
+ Call, ///< Bits 8-31 payload size in words; word 1 thunk, then the payload
+ };
+ static constexpr u64 IsLast = 1 << 4;
+ static constexpr u64 HasSegment = 1 << 5;
+ static constexpr u64 Direct = 1 << 6; ///< Method: call it even if not executable (macros)
+
+ static constexpr size_t RingWords = size_t{1} << 20; // 8 MiB
+ static constexpr size_t RingMask = RingWords - 1;
+ static constexpr u32 MaxMultiWords = 8192;
+ static constexpr size_t PublishEvery = 256;
+
+ template <typename Func>
+ static void CallThunk(const u64* payload) {
+ (*reinterpret_cast<const Func*>(payload))();
+ }
+
+ /// Room for an entry of `words` at the write position (never across the ring's end).
+ u64* Reserve(size_t words) {
+ const size_t index = write_position & RingMask;
+ if (index + words <= RingWords && write_position + words - consumed_cache <= RingWords)
+ [[likely]] {
+ return &ring[index];
+ }
+ return ReserveSlow(words);
+ }
+ u64* ReserveSlow(size_t words);
+ void Commit(size_t words) {
+ write_position += words;
+ if (write_position - published_local >= publish_every) {
+ Publish();
+ }
+ }
+ void WaitForSpace(size_t words);
+ void SelectEngine(Tegra::Engines::EngineInterface* engine) {
+ if (engine != front_engine) [[unlikely]] {
+ SwitchEngine(engine);
+ }
+ }
+ void SwitchEngine(Tegra::Engines::EngineInterface* engine);
+
+ void BackLoop(std::stop_token stop_token);
+ size_t Execute(const u64* entry);
+
+ Core::System* system{};
+ bool active{};
+ bool front_macros{};
+ std::unique_ptr<u64[]> ring;
+
+ // Front side.
+ u64 write_position{};
+ u64 published_local{};
+ u64 consumed_cache{};
+ /// Words between publishes: small right after a drain, while the back has nothing to do.
+ size_t publish_every{PublishEvery};
+ Tegra::Engines::EngineInterface* front_engine{};
+
+ // Back side.
+ Tegra::Engines::EngineInterface* back_engine{};
+
+ static GpuSplit instance;
+
+ alignas(64) std::atomic<u64> published{};
+ alignas(64) std::atomic<u64> consumed{};
+ alignas(64) std::atomic<bool> back_waiting{};
+ std::atomic<u32> wake{};
+ std::jthread back_thread;
+};
+
+} // namespace VideoCommon
diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp
index 72da1904e1..55200b0647 100644
--- a/src/video_core/gpu_thread.cpp
+++ b/src/video_core/gpu_thread.cpp
@@ -17,12 +17,28 @@
#include "video_core/dma_pusher.h"
#include "video_core/gpu.h"
#include "video_core/gpu_thread.h"
+#include "video_core/gpu_split.h"
#include "video_core/nx2_profile.h"
#include "video_core/host1x/host1x.h"
#include "video_core/renderer_base.h"
namespace VideoCommon::GPUThread {
+namespace {
+
+// Tells a CPU thread waiting in PushCommand that its command ran.
+void SignalCommand(SynchState& state, u64 fence, bool block) {
+ state.signaled_fence.store(fence);
+ if (block) {
+ // We have to lock the write_lock to ensure that the condition_variable wait not get a
+ // race between the check and the lock itself.
+ std::scoped_lock lk{state.write_lock};
+ state.cv.notify_all();
+ }
+}
+
+} // Anonymous namespace
+
ThreadManager::ThreadManager(Core::System& system_)
: system{system_}
{}
@@ -42,8 +58,10 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten
// When a pending channel last moved or a command arrived (see the polling below).
auto last_progress = std::chrono::steady_clock::now();
namespace Profile = VideoCommon::Nx2Profile;
+ auto& split = VideoCommon::GpuSplit::Get();
while (!stop_token.stop_requested()) {
Profile::Report();
+ split.MaybeStart(system);
if (scheduler.HasPendingChannels()) {
// Suspended channels wait on semaphores the CPU releases: poll them meanwhile.
{
@@ -71,6 +89,34 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten
break;
}
last_progress = std::chrono::steady_clock::now();
+ if (split.Active()) {
+ // Only command lists are parsed here; everything else runs on the back thread
+ // in order, and so does the signal that the command ran.
+ auto* const rasterizer_ptr = renderer.ReadRasterizer();
+ if (auto* submit_list = std::get_if<SubmitListCommand>(&next.data)) {
+ Profile::Scope scope{Profile::Zone::Push};
+ scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries));
+ } else if (std::holds_alternative<GPUTickCommand>(next.data)) {
+ split.Defer([gpu = &system.GPU()] { gpu->TickWork(); });
+ } else if (auto* run = std::get_if<RunCommand>(&next.data)) {
+ split.DeferFunction(std::move(run->func));
+ } else if (const auto* flush = std::get_if<FlushRegionCommand>(&next.data)) {
+ split.Defer([rasterizer_ptr, addr = flush->addr, size = flush->size] {
+ rasterizer_ptr->FlushRegion(addr, size);
+ });
+ } else if (const auto* invalidate = std::get_if<InvalidateRegionCommand>(&next.data)) {
+ split.Defer([rasterizer_ptr, addr = invalidate->addr, size = invalidate->size] {
+ rasterizer_ptr->OnCacheInvalidation(addr, size);
+ });
+ } else {
+ ASSERT(false);
+ }
+ split.Defer([state_ptr = &state, fence = next.fence, block = next.block] {
+ SignalCommand(*state_ptr, fence, block);
+ });
+ split.Publish();
+ continue;
+ }
if (auto* submit_list = std::get_if<SubmitListCommand>(&next.data)) {
Profile::Scope scope{Profile::Zone::Push};
scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries));
@@ -89,13 +135,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten
} else {
ASSERT(false);
}
- state.signaled_fence.store(next.fence);
- if (next.block) {
- // We have to lock the write_lock to ensure that the condition_variable wait not get a
- // race between the check and the lock itself.
- std::scoped_lock lk{state.write_lock};
- state.cv.notify_all();
- }
+ SignalCommand(state, next.fence, next.block);
}
});
}
@@ -161,6 +201,7 @@ void ThreadManager::NotifyShutdown() {
thread.request_stop();
thread.join();
}
+ VideoCommon::GpuSplit::Get().Stop();
}
} // namespace VideoCommon::GPUThread
diff --git a/src/video_core/macro.cpp b/src/video_core/macro.cpp
index 3422cb16fa..485b8eaf10 100644
--- a/src/video_core/macro.cpp
+++ b/src/video_core/macro.cpp
@@ -17,6 +17,7 @@
#ifdef ARCHITECTURE_x86_64
#include "common/x64/xbyak.h"
+#include "video_core/macro_turing_jit.h"
#endif
#include "common/assert.h"
@@ -1404,7 +1405,10 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u
modes[0], modes[1], modes[2], modes[3]);
}
}
- maxwell3d.CallMethod(system, address, value, true);
+ {
+ VideoCommon::Nx2Profile::Scope method_scope{VideoCommon::Nx2Profile::Zone::Method};
+ maxwell3d.CallMethod(system, address, value, true);
+ }
// Turing MME memory reads (methods 0x550 SET_MME_MEM_ADDRESS_A/B, 0x558 data RAM
// address, 0x55c MME_DMA_READ into data RAM, 0x560 MME_DMA_READ_FIFOED into
// the load FIFO, which the macro then reads after a load barrier).
@@ -1420,6 +1424,14 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u
// from a culling dispatch) and still be only in host buffers: flush them first,
// or the macro draws with an older frame's arguments (objects pop in and out).
const size_t read_bytes = size_t{std::min<u32>(value, 0x1000)} * 4;
+ if (VideoCommon::Nx2Profile::Enabled()) {
+ VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::DmaReadBytes, read_bytes);
+ for (GPUVAddr page = source & ~GPUVAddr{0xfff}; page < source + read_bytes; page += 0x1000) {
+ if (const auto device = memory.GpuToCpuAddress(page)) {
+ VideoCommon::Nx2Profile::DmaReadPages().insert(*device >> 12);
+ }
+ }
+ }
if (memory.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) {
memory.FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache);
static u64 flushes{};
@@ -1451,6 +1463,33 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u
}
return true;
};
+#ifdef ARCHITECTURE_x86_64
+ // Macro code runs compiled to x86-64 (macro_turing_jit.h) instead of through the memo
+ // and the interpreter; EDEN_NX2_MME_JIT=0 goes back to those.
+ static const bool use_jit = [] {
+ const char* value = std::getenv("EDEN_NX2_MME_JIT");
+ return value == nullptr || value[0] != '0';
+ }();
+ if (use_jit) {
+ if (!native_jit) {
+ native_jit = std::make_shared<TuringMacroJit>();
+ }
+ native_jit->Prepare(native_code, native_code_generation);
+ fifo.assign(parameters.begin(), parameters.end());
+ const auto result = native_jit->Execute(
+ method, fifo, std::span<u32>{*native_interpreter.active_dram},
+ [&](u32 address) {
+ return address < Maxwell3D::Regs::NUM_REGS ? maxwell3d.GetRegisterValue(address) : 0U;
+ },
+ emit);
+ if (!result.success) {
+ native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Native MME (JIT) failed: start={:#x}, consumed={}/{}, error={}",
+ method, result.consumed, parameters.size(), result.error);
+ }
+ return;
+ }
+#endif
// EDEN_NX2_MME_MEMO=0: interpret every call (see macro_memo.h).
static const bool use_memo = [] {
const char* value = std::getenv("EDEN_NX2_MME_MEMO");
@@ -1462,10 +1501,11 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u
memo_stats && (++native_calls & 0xffffff) == 0) {
const auto& stats = native_memo.GetStatistics();
LOG_INFO(HW_GPU,
- "Macro memo: {} hits, {} misses, {} guard failures, {} stores ({} in "
- "place), {} clears; {} entries, {} KiB",
- stats.hits, stats.misses, stats.guard_failures, stats.stores,
- stats.overwrites, stats.clears, native_memo.Size(),
+ "Macro memo: {} hits ({} on the last call's slot), {} misses, {} guard "
+ "failures, {} stores ({} in place), {} compactions, {} clears; {} entries, "
+ "{} KiB",
+ stats.hits, stats.slot_hits, stats.misses, stats.guard_failures, stats.stores,
+ stats.overwrites, stats.compactions, stats.clears, native_memo.Size(),
native_memo.ArenaWords() * 4 / 1024);
}
native_memo.Synchronize(native_code_generation);
diff --git a/src/video_core/macro.h b/src/video_core/macro.h
index a6ca4c4bd2..0d574636e9 100644
--- a/src/video_core/macro.h
+++ b/src/video_core/macro.h
@@ -9,6 +9,10 @@
#include <memory>
#include "video_core/macro_memo.h"
#include "video_core/macro_turing.h"
+
+namespace Tegra {
+class TuringMacroJit;
+}
#include <span>
#include <variant>
#include <vector>
@@ -257,6 +261,8 @@ struct MacroEngine {
// Calls replayed instead of interpreted (macro_memo.h), and what the current call wrote.
u64 native_code_generation{};
BasicMacroMemo<::Common::unordered_map<u64, u32, MacroMemoHash>> native_memo;
+ /// The macro code compiled to x86-64 (EDEN_NX2_MME_JIT=0 turns it off), created on first use.
+ std::shared_ptr<TuringMacroJit> native_jit;
MacroMemoPairs native_writes;
u64 native_calls{};
diff --git a/src/video_core/macro_memo.h b/src/video_core/macro_memo.h
index 0e994fc78a..7fe914b4f2 100644
--- a/src/video_core/macro_memo.h
+++ b/src/video_core/macro_memo.h
@@ -41,7 +41,8 @@ public:
};
struct Statistics {
- std::uint64_t hits{}, misses{}, guard_failures{}, stores{}, overwrites{}, clears{};
+ std::uint64_t hits{}, slot_hits{}, misses{}, guard_failures{}, stores{}, overwrites{},
+ compactions{}, clears{};
};
BasicMacroMemo() {
@@ -57,6 +58,7 @@ public:
if (last.first == start && last.second != NoRecord &&
Match(last.second, start, parameters, data_ram, effects)) {
++statistics.hits;
+ ++statistics.slot_hits;
return true;
}
const auto it = entries.find(Hash(start, parameters));
@@ -127,8 +129,7 @@ public:
}
}
if (arena.size() >= MaxArenaWords) {
- Clear();
- ++statistics.clears;
+ Compact();
}
const auto offset = static_cast<std::uint32_t>(arena.size());
arena.push_back(start);
@@ -186,8 +187,31 @@ private:
last_by_start.fill({0, NoRecord});
}
+ /// Copies the records still in the map to a new arena. A call stored again with another
+ /// shape leaves its old record behind: title 1 filled the arena with them every ~2 s, and
+ /// clearing it made the next ~20,000 calls interpret again. Clears only when the live
+ /// records alone would fill most of it.
+ void Compact() {
+ compacted.clear();
+ for (auto& entry : entries) {
+ const std::uint32_t* record = arena.data() + entry.second;
+ const std::size_t words =
+ HeaderWords + record[1] + std::size_t{2} * (record[2] + record[3] + record[4]);
+ entry.second = static_cast<std::uint32_t>(compacted.size());
+ compacted.insert(compacted.end(), record, record + words);
+ }
+ arena.swap(compacted);
+ last_by_start.fill({0, NoRecord});
+ ++statistics.compactions;
+ if (arena.size() >= MaxArenaWords / 4 * 3) {
+ Clear();
+ ++statistics.clears;
+ }
+ }
+
Map entries;
std::vector<std::uint32_t> arena;
+ std::vector<std::uint32_t> compacted; ///< Compact's new arena (kept for its capacity)
/// The last record found or stored per macro start (direct mapped).
std::array<std::pair<std::uint32_t, std::uint32_t>, 256> last_by_start{};
std::uint64_t generation{};
diff --git a/src/video_core/macro_turing_jit.h b/src/video_core/macro_turing_jit.h
new file mode 100644
index 0000000000..d0f24080a3
--- /dev/null
+++ b/src/video_core/macro_turing_jit.h
@@ -0,0 +1,662 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <span>
+#include <string_view>
+#include <vector>
+
+// Eden's Xbyak configuration (its hash maps): every translation unit must see the same one.
+#include "common/x64/xbyak.h"
+
+#include "video_core/macro_turing.h"
+
+namespace Tegra {
+
+// x86-64 code for the TU104/GA10x MME: every instruction of the uploaded macro memory compiled
+// once (each a block that falls through to the next one, branches and loops through a jump
+// table), with the semantics of TuringMacroInterpreter::Execute. Title 1 calls macros ~1.3
+// million times a second; the interpreter and the macro memo's lookups were ~10% of its GPU
+// thread. Emits (method writes) and register reads call back into C++.
+class TuringMacroJit {
+public:
+ using Word = std::uint32_t;
+ using Result = TuringMacroInterpreter::Result;
+
+ struct State {
+ std::array<Word, 24> regs;
+ Word carry;
+ Word method;
+ Word increment;
+ Word loop_count;
+ Word loop_start;
+ Word loop_end;
+ Word next_pc;
+ Word steps;
+ Word start;
+ Word consumed;
+ Word params_size;
+ Word fail_pc;
+ std::uint8_t end_next;
+ std::uint8_t stopped;
+ std::uint8_t cancel_end;
+ std::uint8_t pad;
+ Word scratch_x;
+ Word scratch_y;
+ const Word* params;
+ Word* dram;
+ void* user;
+ bool (*emit)(State*, Word method, Word value);
+ Word (*read)(State*, Word address);
+ const std::vector<Word>* fifo;
+ };
+
+ enum Error : Word {
+ None = 0,
+ FifoUnderflow,
+ InvalidSource,
+ InvalidDestination,
+ UnsupportedExtended,
+ InvalidMerge,
+ InvalidLoop,
+ DataRamReadOutOfBounds,
+ DataRamWriteOutOfBounds,
+ InvalidOutput,
+ UnsupportedEmit,
+ OutsideCode,
+ StepLimit,
+ };
+
+ static std::string_view ErrorText(Word error) {
+ constexpr std::array<std::string_view, 13> texts{
+ "",
+ "parameter FIFO underflow",
+ "invalid source register",
+ "invalid destination register",
+ "unsupported extended operation",
+ "invalid merge destination",
+ "invalid loop",
+ "data RAM read out of bounds",
+ "data RAM write out of bounds",
+ "invalid output selector",
+ "unsupported emitted method",
+ "instruction outside uploaded RAM",
+ "instruction limit exceeded",
+ };
+ return error < texts.size() ? texts[error] : "unknown";
+ }
+
+ /// Compiles `code` (3 words per instruction) when it differs from what was compiled.
+ void Prepare(std::span<const Word> code, std::uint64_t generation) {
+ if (program && generation == compiled_generation) {
+ return;
+ }
+ program = std::make_unique<Program>(code);
+ compiled_generation = generation;
+ }
+
+ template <typename Read, typename Emit>
+ Result Execute(Word start, std::vector<Word>& parameters, std::span<Word> data_ram, Read read,
+ Emit emit) {
+ State state;
+ state.regs.fill(0);
+ state.carry = state.method = state.increment = 0;
+ state.loop_count = state.loop_start = state.loop_end = 0;
+ state.consumed = 0;
+ state.fail_pc = 0;
+ state.end_next = state.stopped = state.cancel_end = 0;
+ state.start = start;
+ state.steps = 1000000;
+ state.params = parameters.data();
+ state.params_size = static_cast<Word>(parameters.size());
+ state.dram = data_ram.data();
+ state.fifo = &parameters;
+ Thunks<Read, Emit> thunks{read, emit};
+ state.user = &thunks;
+ state.emit = &Thunks<Read, Emit>::CallEmit;
+ state.read = &Thunks<Read, Emit>::CallRead;
+ const Word error = program->entry(&state);
+ if (error == None) {
+ return {true, state.fail_pc, state.consumed, {}};
+ }
+ return {false, state.fail_pc, state.consumed, ErrorText(error)};
+ }
+
+private:
+ template <typename Read, typename Emit>
+ struct Thunks {
+ Read& read;
+ Emit& emit;
+ static bool CallEmit(State* state, Word method, Word value) {
+ const bool ok = static_cast<Thunks*>(state->user)->emit(method, value);
+ // MME_DMA_READ_FIFOED appends to the FIFO from inside the emit.
+ state->params = state->fifo->data();
+ state->params_size = static_cast<Word>(state->fifo->size());
+ return ok;
+ }
+ static Word CallRead(State* state, Word address) {
+ return static_cast<Thunks*>(state->user)->read(address);
+ }
+ };
+
+ struct Program : Xbyak::CodeGenerator {
+ explicit Program(std::span<const Word> code)
+ : Xbyak::CodeGenerator(1 << 20, Xbyak::AutoGrow) {
+ Generate(code);
+ ready();
+ entry = getCode<Word (*)(State*)>();
+ }
+
+ Word (*entry)(State*) = nullptr;
+
+ private:
+ using Decoded = TuringMacroInterpreter::Decoded;
+#ifdef _WIN32
+ const Xbyak::Reg64 arg0 = rcx, arg1 = rdx, arg2 = r8;
+#else
+ const Xbyak::Reg64 arg0 = rdi, arg1 = rsi, arg2 = rdx;
+#endif
+ // rbx: State*; r12d/r13d: the instruction's FIFO loads; r14d/r15d: its ALU results.
+ const Xbyak::Reg64 st = rbx;
+
+ static constexpr std::size_t RegOffset(Word reg) {
+ return offsetof(State, regs) + reg * sizeof(Word);
+ }
+
+ void Generate(std::span<const Word> code) {
+ const Word count = static_cast<Word>(code.size() / 3);
+ labels.resize(count);
+ std::vector<Decoded> decoded(count);
+ for (Word pc = 0; pc < count; ++pc) {
+ decoded[pc] = TuringMacroInterpreter::Decode(
+ {code[pc * 3 + 2], code[pc * 3 + 1], code[pc * 3]});
+ }
+ Analyze(decoded);
+ push(rbx); push(rbp); push(r12); push(r13); push(r14); push(r15);
+ sub(rsp, 40);
+ mov(st, arg0);
+ mov(eax, dword[st + offsetof(State, start)]);
+ cmp(eax, count);
+ jae(fail_outside, T_NEAR);
+ lea(rcx, ptr[rip + table]);
+ jmp(ptr[rcx + rax * 8]);
+
+ for (Word pc = 0; pc < count; ++pc) {
+ L(labels[pc]);
+ Instruction(pc, count, decoded[pc]);
+ }
+
+ // Dispatch on eax (the next pc). Every backward edge comes through here, so the
+ // instruction limit is counted here.
+ L(dispatch);
+ sub(dword[st + offsetof(State, steps)], 1);
+ jz(fail_steps, T_NEAR);
+ cmp(eax, count);
+ jae(fail_outside, T_NEAR);
+ lea(rcx, ptr[rip + table]);
+ jmp(ptr[rcx + rax * 8]);
+
+ L(success);
+ xor_(eax, eax);
+ L(leave);
+ add(rsp, 40);
+ pop(r15); pop(r14); pop(r13); pop(r12); pop(rbp); pop(rbx);
+ ret();
+ const auto fail_with = [&](Xbyak::Label& label, Word error) {
+ L(label);
+ mov(eax, error);
+ jmp(leave, T_NEAR);
+ };
+ fail_with(fail_underflow, FifoUnderflow);
+ fail_with(fail_source, InvalidSource);
+ fail_with(fail_destination, InvalidDestination);
+ fail_with(fail_extended, UnsupportedExtended);
+ fail_with(fail_merge, InvalidMerge);
+ fail_with(fail_loop, InvalidLoop);
+ fail_with(fail_dread, DataRamReadOutOfBounds);
+ fail_with(fail_dwrite, DataRamWriteOutOfBounds);
+ fail_with(fail_output, InvalidOutput);
+ fail_with(fail_emit, UnsupportedEmit);
+ fail_with(fail_outside, OutsideCode);
+ fail_with(fail_steps, StepLimit);
+
+ align(8);
+ L(table);
+ for (Word pc = 0; pc < count; ++pc) {
+ putL(labels[pc]);
+ }
+ }
+
+ // Loads MME register `reg` (as an operand of `lane`) into `dst`.
+ void Get(const Xbyak::Reg32& dst, const Decoded& d, Word reg, Word lane) {
+ if (reg < 24) {
+ mov(dst, dword[st + RegOffset(reg)]);
+ return;
+ }
+ switch (reg) {
+ case 24: xor_(dst, dst); return;
+ case 25: mov(dst, Word(std::int32_t(std::int16_t(d.imm[lane])))); return;
+ case 26: mov(dst, Word(std::int32_t(std::int16_t(d.imm[1 - lane])))); return;
+ case 27: mov(dst, (d.imm[0] << 16) | d.imm[1]); return;
+ case 28: mov(dst, r12d); return;
+ case 29: mov(dst, r13d); return;
+ default: jmp(fail_source, T_NEAR); return;
+ }
+ }
+
+ static bool IsFlow(Word op) {
+ return op >= 23 && op <= 29;
+ }
+
+ // Where a loop can end (its back edge is checked only there) and which instructions can
+ // run right after an END (only those check for the end of the macro).
+ void Analyze(const std::vector<Decoded>& decoded) {
+ const Word count = static_cast<Word>(decoded.size());
+ loop_end.assign(count, false);
+ after_end.assign(count, false);
+ std::vector<Word> loop_starts_of_end(count, ~0U);
+ const auto mark = [&](std::vector<bool>& set, std::int64_t pc) {
+ if (pc >= 0 && pc < count) set[static_cast<std::size_t>(pc)] = true;
+ };
+ for (Word pc = 0; pc < count; ++pc) {
+ for (Word lane = 0; lane < 2; ++lane) {
+ const Decoded& d = decoded[pc];
+ if (d.op[lane] == 23 && d.imm[lane] >= 2 && pc + d.imm[lane] - 1 < count) {
+ loop_end[pc + d.imm[lane] - 1] = true;
+ }
+ }
+ }
+ for (Word pc = 0; pc < count; ++pc) {
+ const Decoded& d = decoded[pc];
+ if (!d.end) continue;
+ mark(after_end, std::int64_t{pc} + 1);
+ for (Word lane = 0; lane < 2; ++lane) {
+ const Word imm = d.imm[lane];
+ switch (d.op[lane]) {
+ case 23: mark(after_end, std::int64_t{pc} + imm); break;
+ case 24: mark(after_end, std::int64_t{pc} + (imm & 0x7fff)); break;
+ case 25: case 26: case 27: case 28: case 29:
+ mark(after_end, std::int64_t{pc} + (std::int32_t(imm << 19) >> 19));
+ break;
+ default: break;
+ }
+ }
+ }
+ // An END at a loop's last instruction can be followed by the loop's second one.
+ for (Word pc = 0; pc < count; ++pc) {
+ for (Word lane = 0; lane < 2; ++lane) {
+ const Decoded& d = decoded[pc];
+ if (d.op[lane] == 23 && d.imm[lane] >= 2) {
+ const Word end = pc + d.imm[lane] - 1;
+ if (end < count && decoded[end].end) mark(after_end, std::int64_t{pc} + 1);
+ }
+ }
+ }
+ }
+
+ void Instruction(Word pc, Word count, const Decoded& d) {
+ // FIFO loads, in lane order.
+ for (Word i = 0; i < 2; ++i) {
+ const Xbyak::Reg32 load = i == 0 ? r12d : r13d;
+ if (!d.load[i]) {
+ xor_(load, load);
+ continue;
+ }
+ mov(eax, dword[st + offsetof(State, consumed)]);
+ cmp(eax, dword[st + offsetof(State, params_size)]);
+ jae(fail_underflow, T_NEAR);
+ mov(rcx, qword[st + offsetof(State, params)]);
+ mov(load, dword[rcx + rax * 4]);
+ inc(eax);
+ mov(dword[st + offsetof(State, consumed)], eax);
+ }
+ if (!d.pred_mode) {
+ Body(pc, count, d, 15);
+ return;
+ }
+ Xbyak::Label when_false;
+ Get(eax, d, d.pred_reg, 0);
+ test(eax, eax);
+ jz(when_false, T_NEAR);
+ Body(pc, count, d, d.mask_true);
+ L(when_false);
+ Body(pc, count, d, d.mask_false);
+ }
+
+ void Body(Word pc, Word count, const Decoded& d, Word mask) {
+ // An inactive lane's result reads as 0 when an output selects it.
+ for (Word lane = 0; lane < 2; ++lane) {
+ if (mask & (1U << lane)) continue;
+ bool selected = false;
+ for (Word out = 0; out < 2; ++out) {
+ if (!(mask & (4U << out))) continue;
+ selected |= d.out_method[out] == lane + 1 || d.out_emit[out] == lane + 1;
+ }
+ if (selected) {
+ xor_(lane == 0 ? r14d : r15d, lane == 0 ? r14d : r15d);
+ }
+ }
+ const bool flow = ((mask & 1) && IsFlow(d.op[0])) || ((mask & 2) && IsFlow(d.op[1]));
+ if (flow) {
+ mov(dword[st + offsetof(State, next_pc)], pc + 1);
+ mov(byte[st + offsetof(State, cancel_end)], 0);
+ }
+ // Lane 1's operands are read before lane 0 commits its destination (or calls out).
+ const bool lane0_active = (mask & 1) != 0;
+ const bool early_operands =
+ (mask & 2) && lane0_active &&
+ (d.op[0] == 22 || (d.dst[0] < 24 && (d.dst[0] == d.src_a[1] || d.dst[0] == d.src_b[1])));
+ if (early_operands) {
+ Get(eax, d, d.src_a[1], 1);
+ mov(dword[st + offsetof(State, scratch_x)], eax);
+ Get(eax, d, d.src_b[1], 1);
+ mov(dword[st + offsetof(State, scratch_y)], eax);
+ }
+ for (Word lane = 0; lane < 2; ++lane) {
+ if (!(mask & (1U << lane))) {
+ continue;
+ }
+ if (lane == 1 && early_operands) {
+ mov(ecx, dword[st + offsetof(State, scratch_x)]);
+ mov(edx, dword[st + offsetof(State, scratch_y)]);
+ } else {
+ Get(ecx, d, d.src_a[lane], lane);
+ Get(edx, d, d.src_b[lane], lane);
+ }
+ Lane(pc, d, lane);
+ // eax holds the lane's value.
+ mov(lane == 0 ? r14d : r15d, eax);
+ if (d.dst[lane] < 24) {
+ mov(dword[st + RegOffset(d.dst[lane])], eax);
+ } else if (d.dst[lane] != 24) {
+ jmp(fail_destination, T_NEAR);
+ }
+ }
+ for (Word lane = 0; lane < 2; ++lane) {
+ if (!(mask & (4U << lane))) {
+ continue;
+ }
+ if (d.out_method[lane]) {
+ Output(eax, d, d.out_method[lane]);
+ mov(ecx, eax);
+ and_(ecx, 0xfff);
+ mov(dword[st + offsetof(State, method)], ecx);
+ shr(eax, 12);
+ and_(eax, 15);
+ mov(dword[st + offsetof(State, increment)], eax);
+ }
+ if (d.out_emit[lane]) {
+ Output(eax, d, d.out_emit[lane]);
+ mov(arg2.cvt32(), eax);
+ mov(arg1.cvt32(), dword[st + offsetof(State, method)]);
+ mov(arg0, st);
+ call(ptr[st + offsetof(State, emit)]);
+ test(al, al);
+ jz(fail_emit, T_NEAR);
+ mov(eax, dword[st + offsetof(State, increment)]);
+ add(dword[st + offsetof(State, method)], eax);
+ }
+ }
+ // An END in the previous instruction ends the macro after this one.
+ if (after_end[pc]) {
+ cmp(byte[st + offsetof(State, end_next)], 0);
+ jne(success, T_NEAR);
+ }
+ if (flow) {
+ cmp(byte[st + offsetof(State, stopped)], 0);
+ jne(success, T_NEAR);
+ }
+ if (d.end) {
+ if (flow) {
+ mov(al, byte[st + offsetof(State, cancel_end)]);
+ xor_(al, 1);
+ mov(byte[st + offsetof(State, end_next)], al);
+ } else {
+ mov(byte[st + offsetof(State, end_next)], 1);
+ }
+ }
+ if (loop_end[pc]) {
+ Xbyak::Label no_loop;
+ mov(eax, dword[st + offsetof(State, loop_count)]);
+ test(eax, eax);
+ jz(no_loop);
+ cmp(dword[st + offsetof(State, loop_end)], pc);
+ jne(no_loop);
+ dec(dword[st + offsetof(State, loop_count)]);
+ mov(eax, dword[st + offsetof(State, loop_start)]);
+ inc(eax);
+ jmp(dispatch, T_NEAR);
+ L(no_loop);
+ }
+ if (flow) {
+ mov(eax, dword[st + offsetof(State, next_pc)]);
+ if (pc + 1 < count) {
+ cmp(eax, pc + 1);
+ jne(dispatch, T_NEAR);
+ jmp(labels[pc + 1], T_NEAR);
+ } else {
+ jmp(dispatch, T_NEAR);
+ }
+ } else if (pc + 1 < count) {
+ jmp(labels[pc + 1], T_NEAR);
+ } else {
+ jmp(fail_outside, T_NEAR);
+ }
+ }
+
+ void Output(const Xbyak::Reg32& dst, const Decoded& d, Word selector) {
+ switch (selector) {
+ case 1: mov(dst, r14d); return;
+ case 2: mov(dst, r15d); return;
+ case 3: mov(dst, r12d); return;
+ case 4: mov(dst, r13d); return;
+ case 5: mov(dst, d.imm[0]); return;
+ case 6: mov(dst, d.imm[1]); return;
+ case 8: mov(dst, d.imm[0] >> 12); return;
+ case 9: mov(dst, d.imm[1] >> 12); return;
+ case 10: mov(dst, (d.imm[0] << 16) | d.imm[1]); return;
+ default: jmp(fail_output, T_NEAR); return;
+ }
+ }
+
+ // ecx = x, edx = y; leaves the value in eax.
+ void Compare(Word op) {
+ cmp(ecx, edx);
+ switch (op) {
+ case 17: case 25: setl(al); break;
+ case 18: case 26: setb(al); break;
+ case 19: case 27: setle(al); break;
+ case 20: case 28: setbe(al); break;
+ default: sete(al); break;
+ }
+ movzx(eax, al);
+ }
+
+ void Lane(Word pc, const Decoded& d, Word lane) {
+ const Word imm = d.imm[lane];
+ const Word op = d.op[lane];
+ switch (op) {
+ case 0:
+ mov(eax, ecx);
+ add(eax, edx);
+ setc(r8b);
+ movzx(r8d, r8b);
+ mov(dword[st + offsetof(State, carry)], r8d);
+ return;
+ case 1:
+ lea(eax, ptr[rcx + rdx]);
+ add(eax, dword[st + offsetof(State, carry)]);
+ return;
+ case 2:
+ mov(eax, ecx);
+ sub(eax, edx);
+ setc(r8b);
+ movzx(r8d, r8b);
+ mov(dword[st + offsetof(State, carry)], r8d);
+ return;
+ case 3:
+ mov(eax, ecx);
+ sub(eax, edx);
+ sub(eax, dword[st + offsetof(State, carry)]);
+ return;
+ case 4:
+ movsxd(rax, ecx);
+ movsxd(rdx, edx);
+ imul(rax, rdx);
+ mov(rcx, rax);
+ shr(rcx, 32);
+ mov(dword[st + offsetof(State, carry)], ecx);
+ return;
+ case 6:
+ mov(eax, ecx);
+ mov(edx, edx);
+ imul(rax, rdx);
+ mov(rcx, rax);
+ shr(rcx, 32);
+ mov(dword[st + offsetof(State, carry)], ecx);
+ return;
+ case 5:
+ mov(eax, dword[st + offsetof(State, carry)]);
+ return;
+ case 7:
+ cmp(ecx, 0x1000);
+ jne(fail_extended, T_NEAR);
+ xor_(eax, eax);
+ return;
+ case 8: {
+ Xbyak::Label zero, done;
+ test(ecx, ecx);
+ jz(zero);
+ bsr(eax, ecx);
+ xor_(eax, 31);
+ jmp(done);
+ L(zero);
+ mov(eax, 32);
+ L(done);
+ return;
+ }
+ case 9:
+ mov(eax, ecx);
+ mov(ecx, edx);
+ and_(ecx, 31);
+ shl(eax, cl);
+ return;
+ case 10:
+ mov(eax, ecx);
+ mov(ecx, edx);
+ and_(ecx, 31);
+ shr(eax, cl);
+ return;
+ case 11:
+ mov(eax, ecx);
+ mov(ecx, edx);
+ and_(ecx, 31);
+ sar(eax, cl);
+ return;
+ case 12: mov(eax, ecx); and_(eax, edx); return;
+ case 13: mov(eax, ecx); and_(eax, edx); not_(eax); return;
+ case 14: mov(eax, ecx); or_(eax, edx); return;
+ case 15: mov(eax, ecx); xor_(eax, edx); return;
+ case 16: {
+ const Word dest = (imm >> 10) & 63, bit_count = (imm >> 5) & 31;
+ const Word source = imm & 31;
+ if (dest >= 32) {
+ jmp(fail_merge, T_NEAR);
+ return;
+ }
+ const Word bits_mask = (1U << bit_count) - 1;
+ mov(eax, ecx);
+ and_(eax, ~(bits_mask << dest));
+ shr(edx, source);
+ and_(edx, bits_mask);
+ shl(edx, dest);
+ or_(eax, edx);
+ return;
+ }
+ case 17: case 18: case 19: case 20: case 21:
+ Compare(op);
+ neg(eax);
+ return;
+ case 22:
+ lea(eax, ptr[rcx + rdx]);
+ and_(eax, 0xffff);
+ mov(arg1.cvt32(), eax);
+ mov(arg0, st);
+ call(ptr[st + offsetof(State, read)]);
+ return;
+ case 23: {
+ if (imm < 2) {
+ jmp(fail_loop, T_NEAR);
+ return;
+ }
+ Xbyak::Label skip, done;
+ cmp(dword[st + offsetof(State, loop_count)], 0);
+ jne(fail_loop, T_NEAR);
+ test(ecx, ecx);
+ jz(skip);
+ dec(ecx);
+ mov(dword[st + offsetof(State, loop_count)], ecx);
+ mov(dword[st + offsetof(State, loop_start)], pc);
+ mov(dword[st + offsetof(State, loop_end)], pc + imm - 1);
+ jmp(done);
+ L(skip);
+ mov(dword[st + offsetof(State, next_pc)], pc + imm);
+ L(done);
+ xor_(eax, eax);
+ return;
+ }
+ case 24:
+ mov(dword[st + offsetof(State, next_pc)], pc + (imm & 0x7fff));
+ xor_(eax, eax);
+ return;
+ case 25: case 26: case 27: case 28: case 29: {
+ Xbyak::Label not_taken;
+ Compare(op);
+ cmp(eax, (imm & 0x8000) ? 1 : 0);
+ jne(not_taken);
+ const auto offset = std::int32_t(imm << 19) >> 19;
+ if (offset == -4096) {
+ mov(byte[st + offsetof(State, stopped)], 1);
+ } else {
+ mov(dword[st + offsetof(State, next_pc)], Word(std::int32_t(pc) + offset));
+ mov(byte[st + offsetof(State, cancel_end)], (imm & 0x4000) ? 1 : 0);
+ }
+ L(not_taken);
+ xor_(eax, eax);
+ return;
+ }
+ case 30:
+ cmp(ecx, 0xc00);
+ jae(fail_dread, T_NEAR);
+ mov(rax, qword[st + offsetof(State, dram)]);
+ mov(eax, dword[rax + rcx * 4]);
+ return;
+ case 31:
+ cmp(ecx, 0xc00);
+ jae(fail_dwrite, T_NEAR);
+ mov(rax, qword[st + offsetof(State, dram)]);
+ mov(dword[rax + rcx * 4], edx);
+ xor_(eax, eax);
+ return;
+ default:
+ xor_(eax, eax);
+ return;
+ }
+ }
+
+ std::vector<Xbyak::Label> labels;
+ std::vector<bool> loop_end, after_end;
+ Xbyak::Label table, dispatch, success, leave;
+ Xbyak::Label fail_underflow, fail_source, fail_destination, fail_extended, fail_merge,
+ fail_loop, fail_dread, fail_dwrite, fail_output, fail_emit, fail_outside, fail_steps;
+ };
+
+ std::unique_ptr<Program> program;
+ std::uint64_t compiled_generation{~std::uint64_t{0}};
+};
+
+} // namespace Tegra
diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h
index 105cb14e23..bd5025e9fd 100644
--- a/src/video_core/memory_manager.h
+++ b/src/video_core/memory_manager.h
@@ -165,6 +165,12 @@ public:
void FlushCaching();
+ /// Leaves the caches' invalidation of a range already written for the next FlushCaching
+ /// (before every draw, dispatch, clear and copy), merged with neighbouring ranges.
+ void AccumulateInvalidation(GPUVAddr gpu_addr, std::size_t size) {
+ accumulator.Add(gpu_addr, size);
+ }
+
const u8* GetSpan(const GPUVAddr src_addr, const std::size_t size) const;
u8* GetSpan(const GPUVAddr src_addr, const std::size_t size);
diff --git a/src/video_core/nx2_profile.h b/src/video_core/nx2_profile.h
index a162ce16a3..a921b72082 100644
--- a/src/video_core/nx2_profile.h
+++ b/src/video_core/nx2_profile.h
@@ -7,6 +7,7 @@
#include <chrono>
#include <cstdlib>
#include <string>
+#include <unordered_set>
#include "common/common_types.h"
#include "common/logging.h"
@@ -45,6 +46,11 @@ enum class Zone : u32 {
DynamicState,
DrawRecord,
TextureUpload,
+ Fence, ///< Fences made (SignalFence)
+ FenceSkipped, ///< Empty fences not made
+ Method, ///< Engine method calls (what a split GPU thread's back half would run)
+ SplitDrain, ///< Split GPU thread: the front waits for the back (gpu_split.h)
+ SplitWait, ///< Split GPU thread: the back waits for work
Count,
};
@@ -54,12 +60,15 @@ inline constexpr std::array<const char*, static_cast<size_t>(Zone::Count)> zone_
"clear", "compute", "flush work", "pipeline lookup", "configure",
"sync descriptors", "read handles", "fill image views", "texture buffers", "update buffers",
"descriptor acquire", "bind stage buffers", "push descriptors", "render targets",
- "configure draw", "dynamic state", "draw record", "texture upload",
+ "configure draw", "dynamic state", "draw record", "texture upload", "fence",
+ "fence skipped", "method", "split drain", "split wait",
};
+// Read once at startup: Enabled() runs several times per method write (no guard to check).
+inline const bool g_enabled = std::getenv("EDEN_NX2_GPU_PROFILE") != nullptr;
+
inline bool Enabled() {
- static const bool enabled = std::getenv("EDEN_NX2_GPU_PROFILE") != nullptr;
- return enabled;
+ return g_enabled;
}
// Nanoseconds; from the host's own counter where the wall clock reads it directly (the zones
@@ -71,6 +80,7 @@ inline u64 Now() {
struct ThreadData {
std::array<u64, static_cast<size_t>(Zone::Count)> exclusive_ns{};
std::array<u64, static_cast<size_t>(Zone::Count)> count{};
+ std::array<u64, 8> counters{}; ///< See Counter
u64 child_ns{}; ///< Time of the zones nested in the current one
u64 window_start{};
};
@@ -110,8 +120,34 @@ private:
u64 parent_child_ns{};
};
-/// Called by the GPU thread between commands: logs and resets the totals every 10 s.
-inline void Report() {
+/// Amounts counted on the GPU thread, logged per second with the zones.
+enum class Counter : u32 {
+ DownloadBytes, ///< Buffer data copied back from the host GPU at fences
+ DownloadCopies, ///< Buffer ranges copied back
+ DmaReadBytes, ///< Guest memory the native MME read with DMA
+ DownloadBytesOnDmaPages, ///< Of DownloadBytes: on pages the MME has DMA-read
+ MethodWrites, ///< 3D register writes (Maxwell3D::CallMethod below the macro range)
+ SameValueWrites, ///< Of MethodWrites: the register already held the value
+ AcquireChecks, ///< Semaphore acquires evaluated
+ AcquireWaits, ///< Of AcquireChecks: not met yet (the channel waits)
+};
+
+/// Device pages (4 KiB) the native MME has DMA-read since start (GPU thread only; filled only
+/// while profiling): which buffer downloads anything on the GPU thread reads back.
+inline std::unordered_set<u64>& DmaReadPages() {
+ static std::unordered_set<u64> pages;
+ return pages;
+}
+
+inline void Add(Counter counter, u64 amount) {
+ if (Enabled()) {
+ t_data.counters[static_cast<size_t>(counter)] += amount;
+ }
+}
+
+/// Called by the GPU thread (and the split's back thread) between commands: logs and resets
+/// the totals every 10 s.
+inline void Report(const char* thread_name = "GPU thread") {
if (!Enabled()) {
return;
}
@@ -142,7 +178,19 @@ inline void Report() {
const double calls = static_cast<double>(data.count[i]) * 1e9 / window;
line += fmt::format(" | {} {:.1f} ms ({:.0f}/s)", zone_names[i], ms, calls);
}
- LOG_INFO(HW_GPU, "GPU thread per second:{}", line);
+ line += fmt::format(" | buffer downloads {:.0f} KiB in {:.0f} copies ({:.0f} KiB on pages "
+ "MME DMA reads) | MME DMA reads {:.0f} KiB",
+ static_cast<double>(data.counters[0]) * 1e9 / 1024.0 / window,
+ static_cast<double>(data.counters[1]) * 1e9 / window,
+ static_cast<double>(data.counters[3]) * 1e9 / 1024.0 / window,
+ static_cast<double>(data.counters[2]) * 1e9 / 1024.0 / window);
+ line += fmt::format(" | 3D register writes {:.0f} ({:.0f} same value)",
+ static_cast<double>(data.counters[4]) * 1e9 / window,
+ static_cast<double>(data.counters[5]) * 1e9 / window);
+ line += fmt::format(" | acquires {:.0f} ({:.0f} waited)",
+ static_cast<double>(data.counters[6]) * 1e9 / window,
+ static_cast<double>(data.counters[7]) * 1e9 / window);
+ LOG_INFO(HW_GPU, "{} per second:{}", thread_name, line);
data = {};
data.window_start = now;
}
diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h
index 245ab1996c..60ca50eb1f 100644
--- a/src/video_core/query_cache/query_cache.h
+++ b/src/video_core/query_cache/query_cache.h
@@ -520,6 +520,16 @@ bool QueryCacheBase<Traits>::HasUncommittedFlushes() const {
}
template <typename Traits>
+bool QueryCacheBase<Traits>::HasPendingFenceWork() const {
+ bool result = false;
+ impl->ForEachStreamer([&result](StreamerInterface* streamer) {
+ result |= streamer->HasUnsyncedQueries() || streamer->HasPendingSync();
+ return result;
+ });
+ return result;
+}
+
+template <typename Traits>
bool QueryCacheBase<Traits>::ShouldWaitAsyncFlushes() {
std::scoped_lock lk(impl->flush_guard);
return !impl->flushes_pending.empty() && impl->flushes_pending.front() != 0ULL;
diff --git a/src/video_core/query_cache/query_cache_base.h b/src/video_core/query_cache/query_cache_base.h
index d73e9039f7..5a60e42986 100644
--- a/src/video_core/query_cache/query_cache_base.h
+++ b/src/video_core/query_cache/query_cache_base.h
@@ -109,6 +109,9 @@ public:
bool HasUncommittedFlushes() const;
+ /// Whether a new fence has queries to flush or host writes to sync for this cache.
+ bool HasPendingFenceWork() const;
+
bool ShouldWaitAsyncFlushes();
void PopAsyncFlushes();
diff --git a/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp b/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp
index 83545463ac..ecd6544b10 100644
--- a/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp
+++ b/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp
@@ -20,7 +20,9 @@
#include "video_core/shader_notify.h"
#include "video_core/texture_cache/texture_cache.h"
-#if defined(_MSC_VER) && defined(NDEBUG)
+#if defined(__clang__) && defined(NDEBUG)
+#define LAMBDA_FORCEINLINE __attribute__((always_inline))
+#elif defined(_MSC_VER) && defined(NDEBUG)
#define LAMBDA_FORCEINLINE [[msvc::forceinline]]
#else
#define LAMBDA_FORCEINLINE
diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp
index 0ced890929..9dac98c830 100644
--- a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp
+++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp
@@ -179,7 +179,14 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe
for (size_t i = 0; i < regs.rt.size(); ++i) {
color_formats[i] = static_cast<u8>(regs.rt[i].format);
}
- alpha_test_ref = std::bit_cast<u32>(regs.alpha_test_ref);
+ // The reference is part of the pipeline only while the test compares with it: title 1
+ // animates the register with the test off (text fading in and out), which made a new
+ // pipeline, translated on the GPU thread, for every value (~84% of its runtime pipelines).
+ const u32 packed_alpha_test = alpha_test_func.Value();
+ alpha_test_ref = packed_alpha_test != PackComparisonOp(Maxwell::ComparisonOp::Never_GL) &&
+ packed_alpha_test != PackComparisonOp(Maxwell::ComparisonOp::Always_GL)
+ ? std::bit_cast<u32>(regs.alpha_test_ref)
+ : 0;
point_size = std::bit_cast<u32>(regs.point_size);
if (maxwell3d.dirty.flags[Dirty::VertexInput]) {
diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
index f737ad9f70..94bace432c 100644
--- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
+++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
@@ -31,7 +31,9 @@
#include "common/settings.h"
#include "video_core/nx2_profile.h"
-#if defined(_MSC_VER) && defined(NDEBUG)
+#if defined(__clang__) && defined(NDEBUG)
+#define LAMBDA_FORCEINLINE __attribute__((always_inline))
+#elif defined(_MSC_VER) && defined(NDEBUG)
#define LAMBDA_FORCEINLINE [[msvc::forceinline]]
#else
#define LAMBDA_FORCEINLINE
diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
index 60bb040984..17f57983e3 100644
--- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
@@ -768,10 +768,66 @@ void PipelineCache::QueueVulkanPipelineCacheFlush() {
});
}
+namespace {
+// EDEN_NX2_PIPELINE_DIFF=1: for a new graphics pipeline whose shaders an earlier one uses too,
+// log which parts of the fixed state differ from that one (finds state that splits the same
+// draws into many pipelines, each translated on the GPU thread).
+void ReportNewPipelineKey(const GraphicsPipelineCacheKey& key) {
+ static const bool enabled = std::getenv("EDEN_NX2_PIPELINE_DIFF") != nullptr;
+ if (!enabled) {
+ return;
+ }
+ static std::unordered_map<u64, GraphicsPipelineCacheKey> last_by_shaders;
+ static u32 logged{};
+ const u64 shaders = Common::CityHash64(reinterpret_cast<const char*>(key.unique_hashes.data()),
+ sizeof(key.unique_hashes));
+ const auto [it, first] = last_by_shaders.try_emplace(shaders, key);
+ if (first) {
+ return;
+ }
+ const FixedPipelineState& now = key.state;
+ const FixedPipelineState& before = it->second.state;
+ std::string diffs;
+ if (now.raw1 != before.raw1) diffs += fmt::format(" raw1^{:#x}", now.raw1 ^ before.raw1);
+ if (now.raw2 != before.raw2) diffs += fmt::format(" raw2^{:#x}", now.raw2 ^ before.raw2);
+ if (now.color_formats != before.color_formats) {
+ diffs += fmt::format(" color_formats {:02x} -> {:02x}", fmt::join(before.color_formats, ""),
+ fmt::join(now.color_formats, ""));
+ }
+ if (now.alpha_test_ref != before.alpha_test_ref) {
+ diffs += fmt::format(" alpha_test_ref {:#x} -> {:#x} (func {})", before.alpha_test_ref,
+ now.alpha_test_ref, now.alpha_test_func.Value());
+ }
+ if (now.point_size != before.point_size) {
+ diffs += fmt::format(" point_size {:#x} -> {:#x}", before.point_size, now.point_size);
+ }
+ if (now.viewport_swizzles != before.viewport_swizzles) diffs += " viewport_swizzles";
+ if (now.attribute_types != before.attribute_types) {
+ diffs += fmt::format(" attribute_types^{:#x}", now.attribute_types ^ before.attribute_types);
+ }
+ if (std::memcmp(&now.dynamic_state, &before.dynamic_state, sizeof(now.dynamic_state)) != 0) {
+ diffs += " dynamic_state";
+ }
+ if (now.driver_id != before.driver_id || now.driver_version != before.driver_version) {
+ diffs += " driver";
+ }
+ const size_t size = now.Size();
+ if (diffs.empty() && size > offsetof(FixedPipelineState, attachments)) {
+ diffs = fmt::format(" past attachments (key size {})", size);
+ }
+ it->second = key;
+ if (logged++ < 400) {
+ LOG_INFO(Render_Vulkan, "New pipeline for known shaders {:016x} differs in:{}", shaders,
+ diffs.empty() ? std::string{" nothing in the key prefix"} : diffs);
+ }
+}
+} // Anonymous namespace
+
GraphicsPipeline* PipelineCache::CurrentGraphicsPipelineSlowPath() {
const auto [pair, is_new]{graphics_cache.try_emplace(graphics_key)};
auto& pipeline{pair->second};
if (is_new) {
+ ReportNewPipelineKey(graphics_key);
pipeline = CreateGraphicsPipeline();
}
if (!pipeline) {
diff --git a/src/video_core/renderer_vulkan/vk_present_manager.cpp b/src/video_core/renderer_vulkan/vk_present_manager.cpp
index 9298764546..c1724e1ab4 100644
--- a/src/video_core/renderer_vulkan/vk_present_manager.cpp
+++ b/src/video_core/renderer_vulkan/vk_present_manager.cpp
@@ -434,7 +434,7 @@ void PresentManager::CopyToSwapchainImpl(Frame* frame) {
RecreateSwapchain(frame);
}
- while (swapchain.AcquireNextImage()) {
+ while (swapchain.AcquireNextImage(!use_present_thread)) {
RecreateSwapchain(frame);
}
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index 429b2d63ef..0826820714 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -1250,20 +1250,37 @@ void RasterizerVulkan::AccelerateInlineToMemory(GPUVAddr address, size_t copy_si
return;
}
gpu_memory->WriteBlockUnsafe(address, memory.data(), copy_size);
+ // EDEN_NX2_INLINE_DEFER=0: invalidate every cache at once (for comparisons).
+ static const bool defer = [] {
+ const char* value = std::getenv("EDEN_NX2_INLINE_DEFER");
+ return value == nullptr || value[0] != '0';
+ }();
// One cache at a time: holding both while waiting for either (std::lock) waited longer, the
// fencing thread and CPU write traps holding one of them most of the time.
+ bool uploaded;
{
std::scoped_lock lock{buffer_cache.mutex};
- if (!buffer_cache.InlineMemory(*cpu_addr, copy_size, memory)) {
+ uploaded = buffer_cache.InlineMemory(*cpu_addr, copy_size, memory);
+ if (!uploaded && !defer) {
buffer_cache.WriteMemory(*cpu_addr, copy_size);
}
}
+ // Pending query results must not land over the new words later: drop them now.
+ query_cache.InvalidateRegion(*cpu_addr, copy_size);
+ if (!uploaded && defer) {
+ // Title 1 makes thousands of small inline writes between draws (~10% of the GPU
+ // thread invalidating every cache for each). Guest memory is written; the buffer,
+ // texture and shader caches see the merged ranges at the next FlushCaching, which runs
+ // before anything of theirs is used. Writes into GPU-written buffer data were uploaded
+ // in place above, as before.
+ gpu_memory->AccumulateInvalidation(address, copy_size);
+ return;
+ }
{
std::scoped_lock lock{texture_cache.mutex};
texture_cache.WriteMemory(*cpu_addr, copy_size);
}
pipeline_cache.InvalidateRegion(*cpu_addr, copy_size);
- query_cache.InvalidateRegion(*cpu_addr, copy_size);
}
std::optional<FramebufferTextureInfo> RasterizerVulkan::AccelerateDisplay(
@@ -1309,8 +1326,10 @@ void RasterizerVulkan::FlushWork() {
static constexpr u32 DRAWS_TO_DISPATCH = 512;
static constexpr u32 CHECK_MASK = 3;
#else
+ // Every 32 draws (was 8): each dispatch woke the worker thread, and Switch 2 titles make
+ // ~55,000 draws a second. A full chunk is dispatched on its own (Scheduler::Record).
static constexpr u32 DRAWS_TO_DISPATCH = 4096;
- static constexpr u32 CHECK_MASK = 7;
+ static constexpr u32 CHECK_MASK = 31;
#endif // __ANDROID__
static_assert(DRAWS_TO_DISPATCH % (CHECK_MASK + 1) == 0);
diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp
index 4a7a7d5e78..e1e989c79f 100644
--- a/src/video_core/renderer_vulkan/vk_scheduler.cpp
+++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp
@@ -55,6 +55,9 @@ Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_)
Scheduler::~Scheduler() = default;
u64 Scheduler::Flush(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) {
+ if (VideoCommon::GpuSplit::check_owner) [[unlikely]] {
+ VideoCommon::GpuSplit::CheckOwner("Scheduler::Flush");
+ }
// When flushing, we only send data to the worker thread; no waiting is necessary.
const u64 signal_value = SubmitExecution(signal_semaphore, wait_semaphore);
AllocateNewContext();
@@ -62,6 +65,9 @@ u64 Scheduler::Flush(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) {
}
void Scheduler::Finish(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) {
+ if (VideoCommon::GpuSplit::check_owner) [[unlikely]] {
+ VideoCommon::GpuSplit::CheckOwner("Scheduler::Finish");
+ }
// When finishing, we need to wait for the submission to have executed on the device.
const u64 presubmit_tick = CurrentTick();
SubmitExecution(signal_semaphore, wait_semaphore);
@@ -83,12 +89,21 @@ void Scheduler::WaitWorker() {
}
void Scheduler::DispatchWork() {
+ if (VideoCommon::GpuSplit::check_owner) [[unlikely]] {
+ VideoCommon::GpuSplit::CheckOwner("Scheduler::DispatchWork");
+ }
if (chunk && !chunk->Empty()) {
+ // Waking the worker is a system call (~2% of the GPU thread at one dispatch per 8
+ // draws): skip it while the worker is still busy, it pops the queue before waiting.
+ bool wake;
{
std::scoped_lock ql{queue_mutex};
work_queue.push(std::move(chunk));
+ wake = worker_waiting;
+ }
+ if (wake) {
+ event_cv.notify_all();
}
- event_cv.notify_all();
AcquireNewChunk();
}
}
@@ -290,7 +305,11 @@ void Scheduler::WorkerThread(std::stop_token stop_token) {
std::unique_lock lk{queue_mutex};
// Wait for work.
- event_cv.wait(lk, stop_token, [&] { return TryPopQueue(work); });
+ event_cv.wait(lk, stop_token, [&] {
+ worker_waiting = !TryPopQueue(work);
+ return !worker_waiting;
+ });
+ worker_waiting = false;
// If we've been asked to stop, we're done.
if (stop_token.stop_requested()) {
diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h
index d4bc8d0ec5..568063ddbd 100644
--- a/src/video_core/renderer_vulkan/vk_scheduler.h
+++ b/src/video_core/renderer_vulkan/vk_scheduler.h
@@ -18,6 +18,7 @@
#include "common/common_types.h"
#include "common/polyfill_thread.h"
#include "common/settings.h"
+#include "video_core/gpu_split.h"
#include "video_core/renderer_vulkan/vk_master_semaphore.h"
#include "video_core/vulkan_common/vulkan_wrapper.h"
@@ -103,6 +104,9 @@ public:
template <typename T>
requires std::is_invocable_v<T, vk::CommandBuffer, vk::CommandBuffer>
void RecordWithUploadBuffer(T&& command) {
+ if (VideoCommon::GpuSplit::check_owner) [[unlikely]] {
+ VideoCommon::GpuSplit::CheckOwner("Scheduler::Record");
+ }
if (chunk->Record(command)) {
return;
}
@@ -119,6 +123,18 @@ public:
});
}
+ /// Whether the recording thread submitted `tick` within 20 ms.
+ [[nodiscard]] bool WaitForSubmission(u64 tick) const {
+ const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds{20};
+ while (tick >= master_semaphore->CurrentTick()) {
+ if (std::chrono::steady_clock::now() >= deadline) {
+ return false;
+ }
+ std::this_thread::sleep_for(std::chrono::microseconds{100});
+ }
+ return true;
+ }
+
/// Returns the current command buffer tick.
[[nodiscard]] u64 CurrentTick() const noexcept {
return master_semaphore->CurrentTick();
@@ -129,9 +145,13 @@ public:
return master_semaphore->IsFree(tick);
}
- /// Waits for the given GPU tick, optionally pacing frames.
- void Wait(u64 tick, double target_fps = 0.0) {
- if (tick > 0) {
+ /// Waits for the given GPU tick, optionally pacing frames. Threads other than the one that
+ /// records (the async present thread) pass may_flush = false: flushing from there records
+ /// into the current chunk while that thread does (corrupt command buffers, crashes). They
+ /// wait for the recording thread to submit the tick instead, and give up after 20 ms (it may
+ /// be waiting for them).
+ void Wait(u64 tick, double target_fps = 0.0, bool may_flush = true) {
+ if (tick > 0 && (may_flush || WaitForSubmission(tick))) {
if (tick >= master_semaphore->CurrentTick()) {
Flush();
}
@@ -323,6 +343,7 @@ private:
std::mutex reserve_mutex;
std::mutex queue_mutex;
std::condition_variable_any event_cv;
+ bool worker_waiting{}; ///< The worker is waiting for work (guarded by queue_mutex)
std::jthread worker_thread;
std::chrono::steady_clock::duration frame_interval{};
diff --git a/src/video_core/renderer_vulkan/vk_swapchain.cpp b/src/video_core/renderer_vulkan/vk_swapchain.cpp
index fea5398557..a0ed5a4493 100644
--- a/src/video_core/renderer_vulkan/vk_swapchain.cpp
+++ b/src/video_core/renderer_vulkan/vk_swapchain.cpp
@@ -158,7 +158,7 @@ void Swapchain::Create(
resource_ticks.resize(image_count);
}
-bool Swapchain::AcquireNextImage() {
+bool Swapchain::AcquireNextImage(bool may_flush) {
const VkResult result = device.GetLogical().AcquireNextImageKHR(
*swapchain, (std::numeric_limits<u64>::max)(), *present_semaphores[frame_index],
VK_NULL_HANDLE, &image_index);
@@ -179,29 +179,29 @@ bool Swapchain::AcquireNextImage() {
break;
}
- const auto wait_with_frame_pacing = [this] {
+ const auto wait_with_frame_pacing = [this, may_flush] {
switch (Settings::values.frame_pacing_mode.GetValue()) {
case Settings::FramePacingMode::Target_Auto:
- scheduler.Wait(resource_ticks[image_index]);
+ scheduler.Wait(resource_ticks[image_index], 0.0, may_flush);
break;
case Settings::FramePacingMode::Target_30:
- scheduler.Wait(resource_ticks[image_index], 30.0);
+ scheduler.Wait(resource_ticks[image_index], 30.0, may_flush);
break;
case Settings::FramePacingMode::Target_60:
- scheduler.Wait(resource_ticks[image_index], 60.0);
+ scheduler.Wait(resource_ticks[image_index], 60.0, may_flush);
break;
case Settings::FramePacingMode::Target_90:
- scheduler.Wait(resource_ticks[image_index], 90.0);
+ scheduler.Wait(resource_ticks[image_index], 90.0, may_flush);
break;
case Settings::FramePacingMode::Target_120:
- scheduler.Wait(resource_ticks[image_index], 120.0);
+ scheduler.Wait(resource_ticks[image_index], 120.0, may_flush);
break;
}
};
#ifdef __ANDROID__
if (android_get_device_api_level() >= 30) {
- scheduler.Wait(resource_ticks[image_index]);
+ scheduler.Wait(resource_ticks[image_index], 0.0, may_flush);
} else {
wait_with_frame_pacing();
}
diff --git a/src/video_core/renderer_vulkan/vk_swapchain.h b/src/video_core/renderer_vulkan/vk_swapchain.h
index d926cc118a..415160046c 100644
--- a/src/video_core/renderer_vulkan/vk_swapchain.h
+++ b/src/video_core/renderer_vulkan/vk_swapchain.h
@@ -37,7 +37,8 @@ public:
u32 height);
/// Acquires the next image in the swapchain, waits as needed.
- bool AcquireNextImage();
+ /// `may_flush`: false on the async present thread (see Scheduler::Wait).
+ bool AcquireNextImage(bool may_flush = true);
/// Presents the rendered image to the swapchain.
void Present(VkSemaphore render_semaphore);
diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp
index 96306d9a27..9d189ad4f0 100644
--- a/src/video_core/shader_environment.cpp
+++ b/src/video_core/shader_environment.cpp
@@ -458,7 +458,7 @@ void GenericEnvironment::InitializeNativeCode(u32 source_header_size) {
Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets);
cached_lowest = 0;
cached_highest = static_cast<u32>((code.size() - 1) * sizeof(u64));
- LOG_INFO(Shader, "Lowered native shader at {:#x}: {} source bytes, {} instructions",
+ LOG_DEBUG(Shader, "Lowered native shader at {:#x}: {} source bytes, {} instructions",
program_base, native_source_size, lowered.instructions.size());
}
diff --git a/src/video_core/texture_cache/image_base.cpp b/src/video_core/texture_cache/image_base.cpp
index 9b0f33220d..3185dd8428 100644
--- a/src/video_core/texture_cache/image_base.cpp
+++ b/src/video_core/texture_cache/image_base.cpp
@@ -109,7 +109,9 @@ std::optional<SubresourceBase> ImageBase::TryFindBase(GPUVAddr other_addr) const
}
ImageViewId ImageBase::FindView(const ImageViewInfo& view_info) const noexcept {
- const auto it = std::ranges::find(image_view_infos, view_info);
+ const auto it = std::ranges::find_if(image_view_infos, [&view_info](const ImageViewInfo& candidate) {
+ return candidate == view_info;
+ });
if (it == image_view_infos.end()) {
return ImageViewId{};
}
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index fb8adef161..62cd523073 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -129,6 +129,12 @@ void TextureCache<P>::RunGarbageCollector() {
high_priority_mode = total_used_memory >= expected_memory;
aggressive_mode = allow_aggressive && total_used_memory >= critical_memory;
ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL;
+ // EDEN_NX2_TC_MIN_AGE=<frames>: never evict images used more recently (thrash tests).
+ static const u64 min_age = [] {
+ const char* value = std::getenv("EDEN_NX2_TC_MIN_AGE");
+ return value != nullptr ? std::strtoull(value, nullptr, 10) : 0ULL;
+ }();
+ ticks_to_destroy = (std::max)(ticks_to_destroy, min_age);
num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10);
};
const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) {
@@ -150,6 +156,7 @@ void TextureCache<P>::RunGarbageCollector() {
image.DownloadMemory(map, copies);
runtime.Finish();
SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span, swizzle_data_buffer);
+ ++gc_download_deletes;
}
if (True(image.flags & ImageFlagBits::Tracked)) {
UntrackImage(image, image_id);
@@ -253,7 +260,7 @@ void TextureCache<P>::TickFrame() {
largest.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
for (size_t i = 0; i < std::min<size_t>(12, largest.size()); ++i) {
const Image& image = *largest[i].second;
- LOG_INFO(HW_GPU, " {} MiB: {:#x} {}x{}x{} layers {} levels {} format {} type {} flags {:#x}",
+ LOG_DEBUG(HW_GPU, " {} MiB: {:#x} {}x{}x{} layers {} levels {} format {} type {} flags {:#x}",
largest[i].first >> 20, image.gpu_addr, image.info.size.width, image.info.size.height,
image.info.size.depth, image.info.resources.layers, image.info.resources.levels,
static_cast<u32>(image.info.format), static_cast<u32>(image.info.type),
@@ -263,14 +270,15 @@ void TextureCache<P>::TickFrame() {
for (const auto& [format, total] : by_format) formats.emplace_back(total.first, format);
std::sort(formats.rbegin(), formats.rend());
for (size_t i = 0; i < std::min<size_t>(8, formats.size()); ++i) {
- LOG_INFO(HW_GPU, " format {}: {} MiB in {} images", formats[i].second, formats[i].first >> 20,
+ LOG_DEBUG(HW_GPU, " format {}: {} MiB in {} images", formats[i].second, formats[i].first >> 20,
by_format[formats[i].second].second);
}
if constexpr (requires { runtime.LogMemoryStatistics(); }) {
runtime.LogMemoryStatistics();
}
- LOG_INFO(HW_GPU, "Texture cache last 600 frames: created {} images, deleted {} (collector {}, {} costly), {} async decodes, ASTC {} opaque / {} alpha",
+ LOG_INFO(HW_GPU, "Texture cache last 600 frames: created {} images, deleted {} (collector {}, {} costly, {} downloaded), {} async decodes, ASTC {} opaque / {} alpha",
images_created, images_deleted, gc_deletes + gc_costly_deletes, gc_costly_deletes,
+ gc_download_deletes,
async_decodes_queued, astc_opaque_images, astc_alpha_images);
astc_opaque_images = astc_alpha_images = 0;
std::vector<std::pair<u32, u64>> shapes;
@@ -278,13 +286,13 @@ void TextureCache<P>::TickFrame() {
std::sort(shapes.rbegin(), shapes.rend());
for (size_t i = 0; i < std::min<size_t>(10, shapes.size()); ++i) {
const u64 key = shapes[i].second;
- LOG_INFO(HW_GPU, " created {}x: format {} type {} levels {} layers {} {}x{} (last at {:#x})",
+ LOG_DEBUG(HW_GPU, " created {}x: format {} type {} levels {} layers {} {}x{} (last at {:#x})",
shapes[i].first, key >> 56, (key >> 52) & 0xF, (key >> 48) & 0xF,
(key >> 36) & 0xFFF, (key >> 18) & 0x3FFFF, key & 0x3FFFF,
created_shapes[key].second);
}
created_shapes.clear();
- gc_deletes = gc_costly_deletes = async_decodes_queued = 0;
+ gc_deletes = gc_costly_deletes = gc_download_deletes = async_decodes_queued = 0;
images_created = images_deleted = 0;
}
if (total_used_memory > minimum_memory) {
@@ -1755,7 +1763,7 @@ ImageId TextureCache<P>::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr,
storage_info.native_tiling = gpu_memory->UsesNativeTiling();
if (storage_info.native_tiling && info.type == ImageType::e3D) {
// Switch 2 3D images (lighting/fog volumes) are rare: log each one's shape.
- LOG_INFO(HW_GPU, "3D image at {:#x}: {}x{}x{} levels {} format {} block {}x{}x{}",
+ LOG_DEBUG(HW_GPU, "3D image at {:#x}: {}x{}x{} levels {} format {} block {}x{}x{}",
gpu_addr, info.size.width, info.size.height, info.size.depth, info.resources.levels,
static_cast<u32>(info.format), info.block.width, info.block.height, info.block.depth);
}
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index 11ac8a2421..2e375595de 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -125,6 +125,7 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI
// Periodic statistics (TickFrame).
u64 gc_deletes{};
u64 gc_costly_deletes{};
+ u64 gc_download_deletes{}; ///< Evictions that downloaded the image first (a full GPU wait)
u64 async_decodes_queued{};
u64 images_created{};
bool native_thresholds_applied{};
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
index 27f8fa6b4c..8bf692b00e 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
@@ -465,7 +465,7 @@ void LogMemoryStatistics(const Device& device) {
if (s.blockCount == 0) {
continue;
}
- LOG_INFO(Render_Vulkan, "VMA type {} (heap {}, flags {:#x}): {} blocks {} MiB, {} allocations {} MiB",
+ LOG_DEBUG(Render_Vulkan, "VMA type {} (heap {}, flags {:#x}): {} blocks {} MiB, {} allocations {} MiB",
type, properties->memoryTypes[type].heapIndex,
static_cast<u32>(properties->memoryTypes[type].propertyFlags), s.blockCount,
s.blockBytes >> 20, s.allocationCount, s.allocationBytes >> 20);
diff --git a/src/yuzu/CMakeLists.txt b/src/yuzu/CMakeLists.txt
index 62c2a2bd45..e6906a4ef0 100644
--- a/src/yuzu/CMakeLists.txt
+++ b/src/yuzu/CMakeLists.txt
@@ -432,8 +432,21 @@ if(UNIX AND NOT APPLE)
endif()
if (WIN32 AND NOT YUZU_USE_BUNDLED_QT)
- set(YUZU_EXE_DIR "$<TARGET_FILE_DIR:yuzu>")
- add_custom_command(TARGET yuzu POST_BUILD COMMAND ${WINDEPLOYQT_EXECUTABLE} "${YUZU_EXE_DIR}/eglantine.exe" --dir "${YUZU_EXE_DIR}" --libdir "${YUZU_EXE_DIR}" --plugindir "${YUZU_EXE_DIR}/plugins" --no-compiler-runtime --no-opengl-sw --no-system-d3d-compiler --no-translations --verbose 0)
+ # Copy the Qt DLLs and plugins next to the executable. WINDEPLOYQT_EXECUTABLE was never set, so
+ # this step ran the emulator itself with windeployqt's arguments and failed every build that
+ # uses an installed Qt (the Windows release workflow).
+ if (TARGET Qt6::windeployqt)
+ set(WINDEPLOYQT_EXECUTABLE "$<TARGET_FILE:Qt6::windeployqt>")
+ else()
+ find_program(WINDEPLOYQT_EXECUTABLE NAMES windeployqt6 windeployqt
+ HINTS "${QT_HOST_PATH}/bin" "${QT_TARGET_PATH}/bin")
+ endif()
+ if (WINDEPLOYQT_EXECUTABLE)
+ set(YUZU_EXE_DIR "$<TARGET_FILE_DIR:yuzu>")
+ add_custom_command(TARGET yuzu POST_BUILD COMMAND ${WINDEPLOYQT_EXECUTABLE} "$<TARGET_FILE:yuzu>" --dir "${YUZU_EXE_DIR}" --libdir "${YUZU_EXE_DIR}" --plugindir "${YUZU_EXE_DIR}/plugins" --no-compiler-runtime --no-opengl-sw --no-system-d3d-compiler --no-translations --verbose 0)
+ else()
+ message(WARNING "windeployqt not found: copy the Qt DLLs next to the executable yourself")
+ endif()
endif()
if (ARCHITECTURE_x86_64 OR ARCHITECTURE_arm64)
diff --git a/src/yuzu_cmd/CMakeLists.txt b/src/yuzu_cmd/CMakeLists.txt
index ca8db523ee..b44cc586f5 100644
--- a/src/yuzu_cmd/CMakeLists.txt
+++ b/src/yuzu_cmd/CMakeLists.txt
@@ -58,8 +58,14 @@ endif()
if(WIN32)
# compile as a win32 gui application instead of a console application
if(MSVC)
- # UNICODE is defined, so SDL's entry point is wmain, not main
- set_target_properties(yuzu-cmd PROPERTIES LINK_FLAGS_RELEASE "/SUBSYSTEM:WINDOWS /ENTRY:wmainCRTStartup")
+ # SDL's main implementation provides wmain when UNICODE is defined, which src/CMakeLists.txt
+ # does for cl.exe but not for clang-cl; forcing mainCRTStartup failed to link with cl.exe.
+ if(CXX_CLANG)
+ set(YUZU_CMD_ENTRY mainCRTStartup)
+ else()
+ set(YUZU_CMD_ENTRY wmainCRTStartup)
+ endif()
+ set_target_properties(yuzu-cmd PROPERTIES LINK_FLAGS_RELEASE "/SUBSYSTEM:WINDOWS /ENTRY:${YUZU_CMD_ENTRY}")
elseif(MINGW)
set_target_properties(yuzu-cmd PROPERTIES LINK_FLAGS_RELEASE "-Wl,--subsystem,windows")
endif()
diff --git a/src/yuzu_cmd/nx2_sampler.cpp b/src/yuzu_cmd/nx2_sampler.cpp
index c01b812650..1488a3a078 100644
--- a/src/yuzu_cmd/nx2_sampler.cpp
+++ b/src/yuzu_cmd/nx2_sampler.cpp
@@ -240,9 +240,9 @@ void Report(const std::vector<Target>& targets, const std::vector<Sample>& sampl
}
std::sort(sorted.rbegin(), sorted.rend());
std::fprintf(file, "-- %s\n", title);
- for (size_t i = 0; i < std::min(limit, sorted.size()); ++i) {
+ for (size_t i = 0; i < (std::min)(limit, sorted.size()); ++i) {
std::fprintf(file, "%6u %5.1f%% %s\n", sorted[i].first,
- 100.0 * sorted[i].first / std::max(total, 1U),
+ 100.0 * sorted[i].first / (std::max)(total, 1U),
sorted[i].second.c_str());
}
};
diff --git a/src/yuzu_cmd/yuzu.cpp b/src/yuzu_cmd/yuzu.cpp
index ffa1876c4f..3d9f50d4e5 100644
--- a/src/yuzu_cmd/yuzu.cpp
+++ b/src/yuzu_cmd/yuzu.cpp
@@ -434,7 +434,7 @@ static void RunKeyScript(InputCommon::InputSubsystem& input_subsystem) {
}
size_t start = 0;
while (start < list.size()) {
- const size_t end = std::min(list.find(',', start), list.size());
+ const size_t end = (std::min)(list.find(',', start), list.size());
const std::string item = list.substr(start, end - start);
if (const size_t colon = item.find(':'); colon != std::string::npos) {
const std::string when = item.substr(0, colon);