aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorTakoroka <137927095+Takoroka@users.noreply.github.com>2026-10-29 03:39:52 -0400
committerTakoroka <137927095+Takoroka@users.noreply.github.com>2026-10-29 03:39:52 -0400
commite17fd9cef116a088f080ca502951e500864a58b6 (patch)
treee7ff65f614622515d6a2e4263521909eea89a5d8 /src
parentea6c1193fbfa95e586bb6d129dc684163a12cb3e (diff)
parent80c79333fe6fd28a31b65b997f1cbf2987711c91 (diff)
Merge Switch 2 work from local Eden (nx2 at b0dd3fe)
Diffstat (limited to 'src')
-rw-r--r--src/audio_core/renderer/behavior/info_updater.cpp6
-rw-r--r--src/audio_core/sink/sink_stream.cpp15
-rw-r--r--src/audio_core/sink/sink_stream.h3
-rw-r--r--src/common/CMakeLists.txt3
-rw-r--r--src/common/cpu_features.cpp7
-rw-r--r--src/common/sparse_large_vector.cpp2
-rw-r--r--src/common/spin_recursive_mutex.h98
-rw-r--r--src/core/CMakeLists.txt5
-rw-r--r--src/core/arm/dynarmic/arm_dynarmic_64.cpp20
-rw-r--r--src/core/core.cpp2
-rw-r--r--src/core/core.h4
-rw-r--r--src/core/debugger/gdbstub.cpp23
-rw-r--r--src/core/device_memory_manager.h2
-rw-r--r--src/core/file_sys/program_metadata.cpp22
-rw-r--r--src/core/file_sys/program_metadata.h5
-rw-r--r--src/core/file_sys/vfs/vfs_real.cpp18
-rw-r--r--src/core/hardware_properties.h7
-rw-r--r--src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp2
-rw-r--r--src/core/hle/kernel/board/nintendo/nx/k_system_control.h4
-rw-r--r--src/core/hle/kernel/global_scheduler_context.cpp5
-rw-r--r--src/core/hle/kernel/k_memory_block.h4
-rw-r--r--src/core/hle/kernel/k_page_table_base.cpp3
-rw-r--r--src/core/hle/kernel/k_process.cpp7
-rw-r--r--src/core/hle/kernel/k_spin_lock.cpp48
-rw-r--r--src/core/hle/kernel/k_spin_lock.h10
-rw-r--r--src/core/hle/kernel/kernel.cpp2
-rw-r--r--src/core/hle/kernel/physical_core.cpp19
-rw-r--r--src/core/hle/kernel/physical_core.h3
-rw-r--r--src/core/hle/kernel/svc.cpp53
-rw-r--r--src/core/hle/kernel/svc/svc_info.cpp47
-rw-r--r--src/core/hle/kernel/svc/svc_physical_memory.cpp39
-rw-r--r--src/core/hle/kernel/svc/svc_thread.cpp17
-rw-r--r--src/core/hle/kernel/svc_types.h12
-rw-r--r--src/core/hle/service/am/service/application_functions.cpp20
-rw-r--r--src/core/hle/service/am/service/application_functions.h2
-rw-r--r--src/core/hle/service/am/service/application_proxy.cpp20
-rw-r--r--src/core/hle/service/am/service/application_proxy.h3
-rw-r--r--src/core/hle/service/am/service/application_proxy_service.cpp2
-rw-r--r--src/core/hle/service/am/service/graphics_scale_controller.h35
-rw-r--r--src/core/hle/service/am/service/self_controller.cpp8
-rw-r--r--src/core/hle/service/am/service/self_controller.h1
-rw-r--r--src/core/hle/service/audio/audio.cpp3
-rw-r--r--src/core/hle/service/audio/nx2_audio_user_client.h43
-rw-r--r--src/core/hle/service/hid/hid_server.cpp6
-rw-r--r--src/core/hle/service/hid/hid_server.h3
-rw-r--r--src/core/hle/service/kernel_helpers.cpp23
-rw-r--r--src/core/hle/service/kernel_helpers.h3
-rw-r--r--src/core/hle/service/ns/platform_service_manager.cpp56
-rw-r--r--src/core/hle/service/ns/platform_service_manager.h3
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp141
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h21
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp124
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h5
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp104
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_gpu.h2
-rw-r--r--src/core/hle/service/nvdrv/devices/nx2_stub_device.h204
-rw-r--r--src/core/hle/service/nvdrv/nvdrv.cpp15
-rw-r--r--src/core/hle/service/nvdrv/nvdrv.h2
-rw-r--r--src/core/hle/service/nvdrv/nvdrv_interface.cpp9
-rw-r--r--src/core/hle/service/nvdrv/nvdrv_interface.h3
-rw-r--r--src/core/hle/service/nx2_gsv.h1647
-rw-r--r--src/core/hle/service/nx2_stub_interface.h55
-rw-r--r--src/core/hle/service/service.cpp92
-rw-r--r--src/core/launch_params.cpp2
-rw-r--r--src/core/loader/deconstructed_rom_directory.cpp42
-rw-r--r--src/core/loader/loader.cpp7
-rw-r--r--src/core/loader/nso.cpp1
-rw-r--r--src/core/memory.cpp14
-rw-r--r--src/core/memory.h3
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc18
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp3
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h11
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp190
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/translate/impl/memory_tagging.cpp98
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp73
-rw-r--r--src/dynarmic/src/dynarmic/interface/exclusive_monitor.h2
-rw-r--r--src/shader_recompiler/CMakeLists.txt3
-rw-r--r--src/shader_recompiler/environment.h2
-rw-r--r--src/shader_recompiler/frontend/ampere/lower_to_maxwell.h2249
-rw-r--r--src/shader_recompiler/frontend/ampere/program_address_map.h29
-rw-r--r--src/shader_recompiler/frontend/ampere/program_capture.h54
-rw-r--r--src/shader_recompiler/frontend/ampere/program_override.h85
-rw-r--r--src/shader_recompiler/frontend/ir/breadth_first_search.h22
-rw-r--r--src/shader_recompiler/frontend/maxwell/control_flow.cpp2
-rw-r--r--src/shader_recompiler/frontend/maxwell/translate_program.cpp1
-rw-r--r--src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp22
-rw-r--r--src/shader_recompiler/ir_opt/passes.h1
-rw-r--r--src/shader_recompiler/ir_opt/position_pass.cpp66
-rw-r--r--src/video_core/CMakeLists.txt6
-rw-r--r--src/video_core/buffer_cache/buffer_cache.h33
-rw-r--r--src/video_core/buffer_cache/buffer_cache_base.h6
-rw-r--r--src/video_core/buffer_cache/memory_tracker_base.h2
-rw-r--r--src/video_core/buffer_cache/usage_tracker.h17
-rw-r--r--src/video_core/control/channel_state.cpp30
-rw-r--r--src/video_core/control/scheduler.cpp32
-rw-r--r--src/video_core/control/scheduler.h8
-rw-r--r--src/video_core/dirty_flags.cpp1
-rw-r--r--src/video_core/dma_pusher.cpp278
-rw-r--r--src/video_core/dma_pusher.h61
-rw-r--r--src/video_core/engines/engine_upload.cpp20
-rw-r--r--src/video_core/engines/engine_upload.h3
-rw-r--r--src/video_core/engines/fermi_2d.cpp22
-rw-r--r--src/video_core/engines/kepler_compute.cpp578
-rw-r--r--src/video_core/engines/kepler_compute.h46
-rw-r--r--src/video_core/engines/maxwell_3d.cpp68
-rw-r--r--src/video_core/engines/maxwell_3d.h41
-rw-r--r--src/video_core/engines/maxwell_dma.cpp41
-rw-r--r--src/video_core/engines/puller.cpp216
-rw-r--r--src/video_core/engines/puller.h23
-rw-r--r--src/video_core/fence_manager.h64
-rw-r--r--src/video_core/framebuffer_config.h1
-rw-r--r--src/video_core/gpu.cpp12
-rw-r--r--src/video_core/gpu.h7
-rw-r--r--src/video_core/gpu_thread.cpp44
-rw-r--r--src/video_core/gpu_thread.h11
-rw-r--r--src/video_core/host1x/gpu_device_memory_manager.h4
-rw-r--r--src/video_core/host_shaders/CMakeLists.txt1
-rw-r--r--src/video_core/host_shaders/astc_decoder.comp8
-rw-r--r--src/video_core/host_shaders/bc_encoder.comp183
-rw-r--r--src/video_core/host_shaders/block_linear_unswizzle_2d.comp8
-rw-r--r--src/video_core/host_shaders/block_linear_unswizzle_3d.comp8
-rw-r--r--src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp10
-rw-r--r--src/video_core/macro.cpp172
-rw-r--r--src/video_core/macro.h27
-rw-r--r--src/video_core/macro_memo.h197
-rw-r--r--src/video_core/macro_turing.h274
-rw-r--r--src/video_core/memory_manager.cpp32
-rw-r--r--src/video_core/memory_manager.h15
-rw-r--r--src/video_core/native_qmd.h133
-rw-r--r--src/video_core/native_query_trace.h43
-rw-r--r--src/video_core/nx2.h143
-rw-r--r--src/video_core/nx2_profile.h150
-rw-r--r--src/video_core/query_cache/query_cache.h16
-rw-r--r--src/video_core/renderer_opengl/gl_rasterizer.cpp2
-rw-r--r--src/video_core/renderer_opengl/util_shaders.cpp1
-rw-r--r--src/video_core/renderer_vulkan/present/layer.cpp2
-rw-r--r--src/video_core/renderer_vulkan/vk_compute_pass.cpp164
-rw-r--r--src/video_core/renderer_vulkan/vk_compute_pass.h34
-rw-r--r--src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp116
-rw-r--r--src/video_core/renderer_vulkan/vk_pipeline_cache.cpp37
-rw-r--r--src/video_core/renderer_vulkan/vk_rasterizer.cpp330
-rw-r--r--src/video_core/renderer_vulkan/vk_texture_cache.cpp285
-rw-r--r--src/video_core/renderer_vulkan/vk_texture_cache.h22
-rw-r--r--src/video_core/shader_cache.cpp121
-rw-r--r--src/video_core/shader_cache.h17
-rw-r--r--src/video_core/shader_environment.cpp206
-rw-r--r--src/video_core/shader_environment.h11
-rw-r--r--src/video_core/texture_cache/accelerated_swizzle.cpp2
-rw-r--r--src/video_core/texture_cache/accelerated_swizzle.h2
-rw-r--r--src/video_core/texture_cache/format_lookup_table.cpp1
-rw-r--r--src/video_core/texture_cache/image_base.h6
-rw-r--r--src/video_core/texture_cache/image_info.cpp15
-rw-r--r--src/video_core/texture_cache/image_info.h14
-rw-r--r--src/video_core/texture_cache/pending_download.h58
-rw-r--r--src/video_core/texture_cache/texture_cache.h385
-rw-r--r--src/video_core/texture_cache/texture_cache_base.h36
-rw-r--r--src/video_core/texture_cache/util.cpp62
-rw-r--r--src/video_core/texture_cache/util.h6
-rw-r--r--src/video_core/textures/astc.cpp65
-rw-r--r--src/video_core/textures/astc.h3
-rw-r--r--src/video_core/textures/decoders.cpp33
-rw-r--r--src/video_core/textures/decoders.h8
-rw-r--r--src/video_core/textures/workers.cpp14
-rw-r--r--src/video_core/vulkan_common/vulkan_device.cpp35
-rw-r--r--src/video_core/vulkan_common/vulkan_device.h5
-rw-r--r--src/video_core/vulkan_common/vulkan_memory_allocator.cpp80
-rw-r--r--src/video_core/vulkan_common/vulkan_memory_allocator.h7
-rw-r--r--src/video_core/vulkan_common/vulkan_wrapper.h5
-rw-r--r--src/yuzu_cmd/CMakeLists.txt2
-rw-r--r--src/yuzu_cmd/emu_window/emu_window_sdl3.cpp30
-rw-r--r--src/yuzu_cmd/emu_window/emu_window_sdl3.h9
-rw-r--r--src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp2
-rw-r--r--src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp2
-rw-r--r--src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp2
-rw-r--r--src/yuzu_cmd/nx2_sampler.cpp333
-rw-r--r--src/yuzu_cmd/nx2_sampler.h16
-rw-r--r--src/yuzu_cmd/yuzu.cpp164
177 files changed, 11720 insertions, 297 deletions
diff --git a/src/audio_core/renderer/behavior/info_updater.cpp b/src/audio_core/renderer/behavior/info_updater.cpp
index a88eae91ad..ca3eaece94 100644
--- a/src/audio_core/renderer/behavior/info_updater.cpp
+++ b/src/audio_core/renderer/behavior/info_updater.cpp
@@ -588,7 +588,11 @@ Result InfoUpdater::UpdateBehaviorInfo(BehaviorInfo& behaviour_) {
return Service::Audio::ResultInvalidUpdateInfo;
}
- if (in_params->revision != behaviour_.GetUserRevision()) {
+ // Compare revision numbers: Switch 2 opens the renderer with 'REV?' but tags its updates
+ // 'REv?' (lowercase v) for the same revision.
+ if (GetRevisionNum(in_params->revision) != GetRevisionNum(behaviour_.GetUserRevision())) {
+ LOG_ERROR(Service_Audio, "BehaviorInfo revision {:08X} does not match the renderer's {:08X}",
+ in_params->revision, behaviour_.GetUserRevision());
return Service::Audio::ResultInvalidUpdateInfo;
}
diff --git a/src/audio_core/sink/sink_stream.cpp b/src/audio_core/sink/sink_stream.cpp
index c70020e9da..9dfd3bf747 100644
--- a/src/audio_core/sink/sink_stream.cpp
+++ b/src/audio_core/sink/sink_stream.cpp
@@ -4,8 +4,10 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <algorithm>
#include <array>
#include <atomic>
+#include <cstdlib>
#include <memory>
#include <span>
#include <vector>
@@ -213,6 +215,19 @@ void SinkStream::ProcessAudioOutAndRender(std::span<s16> output_buffer, std::siz
std::memcpy(last_frame.data(), &output_buffer[(frames_written - 1) * frame_size], frame_size_bytes);
+ // EDEN_NX2_AUDIO_LEVEL: log the output peak about every two seconds (is a game audible?).
+ if (static const bool meter = std::getenv("EDEN_NX2_AUDIO_LEVEL") != nullptr; meter) {
+ for (size_t i = 0; i < num_frames * frame_size; ++i) {
+ level_peak = std::max<s32>(level_peak, std::abs(static_cast<s32>(output_buffer[i])));
+ }
+ level_frames += num_frames;
+ if (level_frames >= TargetSampleRate * 2) {
+ LOG_INFO(Audio, "Audio output peak over {} frames: {} of 32767", level_frames, level_peak);
+ level_frames = 0;
+ level_peak = 0;
+ }
+ }
+
{
std::scoped_lock lk{sample_count_lock};
last_sample_count_update_time = system.CoreTiming().GetGlobalTimeNs();
diff --git a/src/audio_core/sink/sink_stream.h b/src/audio_core/sink/sink_stream.h
index acb7537dda..280b7747ec 100644
--- a/src/audio_core/sink/sink_stream.h
+++ b/src/audio_core/sink/sink_stream.h
@@ -245,6 +245,9 @@ private:
SinkBuffer playing_buffer{};
/// The last played (or received) frame of audio, used when the callback underruns
std::array<s16, MaxChannels> last_frame{};
+ /// EDEN_NX2_AUDIO_LEVEL meter state.
+ s32 level_peak{};
+ u64 level_frames{};
/// Number of buffers waiting to be played
std::atomic<u32> queued_buffers{};
/// The ring size for audio out buffers (usually 4, rarely 2 or 8)
diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
index 5d1b30a549..d3e7b03cd2 100644
--- a/src/common/CMakeLists.txt
+++ b/src/common/CMakeLists.txt
@@ -96,6 +96,7 @@ add_library(
${CMAKE_CURRENT_BINARY_DIR}/scm_rev.cpp
scm_rev.h
scope_exit.h
+ spin_recursive_mutex.h
scratch_buffer.h
settings.cpp
settings.h
@@ -151,7 +152,7 @@ add_library(
set_source_files_properties(zbic_compression.cpp PROPERTIES
INCLUDE_DIRECTORIES "${ZBIC_INCLUDE_DIR}"
- COMPILE_OPTIONS "$<$<CXX_COMPILER_ID:Clang,GNU>:-Wno-unused-function;-Wno-missing-declarations;-Wno-shadow>")
+ COMPILE_OPTIONS "$<$<CXX_COMPILER_ID:Clang,GNU>:-Wno-unused-function;-Wno-missing-declarations;-Wno-shadow>;$<$<CXX_COMPILER_ID:MSVC>:/wd4505;/wd5245>")
if(WIN32)
target_sources(common PRIVATE windows/timer_resolution.cpp
diff --git a/src/common/cpu_features.cpp b/src/common/cpu_features.cpp
index 8ccb6847f7..52fd003c44 100644
--- a/src/common/cpu_features.cpp
+++ b/src/common/cpu_features.cpp
@@ -225,6 +225,13 @@ const CPUCaps g_cpu_caps = [] {
}
}
+ // AMD CPUs have no leaf 0x15 (Zen 4 stops at 0x10): measure their invariant TSC too, or every
+ // guest clock read (CNTPCT, which games read thousands of times a frame) goes through the
+ // host's much slower QueryPerformanceCounter.
+ if (caps.invariant_tsc && caps.tsc_frequency == 0) {
+ caps.tsc_frequency = X64::EstimateRDTSCFrequency();
+ }
+
if (max_std_fn >= 0x16) {
__cpuid(cpu_id, 0x16);
caps.base_frequency = cpu_id[0];
diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp
index eee8ff6d18..b96340999d 100644
--- a/src/common/sparse_large_vector.cpp
+++ b/src/common/sparse_large_vector.cpp
@@ -26,7 +26,7 @@ static std::mutex vector_regions_mutex {};
// Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error
static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) {
DWORD code = info->ExceptionRecord->ExceptionCode;
- u64 exception_addr = reinterpret_cast<u64>(info->ExceptionRecord->ExceptionInformation[1]);
+ u64 exception_addr = static_cast<u64>(info->ExceptionRecord->ExceptionInformation[1]);
if (code != EXCEPTION_ACCESS_VIOLATION || info->ExceptionRecord->ExceptionInformation[0] == 1) {
// Not our problem
diff --git a/src/common/spin_recursive_mutex.h b/src/common/spin_recursive_mutex.h
new file mode 100644
index 0000000000..291dd4e41e
--- /dev/null
+++ b/src/common/spin_recursive_mutex.h
@@ -0,0 +1,98 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <atomic>
+#include <cstdint>
+#include <thread>
+
+#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)
+#include <immintrin.h>
+#endif
+
+namespace Common {
+
+/// A recursive mutex that spins briefly before sleeping. The GPU caches' locks are taken tens of
+/// thousands of times a second by the GPU thread, the fencing thread and CPU write traps, and
+/// held for microseconds: std::recursive_mutex put a waiting thread to sleep at once, and its
+/// wake-up took longer than the critical section (title 1's GPU thread spent ~8% asleep there).
+class SpinRecursiveMutex {
+public:
+ void lock() {
+ const std::thread::id self = std::this_thread::get_id();
+ if (owner.load(std::memory_order_relaxed) == self) {
+ ++depth;
+ return;
+ }
+ std::uint32_t expected = 0;
+ if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire,
+ std::memory_order_relaxed)) {
+ LockContended();
+ }
+ owner.store(self, std::memory_order_relaxed);
+ depth = 1;
+ }
+
+ bool try_lock() {
+ const std::thread::id self = std::this_thread::get_id();
+ if (owner.load(std::memory_order_relaxed) == self) {
+ ++depth;
+ return true;
+ }
+ std::uint32_t expected = 0;
+ if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire,
+ std::memory_order_relaxed)) {
+ return false;
+ }
+ owner.store(self, std::memory_order_relaxed);
+ depth = 1;
+ return true;
+ }
+
+ void unlock() {
+ if (--depth != 0) {
+ return;
+ }
+ owner.store(std::thread::id{}, std::memory_order_relaxed);
+ if (state.exchange(0, std::memory_order_release) == 2) {
+ state.notify_one();
+ }
+ }
+
+private:
+ // About 10 microseconds of spinning before sleeping.
+ static constexpr int SpinCount = 1024;
+
+ static void Pause() {
+#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)
+ _mm_pause();
+#elif defined(__aarch64__)
+ asm volatile("yield");
+#endif
+ }
+
+ void LockContended() {
+ for (int spin = 0; spin < SpinCount; ++spin) {
+ Pause();
+ if (state.load(std::memory_order_relaxed) == 0) {
+ std::uint32_t expected = 0;
+ if (state.compare_exchange_weak(expected, 1, std::memory_order_acquire,
+ std::memory_order_relaxed)) {
+ return;
+ }
+ }
+ }
+ // 2: held with (possible) sleepers; taken this way it stays marked, so its unlock wakes
+ // the next sleeper.
+ while (state.exchange(2, std::memory_order_acquire) != 0) {
+ state.wait(2, std::memory_order_relaxed);
+ }
+ }
+
+ std::atomic<std::uint32_t> state{0}; ///< 0 free, 1 held, 2 held with sleepers
+ std::atomic<std::thread::id> owner{};
+ std::uint32_t depth{}; ///< Recursion depth, touched only by the owner
+};
+
+} // namespace Common
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index 1b578f67ba..56fa653738 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -457,6 +457,7 @@ add_library(core STATIC
hle/service/am/service/application_functions.cpp
hle/service/am/service/application_functions.h
hle/service/am/service/application_proxy.cpp
+ hle/service/am/service/graphics_scale_controller.h
hle/service/am/service/application_proxy.h
hle/service/am/service/application_proxy_service.cpp
hle/service/am/service/application_proxy_service.h
@@ -526,6 +527,7 @@ add_library(core STATIC
hle/service/audio/audio_out.h
hle/service/audio/audio_renderer_manager.cpp
hle/service/audio/audio_renderer_manager.h
+ hle/service/audio/nx2_audio_user_client.h
hle/service/audio/audio_renderer.cpp
hle/service/audio/audio_renderer.h
hle/service/audio/audio.cpp
@@ -865,6 +867,7 @@ add_library(core STATIC
hle/service/nvdrv/devices/nvhost_vic.h
hle/service/nvdrv/devices/nvmap.cpp
hle/service/nvdrv/devices/nvmap.h
+ hle/service/nvdrv/devices/nx2_stub_device.h
hle/service/nvdrv/nvdata.h
hle/service/nvdrv/nvdrv.cpp
hle/service/nvdrv/nvdrv.h
@@ -1033,6 +1036,8 @@ add_library(core STATIC
hle/service/server_manager.h
hle/service/service.cpp
hle/service/service.h
+ hle/service/nx2_stub_interface.h
+ hle/service/nx2_gsv.h
hle/service/services.cpp
hle/service/services.h
hle/service/set/factory_settings_server.cpp
diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp
index 894857a86d..c2bb05afba 100644
--- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp
+++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp
@@ -4,6 +4,7 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <atomic>
#include "common/settings.h"
#include "core/arm/dynarmic/arm_dynarmic.h"
#include "core/arm/dynarmic/arm_dynarmic_64.h"
@@ -24,13 +25,24 @@ DynarmicCallbacks64::DynarmicCallbacks64(ArmDynarmic64& parent, Kernel::KProcess
u64 DynarmicCallbacks64::MemoryRead(u64 vaddr, size_t size) {
CheckMemoryAccess(vaddr, size, Kernel::DebugWatchpointType::Read);
+ u64 value{};
switch (size) {
- case sizeof(u64): return m_memory.Read64(vaddr);
- case sizeof(u32): return m_memory.Read32(vaddr);
- case sizeof(u16): return m_memory.Read16(vaddr);
- case sizeof(u8): return m_memory.Read8(vaddr);
+ case sizeof(u64): value = m_memory.Read64(vaddr); break;
+ case sizeof(u32): value = m_memory.Read32(vaddr); break;
+ case sizeof(u16): value = m_memory.Read16(vaddr); break;
+ case sizeof(u8): value = m_memory.Read8(vaddr); break;
default: UNREACHABLE();
}
+ // With memory aborts ignored the guest keeps running on zeros: say where it read from.
+ if (auto& unmapped = Core::Memory::UnmappedReadFlag(); unmapped != 0) {
+ unmapped = 0;
+ static std::atomic<u32> logged{};
+ if (logged.fetch_add(1) < 32) {
+ LOG_ERROR(Core_ARM, "Unmapped read {:#x} ({} bytes) at pc {:#x} lr {:#x} sp {:#x}", vaddr, size,
+ m_parent.m_jit->GetPC(), m_parent.m_jit->GetRegister(30), m_parent.m_jit->GetSP());
+ }
+ }
+ return value;
}
Dynarmic::A64::Vector DynarmicCallbacks64::MemoryRead128(u64 vaddr) {
CheckMemoryAccess(vaddr, 16, Kernel::DebugWatchpointType::Read);
diff --git a/src/core/core.cpp b/src/core/core.cpp
index 06543b0833..eee91ef4fc 100644
--- a/src/core/core.cpp
+++ b/src/core/core.cpp
@@ -68,6 +68,8 @@
namespace Core {
+std::atomic<u64> g_nx2_present_count{};
+
FileSys::VirtualFile GetGameFileFromPath(const FileSys::VirtualFilesystem& vfs,
const std::string& path) {
// To account for split 00+01+etc files.
diff --git a/src/core/core.h b/src/core/core.h
index d88ed64151..0276679d76 100644
--- a/src/core/core.h
+++ b/src/core/core.h
@@ -6,6 +6,7 @@
#pragma once
+#include <atomic>
#include <cstddef>
#include <deque>
#include <functional>
@@ -115,6 +116,9 @@ class RenderdocAPI;
namespace Core {
+/// Frames the guest has presented (Switch 2 vi2 present calls); input replay is locked to it.
+extern std::atomic<u64> g_nx2_present_count;
+
class CpuManager;
class Debugger;
class DeviceMemory;
diff --git a/src/core/debugger/gdbstub.cpp b/src/core/debugger/gdbstub.cpp
index 44ddf11c39..71af2acee3 100644
--- a/src/core/debugger/gdbstub.cpp
+++ b/src/core/debugger/gdbstub.cpp
@@ -25,7 +25,10 @@
#include "core/debugger/gdbstub_arch.h"
#include "core/hle/kernel/k_page_table.h"
#include "core/hle/kernel/k_process.h"
+#include "core/hle/kernel/k_event.h"
+#include "core/hle/kernel/k_readable_event.h"
#include "core/hle/kernel/k_thread.h"
+#include "core/hle/service/kernel_helpers.h"
#include "core/loader/loader.h"
#include "core/memory.h"
@@ -731,8 +734,26 @@ void GDBStub::HandleRcmd(const std::vector<u8>& command) {
break;
cur_addr = next_address;
}
+ } else if (command_str == "handles" || command_str == "get handles") {
+ // Every handle of the process: its object type and, for service events, their name.
+ reply = "Handles:\n";
+ auto& handle_table = debug_process->GetHandleTable();
+ for (size_t i = 0; i < handle_table.GetTableSize(); ++i) {
+ Kernel::Handle handle{};
+ auto object = handle_table.GetObjectByIndex(system.Kernel(), std::addressof(handle), i);
+ if (object.IsNull()) {
+ continue;
+ }
+ std::string name;
+ if (auto* readable = object->DynamicCast<Kernel::KReadableEvent*>(); readable != nullptr) {
+ name = Service::KernelHelpers::ServiceContext::GetEventName(readable->GetParent());
+ } else if (auto* event = object->DynamicCast<Kernel::KEvent*>(); event != nullptr) {
+ name = Service::KernelHelpers::ServiceContext::GetEventName(event);
+ }
+ reply += fmt::format(" {:#010x} {} {}\n", handle, object->GetTypeObj().GetName(), name);
+ }
} else {
- reply += "Commands: fastmem, info, mappings\n";
+ reply += "Commands: fastmem, info, mappings, handles\n";
}
std::span<const u8> reply_span{reinterpret_cast<u8*>(&reply.front()), reply.size()};
diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h
index b4b3b46088..9ed0b6f5e5 100644
--- a/src/core/device_memory_manager.h
+++ b/src/core/device_memory_manager.h
@@ -139,7 +139,7 @@ private:
static constexpr size_t device_virtual_bits = Traits::device_virtual_bits;
static constexpr size_t device_as_size = 1ULL << device_virtual_bits;
static constexpr size_t physical_min_bits = 32;
- static constexpr size_t physical_max_bits = 33;
+ static constexpr size_t physical_max_bits = 35; // covers the 12 GiB layout plus nx2 graphics server memory
static constexpr size_t page_bits = 12;
static constexpr size_t page_size = 1ULL << page_bits;
static constexpr size_t page_mask = page_size - 1ULL;
diff --git a/src/core/file_sys/program_metadata.cpp b/src/core/file_sys/program_metadata.cpp
index 63ca9d3fe3..464e496ea3 100644
--- a/src/core/file_sys/program_metadata.cpp
+++ b/src/core/file_sys/program_metadata.cpp
@@ -5,6 +5,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <cstddef>
+#include <cstring>
#include <vector>
#include "common/logging.h"
@@ -33,13 +34,30 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) {
return Loader::ResultStatus::ErrorBadACIDHeader;
}
+ // Switch 2 ACID blocks have a 0x300-byte signature/key area instead of 0x200, so the
+ // header starts 0x100 bytes later. Offsets inside the header are unchanged.
+ if (std::memcmp(acid_header.magic.data(), "ACID", 4) != 0) {
+ constexpr std::size_t acid_sig_size_v1 = 0x200;
+ constexpr std::size_t acid_sig_size_v2 = 0x300;
+ constexpr std::size_t acid_body_size = sizeof(AcidHeader) - acid_sig_size_v1;
+ auto* const body = reinterpret_cast<u8*>(&acid_header) + acid_sig_size_v1;
+ if (file->ReadBytes(body, acid_body_size, npdm_header.acid_offset + acid_sig_size_v2) !=
+ acid_body_size ||
+ std::memcmp(acid_header.magic.data(), "ACID", 4) != 0) {
+ return Loader::ResultStatus::ErrorBadACIDHeader;
+ }
+ LOG_INFO(Loader, "NPDM uses the Switch 2 ACID layout");
+ is_switch2 = true;
+ }
+
if (sizeof(AciHeader) != file->ReadObject(&aci_header, npdm_header.aci_offset)) {
return Loader::ResultStatus::ErrorBadACIHeader;
}
// Load acid_file_access per-component instead of the entire struct, since this struct does not
// reflect the layout of the real data.
- std::size_t current_offset = acid_header.fac_offset;
+ // FAC/FAH offsets are relative to the start of the ACID/ACI block, like the KAC offsets.
+ std::size_t current_offset = npdm_header.acid_offset + acid_header.fac_offset;
if (sizeof(FileAccessControl::version) != file->ReadBytes(&acid_file_access.version,
sizeof(FileAccessControl::version),
current_offset)) {
@@ -57,7 +75,7 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) {
}
// Load aci_file_access per-component instead of the entire struct, same as acid_file_access
- current_offset = aci_header.fah_offset;
+ current_offset = npdm_header.aci_offset + aci_header.fah_offset;
if (sizeof(FileAccessHeader::version) != file->ReadBytes(&aci_file_access.version,
sizeof(FileAccessHeader::version),
current_offset)) {
diff --git a/src/core/file_sys/program_metadata.h b/src/core/file_sys/program_metadata.h
index 115e6d6cd1..5ce32af1a4 100644
--- a/src/core/file_sys/program_metadata.h
+++ b/src/core/file_sys/program_metadata.h
@@ -72,6 +72,10 @@ public:
KernelCapabilityDescriptors capabilities);
bool Is64BitProgram() const;
+ /// True when the NPDM uses the Switch 2 ACID layout.
+ bool IsSwitch2() const {
+ return is_switch2;
+ }
ProgramAddressSpaceType GetAddressSpaceType() const;
u8 GetMainThreadPriority() const;
u8 GetMainThreadCore() const;
@@ -179,6 +183,7 @@ private:
Header npdm_header{};
AciHeader aci_header{};
AcidHeader acid_header{};
+ bool is_switch2{};
FileAccessControl acid_file_access{};
FileAccessHeader aci_file_access{};
diff --git a/src/core/file_sys/vfs/vfs_real.cpp b/src/core/file_sys/vfs/vfs_real.cpp
index 14e0caad3c..fd36752fee 100644
--- a/src/core/file_sys/vfs/vfs_real.cpp
+++ b/src/core/file_sys/vfs/vfs_real.cpp
@@ -19,6 +19,10 @@
// For FileTimeStampRaw
#include <sys/stat.h>
+#ifdef _WIN32
+#include <cstdio> // _getmaxstdio
+#endif
+
#ifdef _MSC_VER
#define stat _stat64
#endif
@@ -33,7 +37,17 @@ namespace FS = Common::FS;
namespace {
-constexpr size_t MaxOpenFiles = 8192;
+// Files the cache keeps open. On Windows it has to stay below the C runtime's open-file limit with
+// room to spare: at the limit every other open fails (saves, shader caches, and the cache's own
+// next file, which it only evicts for once it reaches its cap).
+size_t MaxOpenFiles() {
+#ifdef _WIN32
+ static const size_t limit = static_cast<size_t>((std::max)(_getmaxstdio() - 512, 64));
+ return (std::min)(size_t{8192}, limit);
+#else
+ return 8192;
+#endif
+}
bool IsWithinRoot(std::string_view root, std::string_view full_path) {
if (root.empty())
@@ -251,7 +265,7 @@ void RealVfsFilesystem::DropReference(std::unique_ptr<FileReference>&& reference
}
void RealVfsFilesystem::EvictSingleReferenceLocked() {
- if (num_open_files < MaxOpenFiles || open_references.empty()) {
+ if (num_open_files < MaxOpenFiles() || open_references.empty()) {
return;
}
diff --git a/src/core/hardware_properties.h b/src/core/hardware_properties.h
index 3f870becbf..aeea5f858f 100644
--- a/src/core/hardware_properties.h
+++ b/src/core/hardware_properties.h
@@ -15,11 +15,14 @@ namespace Hardware {
constexpr u64 BASE_CLOCK_RATE = 1'020'000'000; // Default CPU Frequency = 1020 MHz
constexpr u64 CNTFREQ = 19'200'000; // CNTPCT_EL0 Frequency = 19.2 MHz
-constexpr u32 NUM_CPU_CORES = 4; // Number of CPU Cores - sync with dynarmic exclusive_monitor.h
+// Switch 1 applications get cores 0..3. Switch 2 applications ask for 0..5 (title 4's npdm), and
+// its job system starts a worker per core, so emulate six: with four, cores 0 and 1 each ran two
+// of its workers. Switch 1 titles leave cores 4 and 5 idle.
+constexpr u32 NUM_CPU_CORES = 6; // Number of CPU Cores - sync with dynarmic exclusive_monitor.h
// Virtual to Physical core map.
constexpr std::array<s32, Common::BitSize<u64>()> VirtualToPhysicalCoreMap{
- 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3,
};
diff --git a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp
index 3343d1d282..ba4737e25f 100644
--- a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp
+++ b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp
@@ -94,7 +94,7 @@ size_t KSystemControl::Init::GetIntendedMemorySize() {
case Smc::MemorySize_10GB:
return 10_GiB;
case Smc::MemorySize_12GB:
- return 12_GiB;
+ return 12_GiB + KSystemControl::Nx2GraphicsServerMemorySize;
}
}
diff --git a/src/core/hle/kernel/board/nintendo/nx/k_system_control.h b/src/core/hle/kernel/board/nintendo/nx/k_system_control.h
index 41a25ba1c8..cc5572bde8 100644
--- a/src/core/hle/kernel/board/nintendo/nx/k_system_control.h
+++ b/src/core/hle/kernel/board/nintendo/nx/k_system_control.h
@@ -21,6 +21,10 @@ public:
// This can be overridden as needed.
static constexpr size_t SecureAppletMemorySize = 4 * 1024 * 1024; // 4_MB
+ // Extra DRAM on the 12 GB layout for memory the Switch 2 graphics server keeps itself (nx2:gpu
+ // 214 GPU-only regions). It sits in the non-secure system pool, outside the application pool.
+ static constexpr size_t Nx2GraphicsServerMemorySize = 8ULL * 1024 * 1024 * 1024; // 8_GiB
+
public:
class Init {
public:
diff --git a/src/core/hle/kernel/global_scheduler_context.cpp b/src/core/hle/kernel/global_scheduler_context.cpp
index 6d1cd030e0..98d14c8549 100644
--- a/src/core/hle/kernel/global_scheduler_context.cpp
+++ b/src/core/hle/kernel/global_scheduler_context.cpp
@@ -40,12 +40,15 @@ void GlobalSchedulerContext::RemoveThread(KThread* thread) noexcept {
/// @note This operation happens every 10ms.
void GlobalSchedulerContext::PreemptThreads(KernelCore& kernel) noexcept {
// The priority levels at which the global scheduler preempts threads every 10 ms. They are
- // ordered from Core 0 to Core 3.
+ // ordered from Core 0 to Core 5; core 3 is shared with the system, 4 and 5 are application
+ // cores like 0..2.
static constexpr std::array<u32, Core::Hardware::NUM_CPU_CORES> per_core{
59,
59,
59,
63,
+ 59,
+ 59,
};
ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel));
for (u32 core_id = 0; core_id < per_core.size(); core_id++)
diff --git a/src/core/hle/kernel/k_memory_block.h b/src/core/hle/kernel/k_memory_block.h
index f6543bdb3a..2abca31d87 100644
--- a/src/core/hle/kernel/k_memory_block.h
+++ b/src/core/hle/kernel/k_memory_block.h
@@ -195,7 +195,9 @@ enum class KMemoryAttribute : u8 {
Uncached = static_cast<u8>(Svc::MemoryAttribute::Uncached),
PermissionLocked = static_cast<u8>(Svc::MemoryAttribute::PermissionLocked),
- SetMask = Uncached | PermissionLocked,
+ GpuAccessible = static_cast<u8>(Svc::MemoryAttribute::GpuAccessible),
+
+ SetMask = Uncached | PermissionLocked | GpuAccessible,
};
DECLARE_ENUM_FLAG_OPERATORS(KMemoryAttribute);
diff --git a/src/core/hle/kernel/k_page_table_base.cpp b/src/core/hle/kernel/k_page_table_base.cpp
index 2a0a7d49d5..fd5d2c2dce 100644
--- a/src/core/hle/kernel/k_page_table_base.cpp
+++ b/src/core/hle/kernel/k_page_table_base.cpp
@@ -2102,7 +2102,8 @@ Result KPageTableBase::SetMemoryAttribute(KProcessAddress addr, size_t size, KMe
constexpr KMemoryAttribute AttributeTestMask =
~(KMemoryAttribute::SetMask | KMemoryAttribute::DeviceShared);
const KMemoryState state_test_mask =
- (True(mask & KMemoryAttribute::Uncached) ? KMemoryState::FlagCanChangeAttribute
+ (True(mask & (KMemoryAttribute::Uncached | KMemoryAttribute::GpuAccessible))
+ ? KMemoryState::FlagCanChangeAttribute
: KMemoryState::None) |
(True(mask & KMemoryAttribute::PermissionLocked) ? KMemoryState::FlagCanPermissionLock
: KMemoryState::None);
diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp
index 27d1b1bfc4..4aac818b77 100644
--- a/src/core/hle/kernel/k_process.cpp
+++ b/src/core/hle/kernel/k_process.cpp
@@ -1261,6 +1261,13 @@ Result KProcess::LoadFromMetadata(KernelCore& kernel, const FileSys::ProgramMeta
// Assign remaining properties.
m_ideal_core_id = metadata.GetMainThreadCore();
+ // Switch 2 applications get more memory than a 64-bit heap region holds (nninitStartup sizes
+ // its heap as total memory minus ~6.3 GiB), so count the alias region as well, bounded by the
+ // pool through the resource limit.
+ if (metadata.IsSwitch2()) {
+ m_max_process_memory = m_page_table.GetHeapRegionSize() + m_page_table.GetAliasRegionSize();
+ }
+
// Set up emulation context.
this->InitializeInterfaces(kernel);
diff --git a/src/core/hle/kernel/k_spin_lock.cpp b/src/core/hle/kernel/k_spin_lock.cpp
index 8525320374..107d6f1aa3 100644
--- a/src/core/hle/kernel/k_spin_lock.cpp
+++ b/src/core/hle/kernel/k_spin_lock.cpp
@@ -3,18 +3,60 @@
#include "core/hle/kernel/k_spin_lock.h"
+#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)
+#include <immintrin.h>
+#endif
+
namespace Kernel {
+namespace {
+
+// Spins of about 30-140 cycles each before a contended Lock sleeps (a few microseconds).
+constexpr int SpinCount = 128;
+
+void Pause() {
+#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)
+ _mm_pause();
+#elif defined(__aarch64__)
+ asm volatile("yield");
+#endif
+}
+
+} // Anonymous namespace
+
void KSpinLock::Lock() {
- m_lock.lock();
+ u32 expected = 0;
+ if (m_state.compare_exchange_strong(expected, 1, std::memory_order_acquire,
+ std::memory_order_relaxed)) {
+ return;
+ }
+ for (int spin = 0; spin < SpinCount; ++spin) {
+ Pause();
+ if (m_state.load(std::memory_order_relaxed) == 0) {
+ expected = 0;
+ if (m_state.compare_exchange_weak(expected, 1, std::memory_order_acquire,
+ std::memory_order_relaxed)) {
+ return;
+ }
+ }
+ }
+ // Mark the lock contended and sleep until it is released; taken this way it stays marked,
+ // so its Unlock wakes the next sleeper.
+ while (m_state.exchange(2, std::memory_order_acquire) != 0) {
+ m_state.wait(2, std::memory_order_relaxed);
+ }
}
void KSpinLock::Unlock() {
- m_lock.unlock();
+ if (m_state.exchange(0, std::memory_order_release) == 2) {
+ m_state.notify_one();
+ }
}
bool KSpinLock::TryLock() {
- return m_lock.try_lock();
+ u32 expected = 0;
+ return m_state.compare_exchange_strong(expected, 1, std::memory_order_acquire,
+ std::memory_order_relaxed);
}
} // namespace Kernel
diff --git a/src/core/hle/kernel/k_spin_lock.h b/src/core/hle/kernel/k_spin_lock.h
index 094a1e6be0..f7bc8b4d05 100644
--- a/src/core/hle/kernel/k_spin_lock.h
+++ b/src/core/hle/kernel/k_spin_lock.h
@@ -3,9 +3,11 @@
#pragma once
-#include <mutex>
+#include <atomic>
+#include <mutex> // Kernel headers that use std::mutex get it through this one
#include "common/common_funcs.h"
+#include "common/common_types.h"
#include "core/hle/kernel/k_scoped_lock.h"
namespace Kernel {
@@ -22,7 +24,11 @@ public:
bool TryLock();
private:
- std::mutex m_lock;
+ // 0 free, 1 held, 2 held with (possible) sleepers. The scheduler lock is taken by every
+ // core thousands of times a frame and held for microseconds: a contended Lock spins briefly
+ // before sleeping, instead of putting the host thread to sleep at once and paying its
+ // wake-up latency (std::mutex did).
+ std::atomic<u32> m_state{0};
};
// TODO(bunnei): Alias for now, in case we want to implement these accurately in the future.
diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp
index 4f1d849209..b437dd4b4e 100644
--- a/src/core/hle/kernel/kernel.cpp
+++ b/src/core/hle/kernel/kernel.cpp
@@ -171,7 +171,7 @@ struct KernelCore::Impl {
schedulers[core_id].reset();
}
- // Next host thead ID to use, 0-3 IDs represent core threads, >3 represent others
+ // Next host thead ID to use, IDs below NUM_CPU_CORES represent core threads, the rest others
next_host_thread_id = Core::Hardware::NUM_CPU_CORES;
// Close kernel objects that were not freed on shutdown
diff --git a/src/core/hle/kernel/physical_core.cpp b/src/core/hle/kernel/physical_core.cpp
index c5236c64e3..c3ae7629aa 100644
--- a/src/core/hle/kernel/physical_core.cpp
+++ b/src/core/hle/kernel/physical_core.cpp
@@ -4,6 +4,9 @@
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <chrono>
+#include <cstdlib>
+#include <thread>
#include "common/scope_exit.h"
#include "common/settings.h"
#include "core/core.h"
@@ -222,8 +225,22 @@ void PhysicalCore::LogBacktrace(KernelCore& kernel) {
}
void PhysicalCore::Idle() {
+ // Guest job systems hand work between cores many times per frame; blocking at once adds the
+ // host's thread wake-up latency to every handoff. EDEN_NX2_IDLE_SPIN_US=<n>: poll for that
+ // many microseconds before blocking.
+ static const auto spin = std::chrono::microseconds{[] {
+ const char* text = std::getenv("EDEN_NX2_IDLE_SPIN_US");
+ return text ? std::strtoll(text, nullptr, 10) : 0LL;
+ }()};
+ if (spin.count() > 0) {
+ const auto deadline = std::chrono::steady_clock::now() + spin;
+ while (!m_is_interrupted.load(std::memory_order_acquire) &&
+ std::chrono::steady_clock::now() < deadline) {
+ std::this_thread::yield();
+ }
+ }
std::unique_lock lk{m_guard};
- m_on_interrupt.wait(lk, [this] { return m_is_interrupted; });
+ m_on_interrupt.wait(lk, [this] { return m_is_interrupted.load(); });
}
bool PhysicalCore::IsInterrupted() const {
diff --git a/src/core/hle/kernel/physical_core.h b/src/core/hle/kernel/physical_core.h
index b92201204a..20371d6717 100644
--- a/src/core/hle/kernel/physical_core.h
+++ b/src/core/hle/kernel/physical_core.h
@@ -6,6 +6,7 @@
#pragma once
+#include <atomic>
#include <condition_variable>
#include <cstddef>
#include <memory>
@@ -71,7 +72,7 @@ private:
std::condition_variable m_on_interrupt;
Core::ArmInterface* m_arm_interface{};
KThread* m_current_thread{};
- bool m_is_interrupted{};
+ std::atomic<bool> m_is_interrupted{};
bool m_is_single_core{};
};
diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp
index 3542d5a3b6..53d964ecbc 100644
--- a/src/core/hle/kernel/svc.cpp
+++ b/src/core/hle/kernel/svc.cpp
@@ -6,6 +6,8 @@
// This file is automatically generated using svc_generator.py.
// DO NOT MODIFY IT MANUALLY
+#include <cstdlib>
+#include <cstring>
#include <type_traits>
#include "core/arm/arm_interface.h"
@@ -2505,11 +2507,60 @@ void Call(Core::System& system, u32 imm) {
LOG_TRACE(Kernel_SVC, "{} [0]={:#x} [1]={:#x} [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x} [6]={:#x}",
imm, GetArg64(args, 0), GetArg64(args, 1), GetArg64(args, 2),
GetArg64(args, 3), GetArg64(args, 4), GetArg64(args, 5), GetArg64(args, 6));
- if (process.Is64Bit()) {
+ // Set EDEN_NX2_SVC_TRACE=1 to log every SVC with its arguments and results, or
+ // EDEN_NX2_SVC_TRACE=<first>:<last> for presents first..last only (a frame-time profile).
+ static const std::pair<u64, u64> trace_range = [] {
+ const char* text = std::getenv("EDEN_NX2_SVC_TRACE");
+ if (text == nullptr) return std::pair<u64, u64>{1, 0};
+ const char* colon = std::strchr(text, ':');
+ if (colon == nullptr) return std::pair<u64, u64>{0, ~u64{}};
+ return std::pair<u64, u64>{std::strtoull(text, nullptr, 10), std::strtoull(colon + 1, nullptr, 10)};
+ }();
+ const u64 presents = Core::g_nx2_present_count.load(std::memory_order_relaxed);
+ const bool svc_trace = presents >= trace_range.first && presents <= trace_range.second;
+ if (svc_trace) {
+ LOG_INFO(Kernel_SVC, "svc {:#04x} in t{} [0]={:#x} [1]={:#x} [2]={:#x} [3]={:#x}", imm,
+ GetCurrentThread(kernel).GetThreadId(),
+ GetArg64(args, 0), GetArg64(args, 1), GetArg64(args, 2), GetArg64(args, 3));
+ }
+ if (imm == 0x80) {
+ // Switch 2 only: svcSetGpuMemoryAttribute(address, size, mask, attr). nn::gpu::sdk and
+ // nn::gs native memory set the new "GPU accessible" attribute (0x20) before registering
+ // memory with the GPU and clear it on destruction; NVN can see it through QueryMemory.
+ // Same checks as svcSetMemoryAttribute, restricted to that bit (as in changes.patch).
+ const u64 address = GetArg64(args, 0);
+ const u64 size = GetArg64(args, 1);
+ const u32 mask = static_cast<u32>(GetArg64(args, 2));
+ const u32 attr = static_cast<u32>(GetArg64(args, 3));
+ constexpr u32 Supported = static_cast<u32>(Svc::MemoryAttribute::GpuAccessible);
+ auto& page_table = process.GetPageTable();
+ Result result = ResultSuccess;
+ if (!Common::IsAligned(address, PageSize)) {
+ result = ResultInvalidAddress;
+ } else if (size == 0 || !Common::IsAligned(size, PageSize)) {
+ result = ResultInvalidSize;
+ } else if ((attr & ~mask) != 0 || ((mask | attr) & ~Supported) != 0) {
+ result = ResultInvalidCombination;
+ } else if (address + size <= address || !page_table.Contains(address, size)) {
+ result = ResultInvalidCurrentMemory;
+ } else {
+ result = page_table.SetMemoryAttribute(address, size,
+ static_cast<KMemoryAttribute>(mask),
+ static_cast<KMemoryAttribute>(attr));
+ }
+ LOG_DEBUG(Kernel_SVC, "Switch 2 svc 0x80 addr={:#x} size={:#x} mask={:#x} attr={:#x} -> {:#x}",
+ address, size, mask, attr, result.raw);
+ args[0] = result.raw;
+ } else if (process.Is64Bit()) {
Call64(system, imm, args);
} else {
Call32(system, imm, args);
}
+ if (svc_trace) {
+ LOG_INFO(Kernel_SVC, "svc {:#04x} out t{} [0]={:#x} [1]={:#x} [2]={:#x}", imm,
+ GetCurrentThread(kernel).GetThreadId(), GetArg64(args, 0), GetArg64(args, 1),
+ GetArg64(args, 2));
+ }
//kernel.ExitSVCProfile();
kernel.CurrentPhysicalCore().LoadSvcArguments(process, args);
}
diff --git a/src/core/hle/kernel/svc/svc_info.cpp b/src/core/hle/kernel/svc/svc_info.cpp
index 863bb58fa8..bbd7d32445 100644
--- a/src/core/hle/kernel/svc/svc_info.cpp
+++ b/src/core/hle/kernel/svc/svc_info.cpp
@@ -4,6 +4,8 @@
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <cstdlib>
+
#include "core/core.h"
#include "core/core_timing.h"
#include "core/hle/kernel/k_memory_manager.h"
@@ -40,6 +42,11 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle
case InfoType::UsedNonSystemMemorySize:
case InfoType::IsApplication:
case InfoType::FreeThreadCount:
+ case InfoType::Nx2RegionAddress:
+ case InfoType::Nx2RegionSize:
+ case InfoType::Nx2VammSize:
+ case InfoType::Nx2RemoteMemorySize:
+ case InfoType::Nx2RemoteMemoryUsed:
case InfoType::AliasRegionExtraSize: {
R_UNLESS(info_sub_id == 0, ResultInvalidEnumValue);
@@ -66,6 +73,35 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle
case InfoType::HeapRegionSize:
*result = process->GetPageTable().GetHeapRegionSize();
R_SUCCEED();
+ case InfoType::Nx2RegionAddress:
+ case InfoType::Nx2RegionSize: {
+ // Looks like the region Switch 2 VAMM reserves address space in (a title asks for a
+ // ~7.5 GiB reservation). VAMM backs it with MapPhysicalMemory, which only works in
+ // the alias region here, so hand out the upper half of the alias region.
+ auto& page_table = process->GetPageTable();
+ const u64 half = page_table.GetAliasRegionSize() / 2;
+ *result = info_id_type == InfoType::Nx2RegionAddress
+ ? GetInteger(page_table.GetAliasRegionStart()) + half
+ : half;
+ LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType {}, returning {:#x}",
+ static_cast<u32>(info_id_type), *result);
+ R_SUCCEED();
+ }
+ case InfoType::Nx2RemoteMemorySize:
+ // Unknown real value; nothing backs it yet.
+ *result = 0;
+ LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType 31 (remote memory size), returning 0");
+ R_SUCCEED();
+ case InfoType::Nx2RemoteMemoryUsed:
+ *result = 0;
+ LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType 32 (remote memory used), returning 0");
+ R_SUCCEED();
+ case InfoType::Nx2VammSize:
+ // A page granule: the SDK checks module addresses/sizes against (value - 1) and
+ // VAMM maps one granule up front. 4 KiB matches what Switch 1 kernels use.
+ *result = Core::Memory::YUZU_PAGESIZE;
+ LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType 33, returning {:#x}", *result);
+ R_SUCCEED();
case InfoType::AslrRegionAddress:
*result = GetInteger(process->GetPageTable().GetAliasCodeRegionStart());
R_SUCCEED();
@@ -310,7 +346,16 @@ Result GetSystemInfo(Core::System& system, uint64_t* out, SystemInfoType info_ty
Result GetInfo64(Core::System& system, uint64_t* out, InfoType info_type, Handle handle,
uint64_t info_subtype) {
- R_RETURN(GetInfo(system, out, info_type, handle, info_subtype));
+ const Result result = GetInfo(system, out, info_type, handle, info_subtype);
+ // EDEN_NX2_MEM_TRACE: log the memory layout and size queries of a title (diagnostics).
+ static const bool trace = std::getenv("EDEN_NX2_MEM_TRACE") != nullptr;
+ const u32 type = static_cast<u32>(info_type);
+ if (static u32 logs{}; trace && logs < 200 && (type <= 7 || (type >= 12 && type <= 33))) {
+ ++logs;
+ LOG_INFO(Kernel_SVC, "mem trace: GetInfo type {} sub {:#x} -> {:#x} = {:#x}", type,
+ info_subtype, result.raw, *out);
+ }
+ R_RETURN(result);
}
Result GetSystemInfo64(Core::System& system, uint64_t* out, SystemInfoType info_type, Handle handle,
diff --git a/src/core/hle/kernel/svc/svc_physical_memory.cpp b/src/core/hle/kernel/svc/svc_physical_memory.cpp
index 48915b2b1e..4fde85fdaa 100644
--- a/src/core/hle/kernel/svc/svc_physical_memory.cpp
+++ b/src/core/hle/kernel/svc/svc_physical_memory.cpp
@@ -4,6 +4,8 @@
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <cstdlib>
+
#include "core/core.h"
#include "core/hle/kernel/k_process.h"
#include "core/hle/kernel/svc.h"
@@ -142,16 +144,47 @@ Result SetUnsafeLimit(Core::System& system, uint64_t limit) {
R_THROW(ResultNotImplemented);
}
+// EDEN_NX2_MEM_TRACE: log the heap and physical memory requests of a title (diagnostics).
+static bool MemTrace() {
+ static const bool trace = std::getenv("EDEN_NX2_MEM_TRACE") != nullptr;
+ return trace;
+}
+
Result SetHeapSize64(Core::System& system, uint64_t* out_address, uint64_t size) {
- R_RETURN(SetHeapSize(system, out_address, size));
+ const Result result = SetHeapSize(system, out_address, size);
+ if (MemTrace()) {
+ LOG_INFO(Kernel_SVC, "mem trace: SetHeapSize {:#x} -> {:#x} at {:#x}", size, result.raw,
+ *out_address);
+ }
+ R_RETURN(result);
}
Result MapPhysicalMemory64(Core::System& system, uint64_t address, uint64_t size) {
- R_RETURN(MapPhysicalMemory(system, address, size));
+ const Result result = MapPhysicalMemory(system, address, size);
+ static u64 total{};
+ if (MemTrace()) {
+ if (result.IsSuccess()) {
+ total += size;
+ }
+ static u32 logs{};
+ if (result.IsError() || logs++ < 400) {
+ LOG_INFO(Kernel_SVC, "mem trace: MapPhysicalMemory {:#x}+{:#x} -> {:#x} (mapped total {:#x})",
+ address, size, result.raw, total);
+ }
+ }
+ R_RETURN(result);
}
Result UnmapPhysicalMemory64(Core::System& system, uint64_t address, uint64_t size) {
- R_RETURN(UnmapPhysicalMemory(system, address, size));
+ const Result result = UnmapPhysicalMemory(system, address, size);
+ if (MemTrace()) {
+ static u32 logs{};
+ if (result.IsError() || logs++ < 400) {
+ LOG_INFO(Kernel_SVC, "mem trace: UnmapPhysicalMemory {:#x}+{:#x} -> {:#x}", address,
+ size, result.raw);
+ }
+ }
+ R_RETURN(result);
}
Result MapPhysicalMemoryUnsafe64(Core::System& system, uint64_t address, uint64_t size) {
diff --git a/src/core/hle/kernel/svc/svc_thread.cpp b/src/core/hle/kernel/svc/svc_thread.cpp
index dbca13dd26..7f6b959256 100644
--- a/src/core/hle/kernel/svc/svc_thread.cpp
+++ b/src/core/hle/kernel/svc/svc_thread.cpp
@@ -20,6 +20,18 @@ constexpr bool IsValidVirtualCoreId(int32_t core_id) {
return (0 <= core_id && core_id < static_cast<int32_t>(Core::Hardware::NUM_CPU_CORES));
}
+// Switch 2 hardware has eight cores. Six are emulated (the most an application is given), so
+// fold any id past them back onto the low cores, both in core ids and in affinity masks.
+constexpr int32_t FoldExtraCoreId(int32_t core_id) {
+ constexpr auto num_cores = static_cast<int32_t>(Core::Hardware::NUM_CPU_CORES);
+ return core_id >= num_cores && core_id < 2 * num_cores ? core_id - num_cores : core_id;
+}
+
+constexpr u64 FoldExtraCoreMask(u64 mask) {
+ constexpr u64 low = (1ULL << Core::Hardware::NUM_CPU_CORES) - 1;
+ return (mask & low) | ((mask >> Core::Hardware::NUM_CPU_CORES) & low);
+}
+
} // Anonymous namespace
/// Creates a new thread
@@ -36,6 +48,7 @@ Result CreateThread(Core::System& system, Handle* out_handle, u64 entry_point, u
if (core_id == IdealCoreUseProcessValue) {
core_id = process.GetIdealCoreId();
}
+ core_id = FoldExtraCoreId(core_id);
// Validate arguments.
R_UNLESS(IsValidVirtualCoreId(core_id), ResultInvalidCoreId);
@@ -261,6 +274,10 @@ Result SetThreadCoreMask(Core::System& system, Handle thread_handle, s32 core_id
R_UNLESS((affinity_mask | process_core_mask) == process_core_mask, ResultInvalidCoreId);
R_UNLESS(affinity_mask != 0, ResultInvalidCombination);
+ // Validated against the real mask above; fold to the emulated cores from here on.
+ affinity_mask = FoldExtraCoreMask(affinity_mask);
+ core_id = FoldExtraCoreId(core_id);
+
// Validate the core id.
if (IsValidVirtualCoreId(core_id)) {
R_UNLESS(((1ULL << core_id) & affinity_mask) != 0, ResultInvalidCombination);
diff --git a/src/core/hle/kernel/svc_types.h b/src/core/hle/kernel/svc_types.h
index b0f4796da4..0bf2aaaa2d 100644
--- a/src/core/hle/kernel/svc_types.h
+++ b/src/core/hle/kernel/svc_types.h
@@ -50,6 +50,7 @@ enum class MemoryAttribute : u32 {
DeviceShared = (1 << 2),
Uncached = (1 << 3),
PermissionLocked = (1 << 4),
+ GpuAccessible = (1 << 5), // Switch 2 (svc 0x80)
};
DECLARE_ENUM_FLAG_OPERATORS(MemoryAttribute);
@@ -157,6 +158,17 @@ enum class InfoType : u32 {
IsSvcPermitted = 26,
IoRegionHint = 27,
AliasRegionExtraSize = 28,
+ // Switch 2 kernels: address/size of a region the SDK queries between the alias and
+ // ASLR regions during nnosInitialize; likely the VAMM address-space region.
+ Nx2RegionAddress = 29,
+ Nx2RegionSize = 30,
+ // Switch 2 kernels: nn::os::GetRemoteMemoryUsage returns {id 31, id 32}; used by
+ // nn::gsv/nn::nvsrv BootWithRemoteMemory (memory set aside for the graphics stack).
+ Nx2RemoteMemorySize = 31,
+ Nx2RemoteMemoryUsed = 32,
+ // Switch 2 kernels: a page granule. The SDK aligns module ranges and VAMM's first
+ // MapPhysicalMemory to it.
+ Nx2VammSize = 33,
TransferMemoryHint = 34,
diff --git a/src/core/hle/service/am/service/application_functions.cpp b/src/core/hle/service/am/service/application_functions.cpp
index 90d292c4a2..d4f7cd6041 100644
--- a/src/core/hle/service/am/service/application_functions.cpp
+++ b/src/core/hle/service/am/service/application_functions.cpp
@@ -118,7 +118,8 @@ IApplicationFunctions::IApplicationFunctions(Core::System& system_, std::shared_
{220, nullptr, "Unknown220"}, // [20.0.0+]
{300, nullptr, "CreateMovieWriter"}, // [19.0.0+]
{310, nullptr, "Unknown310"}, // [20.0.0+]
- {320, nullptr, "Unknown320"}, // [20.0.0+]
+ {320, D<&IApplicationFunctions::Unknown320>, "Unknown320"}, // [20.0.0+]
+ {321, D<&IApplicationFunctions::Unknown321>, "Unknown321"}, // Switch 2
{330, D<&IApplicationFunctions::Unknown330>, "Unknown330"}, // [20.0.0+]
{500, nullptr, "StartContinuousRecordingFlushForDebug"},
{1000, nullptr, "CreateMovieMaker"},
@@ -352,10 +353,11 @@ Result IApplicationFunctions::GetPseudoDeviceId(Out<Common::UUID> out_pseudo_dev
// This should be hashed with the device specific hash
// for now this will do
const auto res = FileSys::PatchManager::GetMetadataFromBaseOrUpdate(system, m_applet->program_id);
- R_UNLESS(res.first != nullptr, ResultUnknown);
std::array<u8, EVP_MAX_MD_SIZE> hash;
unsigned int hash_len = 0;
- auto const seed = res.first->raw.seed_for_pseudo_device_id;
+ // Loose ExeFS boots have no control.nacp; seed from the program id instead of failing.
+ auto const seed = res.first != nullptr ? res.first->raw.seed_for_pseudo_device_id
+ : m_applet->program_id;
EVP_MD_CTX *ctx = EVP_MD_CTX_new();
auto const algorithm = EVP_sha1();
EVP_DigestInit_ex(ctx, algorithm, nullptr);
@@ -544,6 +546,18 @@ Result IApplicationFunctions::GetUnknownEvent210(
R_SUCCEED();
}
+Result IApplicationFunctions::Unknown320() {
+ // Called by Switch 2 titles during nn::oe::Initialize; arguments unknown.
+ LOG_WARNING(Service_AM, "(STUBBED) called");
+ R_SUCCEED();
+}
+
+Result IApplicationFunctions::Unknown321() {
+ // Called by Switch 2 titles right after 320; arguments unknown.
+ LOG_WARNING(Service_AM, "(STUBBED) called");
+ R_SUCCEED();
+}
+
Result IApplicationFunctions::Unknown330(Out<u8> out) {
LOG_DEBUG(Service_AM, "called");
*out = 0;
diff --git a/src/core/hle/service/am/service/application_functions.h b/src/core/hle/service/am/service/application_functions.h
index b97ed7e25e..72bf8998b1 100644
--- a/src/core/hle/service/am/service/application_functions.h
+++ b/src/core/hle/service/am/service/application_functions.h
@@ -82,6 +82,8 @@ private:
Result GetHealthWarningDisappearedSystemEvent(OutCopyHandle<Kernel::KReadableEvent> out_event);
Result CreateApplicationAndRequestToStart(u64 application_id);
Result GetUnknownEvent210(OutCopyHandle<Kernel::KReadableEvent> out_event);
+ Result Unknown320();
+ Result Unknown321();
Result Unknown330(Out<u8> out);
Result PrepareForJit();
diff --git a/src/core/hle/service/am/service/application_proxy.cpp b/src/core/hle/service/am/service/application_proxy.cpp
index df438e1e17..430f3c6888 100644
--- a/src/core/hle/service/am/service/application_proxy.cpp
+++ b/src/core/hle/service/am/service/application_proxy.cpp
@@ -11,11 +11,13 @@
#include "core/hle/service/am/service/common_state_getter.h"
#include "core/hle/service/am/service/debug_functions.h"
#include "core/hle/service/am/service/display_controller.h"
+#include "core/hle/service/am/service/graphics_scale_controller.h"
#include "core/hle/service/am/service/library_applet_creator.h"
#include "core/hle/service/am/service/process_winding_controller.h"
#include "core/hle/service/am/service/self_controller.h"
#include "core/hle/service/am/service/window_controller.h"
#include "core/hle/service/cmif_serialization.h"
+#include "core/hle/service/nx2_stub_interface.h"
namespace Service::AM {
@@ -33,6 +35,13 @@ IApplicationProxy::IApplicationProxy(Core::System& system_, std::shared_ptr<Appl
{10, D<&IApplicationProxy::GetProcessWindingController>, "GetProcessWindingController"},
{11, D<&IApplicationProxy::GetLibraryAppletCreator>, "GetLibraryAppletCreator"},
{20, D<&IApplicationProxy::GetApplicationFunctions>, "GetApplicationFunctions"},
+ // Switch 2-only getters; see nx2_stub_interface.h.
+ {40, &IApplicationProxy::GetNx2Interface, "Nx2Getter40"},
+ {50, &IApplicationProxy::GetNx2Interface, "Nx2Getter50"},
+ {60, D<&IApplicationProxy::GetGraphicsScaleController>, "GetGraphicsScaleController (Switch 2)"},
+ {70, &IApplicationProxy::GetNx2Interface, "Nx2Getter70"},
+ {80, &IApplicationProxy::GetNx2Interface, "Nx2Getter80"},
+ {90, &IApplicationProxy::GetNx2Interface, "Nx2Getter90"},
{1000, D<&IApplicationProxy::GetDebugFunctions>, "GetDebugFunctions"},
};
// clang-format on
@@ -42,6 +51,17 @@ IApplicationProxy::IApplicationProxy(Core::System& system_, std::shared_ptr<Appl
IApplicationProxy::~IApplicationProxy() = default;
+Result IApplicationProxy::GetGraphicsScaleController(
+ Out<SharedPointer<IGraphicsScaleController>> out_controller) {
+ LOG_DEBUG(Service_AM, "called");
+ *out_controller = std::make_shared<IGraphicsScaleController>(system);
+ R_SUCCEED();
+}
+
+void IApplicationProxy::GetNx2Interface(HLERequestContext& ctx) {
+ ReturnNx2StubInterface(system, ctx, "IApplicationProxy");
+}
+
Result IApplicationProxy::GetAudioController(
Out<SharedPointer<IAudioController>> out_audio_controller) {
LOG_DEBUG(Service_AM, "called");
diff --git a/src/core/hle/service/am/service/application_proxy.h b/src/core/hle/service/am/service/application_proxy.h
index b56f597605..0a706be4fd 100644
--- a/src/core/hle/service/am/service/application_proxy.h
+++ b/src/core/hle/service/am/service/application_proxy.h
@@ -17,6 +17,7 @@ class IApplicationFunctions;
class ICommonStateGetter;
class IDebugFunctions;
class IDisplayController;
+class IGraphicsScaleController;
class ILibraryAppletCreator;
class IProcessWindingController;
class ISelfController;
@@ -30,6 +31,8 @@ public:
~IApplicationProxy();
private:
+ void GetNx2Interface(HLERequestContext& ctx);
+ Result GetGraphicsScaleController(Out<SharedPointer<IGraphicsScaleController>> out_controller);
Result GetAudioController(Out<SharedPointer<IAudioController>> out_audio_controller);
Result GetDisplayController(Out<SharedPointer<IDisplayController>> out_display_controller);
Result GetProcessWindingController(
diff --git a/src/core/hle/service/am/service/application_proxy_service.cpp b/src/core/hle/service/am/service/application_proxy_service.cpp
index 7727951d8c..67306aadc0 100644
--- a/src/core/hle/service/am/service/application_proxy_service.cpp
+++ b/src/core/hle/service/am/service/application_proxy_service.cpp
@@ -19,6 +19,8 @@ IApplicationProxyService::IApplicationProxyService(Core::System& system_,
: ServiceFramework{system_, "appletOE"}, m_window_system{window_system} {
static const FunctionInfo functions[] = {
{0, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy"},
+ // Switch 2 SDKs open the proxy with command 10, same arguments.
+ {10, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy (Switch 2)"},
};
RegisterHandlers(functions);
}
diff --git a/src/core/hle/service/am/service/graphics_scale_controller.h b/src/core/hle/service/am/service/graphics_scale_controller.h
new file mode 100644
index 0000000000..07e38c6ac8
--- /dev/null
+++ b/src/core/hle/service/am/service/graphics_scale_controller.h
@@ -0,0 +1,35 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "core/hle/service/cmif_types.h"
+#include "core/hle/service/nx2_stub_interface.h"
+#include "core/hle/service/service.h"
+
+namespace Service::AM {
+
+// Switch 2 interface returned by IApplicationProxy command 60 (name is a guess). nn::oe's
+// GetCurrentGraphicsScale reads command 10 and accepts 10, 20 or 30 (scale enum 0, 1, 2).
+// Commands without a handler are auto-stubbed because the name carries the nx2: prefix.
+class IGraphicsScaleController final : public ServiceFramework<IGraphicsScaleController> {
+public:
+ explicit IGraphicsScaleController(Core::System& system_)
+ : ServiceFramework{system_, "nx2:IGraphicsScaleController"} {
+ // clang-format off
+ static const FunctionInfo functions[] = {
+ {10, D<&IGraphicsScaleController::GetCurrentGraphicsScale>, "GetCurrentGraphicsScale"},
+ };
+ // clang-format on
+ RegisterHandlers(functions);
+ }
+
+private:
+ Result GetCurrentGraphicsScale(Out<u32> out_scale) {
+ LOG_WARNING(Service_AM, "(STUBBED) called, returning 10");
+ *out_scale = 10;
+ R_SUCCEED();
+ }
+};
+
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/self_controller.cpp b/src/core/hle/service/am/service/self_controller.cpp
index 3a654b9daa..a8078e6048 100644
--- a/src/core/hle/service/am/service/self_controller.cpp
+++ b/src/core/hle/service/am/service/self_controller.cpp
@@ -12,6 +12,7 @@
#include "core/hle/service/caps/caps_su.h"
#include "core/hle/service/cmif_serialization.h"
#include "core/hle/service/nvnflinger/nvnflinger.h"
+#include "core/hle/service/nx2_gsv.h"
#include "core/hle/service/sm/sm.h"
#include "core/hle/service/vi/vi_results.h"
@@ -73,6 +74,7 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet>
{230, D<&ISelfController::Unknown230>, "Unknown230"},
{240, D<&ISelfController::Unknown240>, "Unknown240"},
{1000, nullptr, "GetDebugStorageChannel"},
+ {2000, &ISelfController::GetNx2GraphicsSystemSession, "GetGraphicsSystemSession (Switch 2, guessed)"},
};
// clang-format on
@@ -83,6 +85,12 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet>
m_applet->library_applet_mode);
}
+void ISelfController::GetNx2GraphicsSystemSession(HLERequestContext& ctx) {
+ // nn::gsv::BootWithRemoteMemory sends TIPC on the handle this returns.
+ LOG_WARNING(Service_AM, "(STUBBED) Switch 2 ISelfController::2000, returning a nx2:gsv session");
+ ReturnRawSession(system, ctx, std::make_shared<INx2GraphicsSystemSession>(system));
+}
+
ISelfController::~ISelfController() {
std::scoped_lock lk{m_applet->lock};
m_applet->display_layer_manager.Finalize();
diff --git a/src/core/hle/service/am/service/self_controller.h b/src/core/hle/service/am/service/self_controller.h
index c9f480f575..221fbb0981 100644
--- a/src/core/hle/service/am/service/self_controller.h
+++ b/src/core/hle/service/am/service/self_controller.h
@@ -30,6 +30,7 @@ public:
~ISelfController() override;
private:
+ void GetNx2GraphicsSystemSession(HLERequestContext& ctx);
Result Exit();
Result LockExit();
Result UnlockExit();
diff --git a/src/core/hle/service/audio/audio.cpp b/src/core/hle/service/audio/audio.cpp
index fde766a275..211e1cb6bc 100644
--- a/src/core/hle/service/audio/audio.cpp
+++ b/src/core/hle/service/audio/audio.cpp
@@ -6,6 +6,7 @@
#include "core/core.h"
#include "core/hle/service/audio/audio.h"
+#include "core/hle/service/audio/nx2_audio_user_client.h"
#include "core/hle/service/audio/audio_controller.h"
#include "core/hle/service/audio/audio_in_manager.h"
#include "core/hle/service/audio/audio_out_manager.h"
@@ -203,6 +204,8 @@ void LoopProcess(Core::System& system) {
server_manager->RegisterNamedService("audrec:a", std::make_shared<IFinalOutputRecorderManagerForApplet>(system), 30);
server_manager->RegisterNamedService("audrec:u", std::make_shared<IFinalOutputRecorderManager>(system), 30);
server_manager->RegisterNamedService("audren:u", std::make_shared<IAudioRendererManager>(system), 30);
+ // Switch 2 audio entry point (nn::audio::CreateUserClientSessionByHipc).
+ server_manager->RegisterNamedService("aud:u", std::make_shared<INx2AudioUserClient>(system), 30);
server_manager->RegisterNamedService("audren:a", std::make_shared<IAudioRendererManagerForApplet>(system), 30);
server_manager->RegisterNamedService("hwopus", std::make_shared<IHardwareOpusDecoderManager>(system), 25);
ServerManager::RunServer(std::move(server_manager));
diff --git a/src/core/hle/service/audio/nx2_audio_user_client.h b/src/core/hle/service/audio/nx2_audio_user_client.h
new file mode 100644
index 0000000000..2d1a158c84
--- /dev/null
+++ b/src/core/hle/service/audio/nx2_audio_user_client.h
@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "core/hle/service/audio/audio_in_manager.h"
+#include "core/hle/service/audio/audio_out_manager.h"
+#include "core/hle/service/audio/audio_renderer_manager.h"
+#include "core/hle/service/ipc_helpers.h"
+#include "core/hle/service/service.h"
+
+namespace Service::Audio {
+
+// Switch 2 "aud:u" (nn::audio::CreateUserClientSessionByHipc). The SDK reaches the Switch 1
+// audio interfaces through it: command 2 returns an object used exactly like
+// IAudioRendererManager (vtable slot 1 = GetWorkBufferSize with a Switch 1 parameter block), so
+// it is backed by Eden's audren:u implementation. The nx2: name keeps unknown commands
+// auto-stubbed and logged.
+class INx2AudioUserClient final : public ServiceFramework<INx2AudioUserClient> {
+public:
+ explicit INx2AudioUserClient(Core::System& system_)
+ : ServiceFramework{system_, "nx2:aud:u"} {
+ // clang-format off
+ static const FunctionInfo functions[] = {
+ {2, &INx2AudioUserClient::GetAudioRendererManager, "GetAudioRendererManager (guessed)"},
+ };
+ // clang-format on
+ RegisterHandlers(functions);
+ }
+
+private:
+ void GetAudioRendererManager(HLERequestContext& ctx) {
+ IPC::RequestParser rp{ctx};
+ const u32 arg = rp.Pop<u32>();
+ LOG_WARNING(Service_Audio, "(STUBBED) Switch 2 aud:u 2 arg={}, returning IAudioRendererManager",
+ arg);
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(ResultSuccess);
+ rb.PushIpcInterface<IAudioRendererManager>(ctx, system);
+ }
+};
+
+} // namespace Service::Audio
diff --git a/src/core/hle/service/hid/hid_server.cpp b/src/core/hle/service/hid/hid_server.cpp
index 9689bce8dd..04b4663bc2 100644
--- a/src/core/hle/service/hid/hid_server.cpp
+++ b/src/core/hle/service/hid/hid_server.cpp
@@ -132,6 +132,7 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r
{210, C<&IHidServer::EndPermitVibrationSession>, "EndPermitVibrationSession"},
{211, C<&IHidServer::IsVibrationDeviceMounted>, "IsVibrationDeviceMounted"},
{212, C<&IHidServer::SendVibrationValueInBool>, "SendVibrationValueInBool"},
+ {214, C<&IHidServer::SendNx2VibrationValues>, "SendNx2VibrationValues (guessed)"},
{300, C<&IHidServer::ActivateConsoleSixAxisSensor>, "ActivateConsoleSixAxisSensor"},
{301, C<&IHidServer::StartConsoleSixAxisSensor>, "StartConsoleSixAxisSensor"},
{302, C<&IHidServer::StopConsoleSixAxisSensor>, "StopConsoleSixAxisSensor"},
@@ -1069,6 +1070,11 @@ Result IHidServer::IsVibrationDeviceMounted(
R_SUCCEED();
}
+Result IHidServer::SendNx2VibrationValues() {
+ LOG_DEBUG(Service_HID, "(STUBBED) called");
+ R_SUCCEED();
+}
+
Result IHidServer::SendVibrationValueInBool(
bool is_vibrating, Core::HID::VibrationDeviceHandle vibration_device_handle,
ClientAppletResourceUserId aruid) {
diff --git a/src/core/hle/service/hid/hid_server.h b/src/core/hle/service/hid/hid_server.h
index 57a1a04b9c..574d54c415 100644
--- a/src/core/hle/service/hid/hid_server.h
+++ b/src/core/hle/service/hid/hid_server.h
@@ -189,6 +189,9 @@ private:
Result SendVibrationValueInBool(bool is_vibrating,
Core::HID::VibrationDeviceHandle vibration_device_handle,
ClientAppletResourceUserId aruid);
+ // Switch 2 sends vibration every frame with 214 (pointer buffers of device handles and
+ // values, like 206); Eden doesn't emulate vibration, so accept and drop it.
+ Result SendNx2VibrationValues();
Result ActivateConsoleSixAxisSensor(ClientAppletResourceUserId aruid);
Result StartConsoleSixAxisSensor(Core::HID::ConsoleSixAxisSensorHandle console_sixaxis_handle,
ClientAppletResourceUserId aruid);
diff --git a/src/core/hle/service/kernel_helpers.cpp b/src/core/hle/service/kernel_helpers.cpp
index f7a5cb7049..e4d6925f19 100644
--- a/src/core/hle/service/kernel_helpers.cpp
+++ b/src/core/hle/service/kernel_helpers.cpp
@@ -4,6 +4,9 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <mutex>
+#include <unordered_map>
+
#include "core/core.h"
#include "core/core_timing.h"
#include "core/hle/kernel/k_event.h"
@@ -16,6 +19,12 @@
namespace Service::KernelHelpers {
+namespace {
+// Event names for the debugger's handle listing.
+std::mutex event_names_mutex;
+std::unordered_map<const Kernel::KEvent*, std::string> event_names;
+} // namespace
+
ServiceContext::ServiceContext(Core::System& system_, std::string name_)
: kernel(system_.Kernel())
{
@@ -64,14 +73,28 @@ Kernel::KEvent* ServiceContext::CreateEvent(std::string&& name) {
// Register the event.
Kernel::KEvent::Register(kernel, event);
+ {
+ std::scoped_lock lk{event_names_mutex};
+ event_names[event] = std::move(name);
+ }
return event;
}
void ServiceContext::CloseEvent(Kernel::KEvent* event) {
if (event) {
+ {
+ std::scoped_lock lk{event_names_mutex};
+ event_names.erase(event);
+ }
event->GetReadableEvent().Close(kernel);
event->Close(kernel);
}
}
+std::string ServiceContext::GetEventName(const Kernel::KEvent* event) {
+ std::scoped_lock lk{event_names_mutex};
+ const auto it = event_names.find(event);
+ return it == event_names.end() ? std::string{} : it->second;
+}
+
} // namespace Service::KernelHelpers
diff --git a/src/core/hle/service/kernel_helpers.h b/src/core/hle/service/kernel_helpers.h
index 4e21c8857f..a62dd838aa 100644
--- a/src/core/hle/service/kernel_helpers.h
+++ b/src/core/hle/service/kernel_helpers.h
@@ -29,6 +29,9 @@ public:
void CloseEvent(Kernel::KEvent* event);
+ /// Name an event was created with (debugger handle listing); empty when unknown.
+ static std::string GetEventName(const Kernel::KEvent* event);
+
Kernel::KernelCore& kernel;
Kernel::KProcess* process{};
bool process_created{false};
diff --git a/src/core/hle/service/ns/platform_service_manager.cpp b/src/core/hle/service/ns/platform_service_manager.cpp
index bf93a2edfa..88aac56b1c 100644
--- a/src/core/hle/service/ns/platform_service_manager.cpp
+++ b/src/core/hle/service/ns/platform_service_manager.cpp
@@ -22,7 +22,9 @@
#include "core/hle/kernel/k_shared_memory.h"
#include "core/hle/kernel/kernel.h"
#include "core/hle/service/cmif_serialization.h"
+#include "common/alignment.h"
#include "core/hle/service/filesystem/filesystem.h"
+#include "core/hle/service/ipc_helpers.h"
#include "core/hle/service/ns/platform_service_manager.h"
namespace Service::NS {
@@ -83,6 +85,8 @@ struct IPlatformServiceManager::Impl {
// Automatically populated based on shared_fonts dump or system archives.
// 6 builtin fonts + extra 2 for whatever may come after
boost::container::static_vector<FontRegion, 8> shared_font_regions;
+ /// Plain TTF copies of the same fonts (Switch 2 GetBuiltInFontData), after the BFTTF data.
+ boost::container::static_vector<FontRegion, 8> builtin_font_regions;
/// Backing memory for the shared font data
std::array<u8, SHARED_FONT_MEM_SIZE> shared_font;
};
@@ -107,6 +111,8 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch
{106, nullptr, "GetFunctionBlackListVersion"},
{1000, nullptr, "LoadNgWordDataForPlatformRegionChina"},
{1001, nullptr, "GetNgWordDataSizeForPlatformRegionChina"},
+ {2000, &IPlatformServiceManager::GetBuiltInFontSharedMemory, "GetBuiltInFontSharedMemory (Switch 2)"},
+ {2010, &IPlatformServiceManager::GetBuiltInFontData, "GetBuiltInFontData (Switch 2)"},
};
// clang-format on
RegisterHandlers(functions);
@@ -116,6 +122,7 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch
// Attempt to load shared font data from disk
const auto* nand = fsc.GetSystemNANDContents();
std::size_t offset = 0;
+ std::vector<std::vector<u8>> plain_fonts;
// Rebuild shared fonts from data ncas or synthesize
for (auto& font : SHARED_FONTS) {
FileSys::VirtualFile romfs;
@@ -151,9 +158,58 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch
Common::swap32);
// Font offset and size do not account for the header
const FontRegion region{u32(offset + 8), u32((font_data_u32.size() * sizeof(u32)) - 8)};
+ std::vector<u8> plain(region.size);
+ DecryptSharedFontToTTF(font_data_u32, plain);
+ plain_fonts.push_back(std::move(plain));
DecryptSharedFont(font_data_u32, impl->shared_font, offset);
impl->shared_font_regions.push_back(region);
}
+
+ for (const auto& plain : plain_fonts) {
+ offset = Common::AlignUp(offset, 0x1000);
+ if (offset + plain.size() > SHARED_FONT_MEM_SIZE) {
+ LOG_WARNING(Service_NS, "No shared memory left for plain built-in font copies");
+ break;
+ }
+ std::memcpy(impl->shared_font.data() + offset, plain.data(), plain.size());
+ impl->builtin_font_regions.push_back({u32(offset), u32(plain.size())});
+ offset += plain.size();
+ }
+}
+
+// Switch 2: the shared memory GetBuiltInFontData offsets point into (copy handle, u32 size); the
+// SDK maps it read-only. Eden serves its font shared memory, which holds plain copies too.
+void IPlatformServiceManager::GetBuiltInFontSharedMemory(HLERequestContext& ctx) {
+ LOG_WARNING(Service_NS, "(STUBBED) Switch 2 GetBuiltInFontSharedMemory");
+ std::memcpy(kernel.GetFontSharedMem().GetPointer(), impl->shared_font.data(), impl->shared_font.size());
+ IPC::ResponseBuilder rb{ctx, 3, 1};
+ rb.Push(ResultSuccess);
+ rb.Push<u32>(static_cast<u32>(SHARED_FONT_MEM_SIZE));
+ rb.PushCopyObjects(ctx, kernel.GetFontSharedMem());
+}
+
+// nn::pl::GetBuiltInFontData(BuiltInFontDataOunce*, BuiltInFontTypeOunce, bool): returns a 16-byte
+// bool that says whether the font exists and a 16-byte descriptor (u8 flags, u8 flag bit, u16,
+// u32 offset into the font shared memory, u32 size, u32). The client expects plain TrueType/OpenType data
+// there. The type numbering is unknown, so types map onto Eden's shared fonts in order.
+void IPlatformServiceManager::GetBuiltInFontData(HLERequestContext& ctx) {
+ IPC::RequestParser rp{ctx};
+ const u32 type = rp.Pop<u32>();
+ const bool flag = rp.Pop<u8>() != 0;
+ const auto& regions = impl->builtin_font_regions;
+ const bool found = !regions.empty();
+ const FontRegion region = found ? regions[std::min<size_t>(type, regions.size() - 1)] : EMPTY_REGION;
+ LOG_WARNING(Service_NS, "(STUBBED) Switch 2 GetBuiltInFontData type={} flag={} -> offset={:#x} size={:#x}",
+ type, flag, region.offset, region.size);
+ // Raw reply: the bool at offset 0, the descriptor at offset 8.
+ IPC::ResponseBuilder rb{ctx, 2 + 6};
+ rb.Push(ResultSuccess);
+ rb.Push<u32>(found ? 1 : 0);
+ rb.Push<u32>(0);
+ rb.Push<u32>(0);
+ rb.Push<u32>(region.offset);
+ rb.Push<u32>(region.size);
+ rb.Push<u32>(0);
}
IPlatformServiceManager::~IPlatformServiceManager() = default;
diff --git a/src/core/hle/service/ns/platform_service_manager.h b/src/core/hle/service/ns/platform_service_manager.h
index 73521abbc2..ed5b1c1e39 100644
--- a/src/core/hle/service/ns/platform_service_manager.h
+++ b/src/core/hle/service/ns/platform_service_manager.h
@@ -73,6 +73,9 @@ private:
Out<bool> out_fonts_are_loaded, Out<u32> out_font_count,
Set::LanguageCode language_code);
+ void GetBuiltInFontSharedMemory(HLERequestContext& ctx);
+ void GetBuiltInFontData(HLERequestContext& ctx);
+
struct Impl;
std::unique_ptr<Impl> impl;
};
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
index 3c9e42f602..2634902641 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
@@ -14,13 +14,16 @@
#include "common/logging.h"
#include "core/core.h"
#include "core/hle/service/nvdrv/core/container.h"
+#include "core/hle/service/nvdrv/core/heap_mapper.h"
#include "core/hle/service/nvdrv/core/nvmap.h"
#include "core/hle/service/nvdrv/devices/ioctl_serialization.h"
#include "core/hle/service/nvdrv/devices/nvhost_as_gpu.h"
#include "core/hle/service/nvdrv/devices/nvhost_gpu.h"
+#include "core/hle/service/nvdrv/devices/nx2_stub_device.h"
#include "core/hle/service/nvdrv/nvdrv.h"
#include "video_core/control/channel_state.h"
#include "video_core/gpu.h"
+#include "video_core/host1x/host1x.h"
#include "video_core/memory_manager.h"
#include "video_core/rasterizer_interface.h"
@@ -30,7 +33,9 @@ nvhost_as_gpu::nvhost_as_gpu(Core::System& system_, Module& module_, NvCore::Con
: nvdevice{system_}, module{module_}, container{core}, nvmap{core.GetNvMapFile()}, vm{},
gmmu{} {}
-nvhost_as_gpu::~nvhost_as_gpu() = default;
+nvhost_as_gpu::~nvhost_as_gpu() {
+ Nx2GpuMemoryRegistry::Get().RemoveListener(this);
+}
NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input,
std::span<u8> output) {
@@ -49,8 +54,24 @@ NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> i
return WrapFixed(this, &nvhost_as_gpu::MapBufferEx, input, output);
case 0x8:
return WrapFixed(this, &nvhost_as_gpu::GetVARegions1, input, output);
- case 0x9:
- return WrapFixed(this, &nvhost_as_gpu::AllocAsEx, input, output);
+ case 0x9: {
+ const auto result = WrapFixed(this, &nvhost_as_gpu::AllocAsEx, input, output);
+ if (result == NvResult::Success && g_nx2_driver) {
+ Nx2GpuMemoryRegistry::Get().AddListener(
+ this,
+ [this](u64 gpu_address, u64 cpu_address, u64 size) {
+ Nx2Map(gpu_address, cpu_address, size);
+ },
+ [this](u64 gpu_address, u64 size) { Nx2Unmap(gpu_address, size); },
+ [this](u64 cpu_address, u64 size) { Nx2Release(cpu_address, size); });
+ }
+ return result;
+ }
+ case 0xb:
+ if (g_nx2_driver) {
+ return WrapFixed(this, &nvhost_as_gpu::Nx2GetSyncpointShimInfo, input, output);
+ }
+ break;
case 0x14:
return WrapVariable(this, &nvhost_as_gpu::Remap, input, output);
default:
@@ -90,11 +111,105 @@ NvResult nvhost_as_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> i
return NvResult::NotImplemented;
}
-void nvhost_as_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) {}
+void nvhost_as_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) {
+ nx2_session = session_id;
+}
+
+void nvhost_as_gpu::Nx2Map(u64 gpu_address, u64 cpu_address, u64 size) {
+ std::scoped_lock lock(mutex);
+ auto* session = container.GetSession(nx2_session);
+ if (!gmmu || session == nullptr) {
+ LOG_ERROR(Service_NVDRV, "Switch 2 GPU mapping {:#x}+{:#x}: no address space or session",
+ gpu_address, size);
+ return;
+ }
+ // The same CPU range is often mapped twice (at its own address and in a sparse region), so
+ // reuse its SMMU mapping.
+ DAddr device_address{};
+ bool heap = false;
+ if (const auto it = nx2_device_ranges.find({cpu_address, size}); it != nx2_device_ranges.end()) {
+ device_address = it->second.address;
+ heap = it->second.heap;
+ } else if (session->has_preallocated_area && session->mapper->IsInBounds(cpu_address, size)) {
+ device_address = session->mapper->Map(cpu_address, size);
+ heap = true;
+ } else {
+ auto& smmu = system.Host1x().MemoryManager();
+ device_address = smmu.Allocate(Common::AlignUp(size, 0x20000));
+ if (device_address == 0) {
+ LOG_CRITICAL(Service_NVDRV, "Switch 2 GPU memory {:#x}+{:#x}: out of SMMU space",
+ cpu_address, size);
+ return;
+ }
+ smmu.Map(device_address, cpu_address, size, session->asid, true);
+ }
+ nx2_device_ranges[{cpu_address, size}] = {device_address, heap};
+ Nx2GpuMemoryRegistry::Get().RecordDeviceRange(cpu_address, size, device_address);
+ gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false);
+ LOG_DEBUG(Service_NVDRV, "Switch 2 GPU mapping {:#x}+{:#x} -> CPU {:#x}", gpu_address, size,
+ cpu_address);
+}
+
+void nvhost_as_gpu::Nx2Unmap(u64 gpu_address, u64 size) {
+ std::scoped_lock lock(mutex);
+ if (!gmmu) {
+ return;
+ }
+ // The GPU thread may still be running command lists that use the range.
+ system.GPU().RunInOrder([gmmu = gmmu.get(), gpu_address, size] { gmmu->Unmap(gpu_address, size); });
+ LOG_DEBUG(Service_NVDRV, "Switch 2 GPU unmapping {:#x}+{:#x}", gpu_address, size);
+}
+
+void nvhost_as_gpu::Nx2Release(u64 cpu_address, u64 size) {
+ std::scoped_lock lock(mutex);
+ const auto it = nx2_device_ranges.find({cpu_address, size});
+ if (it == nx2_device_ranges.end()) {
+ return;
+ }
+ const Nx2DeviceRange range = it->second;
+ nx2_device_ranges.erase(it);
+ auto* session = container.GetSession(nx2_session);
+ auto* const mapper = range.heap && session != nullptr ? session->mapper.get() : nullptr;
+ auto& smmu = system.Host1x().MemoryManager();
+ // Queued after the GPU unmaps of the range, so the GPU thread is done with it.
+ system.GPU().RunInOrder([&smmu, mapper, range, cpu_address, size] {
+ if (range.heap) {
+ if (mapper != nullptr) {
+ mapper->Unmap(cpu_address, size);
+ }
+ return;
+ }
+ smmu.Unmap(range.address, size);
+ smmu.Free(range.address, Common::AlignUp(size, 0x20000));
+ });
+ LOG_DEBUG(Service_NVDRV, "Switch 2 released backing {:#x}+{:#x}", cpu_address, size);
+}
+
void nvhost_as_gpu::OnClose(DeviceFD fd) {}
+// Switch 2 NvRm (with characteristics flag 26) asks for the read-only syncpoint shim: a GPU page
+// whose u32 at id * stride reads as that syncpoint's value; NVN waits on syncpoints through it and
+// asserts without one. Same scheme as changes.patch, placed on the last page of the VA range.
+NvResult nvhost_as_gpu::Nx2GetSyncpointShimInfo(IoctlNx2SyncpointShimInfo& params) {
+ std::scoped_lock lock(mutex);
+ if (!vm.initialised || !gmmu) {
+ return NvResult::NotSupported;
+ }
+ const u64 address = vm.va_range_end - VM::YUZU_PAGESIZE;
+ if (!nx2_syncpoint_shim) {
+ nx2_syncpoint_shim = true;
+ gmmu->SetNativeSyncpointShim(address);
+ }
+ params.address = address;
+ params.stride = sizeof(u32);
+ params.count = Tegra::MemoryManager::NumShimSyncpoints;
+ LOG_INFO(Service_NVDRV, "Switch 2 syncpoint shim at {:#x} ({} syncpoints)", address,
+ params.count);
+ return NvResult::Success;
+}
+
NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) {
- LOG_DEBUG(Service_NVDRV, "called, big_page_size={:#x}", params.big_page_size);
+ LOG_DEBUG(Service_NVDRV, "called, flags={:#x} big_page_size={:#x} va_range_start={:#x} va_range_end={:#x} va_range_split={:#x}", params.flags, params.big_page_size, params.va_range_start, params.va_range_end, params.va_range_split);
std::scoped_lock lock(mutex);
@@ -125,6 +240,15 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) {
vm.va_range_start = params.va_range_start;
vm.va_range_split = params.va_range_split;
vm.va_range_end = params.va_range_end;
+ if (vm.va_range_split == 0) {
+ // Switch 2 NvRm (flags 5, no big page size) asks for small pages over the whole range
+ // and then reserves 4 KiB VA up to 512 GiB. Keep a small big-page region past the end
+ // so both allocators stay valid.
+ vm.va_range_split = vm.va_range_end;
+ vm.va_range_end += 4ULL << 30;
+ vm.absolute_big_page_limit = true;
+ LOG_WARNING(Service_NVDRV, "No VA split given; small pages up to {:#x}", vm.va_range_split);
+ }
}
const u64 max_big_page_bits = Common::Log2Ceil(vm.va_range_end);
@@ -134,8 +258,11 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) {
vm.small_page_allocator.emplace(start_pages, end_pages);
const auto start_big_pages{static_cast<u32>(vm.va_range_split >> vm.big_page_size_bits)};
- const auto end_big_pages{
- static_cast<u32>((vm.va_range_end - vm.va_range_split) >> vm.big_page_size_bits)};
+ // The Switch 1 layout passes the big-page region length as the limit (so the limit stays above
+ // the start there); a split past the region length needs the absolute end instead.
+ const auto end_big_pages{static_cast<u32>(
+ (vm.absolute_big_page_limit ? vm.va_range_end : vm.va_range_end - vm.va_range_split) >>
+ vm.big_page_size_bits)};
vm.big_page_allocator.emplace(start_big_pages, end_big_pages);
gmmu = std::make_unique<Tegra::MemoryManager>(system, max_big_page_bits, vm.va_range_split, vm.big_page_size_bits, VM::PAGE_SIZE_BITS);
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
index b13b1c9efa..d64ec85dec 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
+++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
@@ -218,8 +218,29 @@ private:
std::optional<Allocator> small_page_allocator; //! Shared as this is also used by nvhost::GpuChannel
bool initialised{};
+ bool absolute_big_page_limit{}; //!< Switch 2 layout: big-page limit is va_range_end
} vm;
std::unique_ptr<Tegra::MemoryManager> gmmu;
+
+ struct IoctlNx2SyncpointShimInfo {
+ u64_le address;
+ u32_le stride;
+ u32_le count;
+ };
+ static_assert(sizeof(IoctlNx2SyncpointShimInfo) == 16);
+ NvResult Nx2GetSyncpointShimInfo(IoctlNx2SyncpointShimInfo& params);
+ bool nx2_syncpoint_shim{};
+
+ // Switch 2: maps GPU-accessible process memory (CPU range) at a GPU address.
+ void Nx2Map(u64 gpu_address, u64 cpu_address, u64 size);
+ void Nx2Unmap(u64 gpu_address, u64 size);
+ NvCore::SessionId nx2_session{};
+ struct Nx2DeviceRange {
+ DAddr address;
+ bool heap; ///< Mapped through the session's heap mapper (else an SMMU allocation)
+ };
+ std::map<std::pair<u64, u64>, Nx2DeviceRange> nx2_device_ranges;
+ void Nx2Release(u64 cpu_address, u64 size);
};
} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
index 14cdd8fda4..df2421cd13 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
@@ -4,6 +4,8 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <array>
+#include <algorithm>
#include <cstring>
#include "common/assert.h"
#include "common/logging.h"
@@ -11,10 +13,16 @@
#include "core/core_timing.h"
#include "core/hle/service/nvdrv/devices/ioctl_serialization.h"
#include "core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h"
+#include "core/hle/service/nvdrv/devices/nx2_stub_device.h"
+#include "video_core/nx2.h"
#include "core/hle/service/nvdrv/nvdrv.h"
namespace Service::Nvidia::Devices {
+namespace {
+void ApplyNx2CharacteristicsTail(std::span<u8> block);
+} // namespace
+
nvhost_ctrl_gpu::nvhost_ctrl_gpu(Core::System& system_, EventInterface& events_interface_)
: nvdevice{system_}, events_interface{events_interface_} {
error_notifier_event = events_interface.CreateEvent("CtrlGpuErrorNotifier");
@@ -93,9 +101,14 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8>
switch (command.group) {
case 'G':
switch (command.cmd) {
- case 0x5:
- return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetCharacteristics3, input, output,
- inline_output);
+ case 0x5: {
+ const auto result = WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetCharacteristics3,
+ input, output, inline_output);
+ if (nx2_caller) {
+ ApplyNx2CharacteristicsTail(inline_output);
+ }
+ return result;
+ }
case 0x6:
return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetTPCMasks3, input, output,
inline_output);
@@ -112,14 +125,108 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8>
default:
break;
}
+ if (nx2_caller && command.group == 'G' && command.cmd == 0x1F) {
+ return Nx2GetEngineInfo(input, output, inline_output);
+ }
+ if (nx2_caller) {
+ // Switch 2 NvRm uses ioctls Switch 1 never had; answer them with a zeroed success so
+ // device setup can continue, and log each one.
+ LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 ioctl={:08X} group={:01X} command={:01X}",
+ command.raw, command.group, command.cmd);
+ std::fill(output.begin(), output.end(), u8{0});
+ std::fill(inline_output.begin(), inline_output.end(), u8{0});
+ return NvResult::Success;
+ }
UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}, group={:01X}, command={:01X}", command.raw,
command.group, command.cmd);
return NvResult::NotImplemented;
}
+// nvgpu GET_ENGINE_INFO as the Switch 2 NvRm uses it: args {u32 buf_size, u32 reserved,
+// u64 buf_addr}, queried once for the size and once with an inline output buffer of 16-byte items
+// {u32 engine_id, u32 engine_instance, s32 runlist_id, u32 reserved}. NvRm picks the channel
+// runlist from this list. Report the graphics engine (runlist 0) and an async copy engine (1).
+NvResult nvhost_ctrl_gpu::Nx2GetEngineInfo(std::span<const u8> input, std::span<u8> output,
+ std::span<u8> inline_output) {
+ struct Item {
+ u32 engine_id;
+ u32 engine_instance;
+ s32 runlist_id;
+ u32 reserved;
+ };
+ static constexpr std::array<Item, 2> items{{
+ {0, 0, 0, 0}, // NVGPU_ENGINE_GR
+ {2, 0, 1, 0}, // NVGPU_ENGINE_ASYNC_COPY
+ }};
+ constexpr u32 needed = static_cast<u32>(items.size() * sizeof(Item));
+
+ u32 buf_size{};
+ if (input.size() >= sizeof(buf_size)) {
+ std::memcpy(&buf_size, input.data(), sizeof(buf_size));
+ }
+ LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 GetEngineInfo buf_size={:#x} inline_out={:#x}",
+ buf_size, inline_output.size());
+ if (buf_size >= needed && inline_output.size() >= needed) {
+ std::memcpy(inline_output.data(), items.data(), needed);
+ }
+ std::fill(output.begin(), output.end(), u8{0});
+ std::copy_n(input.begin(), std::min(input.size(), output.size()), output.begin());
+ if (output.size() >= sizeof(needed)) {
+ std::memcpy(output.data(), &needed, sizeof(needed));
+ }
+ return NvResult::Success;
+}
+
void nvhost_ctrl_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) {}
void nvhost_ctrl_gpu::OnClose(DeviceFD fd) {}
+namespace {
+
+// Switch 2 SDKs ask for a 0xD0-byte characteristics block (Switch 1: 0xA0). Their NvRm only
+// accepts chip ids 0x17E, 0x12B and 0x12E and its NVN path rejects the Maxwell ones, so report the
+// Switch 2 Ampere chip. Fields past the Switch 1 layout are left zero. Values are best guesses.
+constexpr u64 Nx2CharacteristicsSize = 0xD0;
+
+template <typename GC>
+void ApplyNx2Characteristics(GC& gc) {
+ gc.arch = 0x170; // Ampere
+ gc.impl = 0xE; // chip id 0x17E as listed in the Switch 2 NvRm chip table
+ gc.num_tpc_per_gpc = 0x6;
+ gc.sm_arch_sm_version = 0x808; // NVN2 only accepts SM 0x808 (its class table lookup)
+ gc.sm_arch_spa_version = 0x808;
+ gc.threed_class = 0xC797; // AMPERE_B
+ gc.compute_class = 0xC7C0; // AMPERE_COMPUTE_B
+ gc.gpfifo_class = 0xC56F; // AMPERE_CHANNEL_GPFIFO_A
+ gc.dma_copy_class = 0xC7B5; // AMPERE_DMA_COPY_B
+ gc.chipname = 0x6631306167; // "ga10f"
+ gc.flags |= 0x100; // NVGPU_GPU_FLAGS_SUPPORT_TSG: NvRm binds channels to a TSG
+ gc.flags |= 1ULL << 22; // NVGPU_GPU_FLAGS_SUPPORT_TSG_SUBCONTEXTS (see below)
+ gc.flags |= 1ULL << 26; // NVGPU_GPU_FLAGS_SUPPORT_SYNCPOINT_ADDRESS: syncpoint shim
+
+ // The Switch 2 layout keeps nine s16 interface versions at 0x70..0x80 (like nvgpu's
+ // *_ioctl_nr_last); its NvRm requires e.g. [0x78] >= 6 and enables features above 0xa, 0x1f
+ // and 0x26. In the Switch 1 layout these bytes are max_fbps_count..max_tex_per_tpc.
+ auto* const raw = reinterpret_cast<u8*>(&gc);
+ for (size_t off = 0x70; off <= 0x80; off += 2) {
+ const s16 version = 0x40;
+ std::memcpy(raw + off, &version, sizeof(version));
+ }
+}
+
+// Fields of the Switch 2 block past the Switch 1 layout. NvRm sizes each TSG's subcontext (VEID)
+// allocator from the u32 at 0xb4, but only when the TSG interface version (0x72) is above 0xa and
+// flags bits 8 and 22 are set; otherwise every TSG gets one subcontext and a second queue's
+// NvRmGpuTSGSubcontextCreate fails (0x11), leaving that NVN queue without command memory.
+void ApplyNx2CharacteristicsTail(std::span<u8> block) {
+ constexpr size_t MaxSubcontextsOffset = 0xb4;
+ constexpr u32 MaxSubcontexts = 64; // Ampere VEIDs per TSG
+ if (block.size() >= MaxSubcontextsOffset + sizeof(MaxSubcontexts)) {
+ std::memcpy(block.data() + MaxSubcontextsOffset, &MaxSubcontexts, sizeof(MaxSubcontexts));
+ }
+}
+
+} // namespace
+
NvResult nvhost_ctrl_gpu::GetCharacteristics1(IoctlCharacteristics& params) {
LOG_DEBUG(Service_NVDRV, "called");
params.gc.arch = 0x120;
@@ -164,7 +271,8 @@ NvResult nvhost_ctrl_gpu::GetCharacteristics1(IoctlCharacteristics& params) {
NvResult nvhost_ctrl_gpu::GetCharacteristics3(
IoctlCharacteristics& params, std::span<IoctlGpuCharacteristics> gpu_characteristics) {
- LOG_DEBUG(Service_NVDRV, "called");
+ LOG_DEBUG(Service_NVDRV, "called, buf_size={:#x}", params.gpu_characteristics_buf_size);
+ const bool nx2 = params.gpu_characteristics_buf_size >= Nx2CharacteristicsSize;
params.gc.arch = 0x120;
params.gc.impl = 0xb;
@@ -203,6 +311,14 @@ NvResult nvhost_ctrl_gpu::GetCharacteristics3(
params.gc.gr_compbit_store_base_hw = 0x0;
params.gpu_characteristics_buf_size = 0xA0;
params.gpu_characteristics_buf_addr = 0xdeadbeef; // Cannot be 0 (UNUSED)
+ if (nx2) {
+ nx2_caller = true;
+ g_nx2_driver = true;
+ Tegra::g_nx2_ampere = true;
+ ApplyNx2Characteristics(params.gc);
+ params.gpu_characteristics_buf_size = Nx2CharacteristicsSize;
+ LOG_WARNING(Service_NVDRV, "Switch 2 caller, reporting Ampere chip 0x17E");
+ }
if (!gpu_characteristics.empty()) {
gpu_characteristics.front() = params.gc;
}
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h
index 80b7abd6a1..507cdac71d 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h
+++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h
@@ -208,6 +208,11 @@ private:
EventInterface& events_interface;
// Events
+ NvResult Nx2GetEngineInfo(std::span<const u8> input, std::span<u8> output,
+ std::span<u8> inline_output);
+
+ // Set once a caller asked for the Switch 2 characteristics layout.
+ bool nx2_caller{};
Kernel::KEvent* error_notifier_event;
Kernel::KEvent* unknown_event;
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp
index 1e571be4c9..2a144cc7ab 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp
@@ -4,7 +4,9 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <cstdlib>
#include <cstring>
+#include <vector>
#include "common/assert.h"
#include "common/logging.h"
#include "core/core.h"
@@ -14,6 +16,7 @@
#include "core/hle/service/nvdrv/core/syncpoint_manager.h"
#include "core/hle/service/nvdrv/devices/ioctl_serialization.h"
#include "core/hle/service/nvdrv/devices/nvhost_gpu.h"
+#include "core/hle/service/nvdrv/devices/nx2_stub_device.h"
#include "core/hle/service/nvdrv/nvdrv.h"
#include "core/memory.h"
#include "video_core/control/channel_state.h"
@@ -48,6 +51,9 @@ nvhost_gpu::~nvhost_gpu() {
events_interface.FreeEvent(sm_exception_breakpoint_int_report_event);
events_interface.FreeEvent(sm_exception_breakpoint_pause_report_event);
events_interface.FreeEvent(error_notifier_event);
+ for (auto& [id, event] : nx2_events) {
+ events_interface.FreeEvent(event);
+ }
syncpoint_manager.FreeSyncpoint(channel_syncpoint);
}
@@ -101,6 +107,9 @@ NvResult nvhost_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> inpu
}
break;
}
+ if (g_nx2_driver) {
+ return Nx2UnknownIoctl("nvhost-gpu", command, input, output);
+ }
UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}", command.raw);
return NvResult::NotImplemented;
};
@@ -116,12 +125,19 @@ NvResult nvhost_gpu::Ioctl2(DeviceFD fd, Ioctl command, std::span<const u8> inpu
}
break;
}
+ if (g_nx2_driver) {
+ return Nx2UnknownIoctl("nvhost-gpu", command, input, output);
+ }
UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}", command.raw);
return NvResult::NotImplemented;
}
NvResult nvhost_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> input,
std::span<u8> output, std::span<u8> inline_output) {
+ if (g_nx2_driver) {
+ std::fill(inline_output.begin(), inline_output.end(), u8{0});
+ return Nx2UnknownIoctl("nvhost-gpu", command, input, output);
+ }
UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}", command.raw);
return NvResult::NotImplemented;
}
@@ -278,6 +294,35 @@ NvResult nvhost_gpu::AllocateObjectContext(IoctlAllocObjCtx& params) {
params.flags = allowed_mask;
}
+ if (g_nx2_driver) {
+ // Switch 2 NvRm asks for Ampere classes, or class 0 (it reads the class numbers from a
+ // characteristics layout we don't know yet). Map them onto the Maxwell engine slots.
+ u32 mapped = params.class_num;
+ switch (params.class_num) {
+ case 0xC797: case 0xC697: case 0xC597: mapped = static_cast<u32>(CtxClasses::Ctx3D); break;
+ case 0xC7C0: case 0xC6C0: case 0xC5C0: mapped = static_cast<u32>(CtxClasses::CtxCompute); break;
+ case 0xC7B5: case 0xC6B5: case 0xC5B5: mapped = static_cast<u32>(CtxClasses::CtxDMA); break;
+ case 0xC56F: case 0xC76F: mapped = static_cast<u32>(CtxClasses::CtxChannelGPFIFO); break;
+ case 0: {
+ for (const auto candidate : {CtxClasses::Ctx3D, CtxClasses::CtxCompute, CtxClasses::CtxDMA,
+ CtxClasses::Ctx2D, CtxClasses::CtxKepler}) {
+ const auto index = GetObjectContextClassNumberIndex(candidate);
+ if (index >= 0 && !ctxObjs[index].has_value()) {
+ mapped = static_cast<u32>(candidate);
+ break;
+ }
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ if (mapped != params.class_num) {
+ LOG_WARNING(Service_NVDRV, "Switch 2 object class {:#x} -> {:#x}", params.class_num, mapped);
+ params.class_num = mapped;
+ }
+ }
+
s32_le ctx_class_number_index =
GetObjectContextClassNumberIndex(static_cast<CtxClasses>(params.class_num));
if (ctx_class_number_index < 0) {
@@ -327,6 +372,27 @@ static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementC
return result;
}
+// Switch 2 pushbuffers signal completion with semaphore releases, never host1x syncpoint methods,
+// but the driver still has the kernel raise the syncpoint max by the submission's increment value
+// and then waits for that value. Append the increments after its commands.
+static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementCountCommandList(
+ NvFence fence, u32 count) {
+ boost::container::small_vector<Tegra::CommandHeader, 512> result{
+ Tegra::BuildCommandHeader(Tegra::BufferMethods::WaitForIdle, 1,
+ Tegra::SubmissionMode::Increasing),
+ {},
+ Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointPayload, 1,
+ Tegra::SubmissionMode::Increasing),
+ {}};
+ for (u32 i = 0; i < count; ++i) {
+ result.push_back(Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointOperation, 1,
+ Tegra::SubmissionMode::Increasing));
+ result.push_back(
+ BuildFenceAction(Tegra::Engines::Puller::FenceOperation::Increment, fence.id));
+ }
+ return result;
+}
+
static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementWithWfiCommandList(
NvFence fence) {
boost::container::small_vector<Tegra::CommandHeader, 512> result{
@@ -353,6 +419,23 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL
const auto bind_id = channel_state->bind_id;
+ if (g_nx2_driver) {
+ for (const auto& entry : entries.command_lists) {
+ LOG_TRACE(Service_NVDRV, "nx2 GPFIFO entry raw={:#018x} addr={:#x} size={:#x}", entry.raw,
+ entry.addr.Value(), entry.size.Value());
+ }
+ static const bool check = std::getenv("EDEN_NX2_PB_CHECK") != nullptr;
+ if (check) {
+ std::vector<u32> words;
+ for (const auto& entry : entries.command_lists) {
+ words.resize(entry.size);
+ system.ApplicationMemory().ReadBlock(entry.addr.Value(), words.data(),
+ words.size() * sizeof(u32));
+ entries.submit_hashes.push_back(Tegra::HashPushbufferWords(words));
+ }
+ }
+ }
+
auto& flags = params.flags;
if (flags.fence_wait.Value()) {
@@ -372,6 +455,11 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL
params.fence.value = syncpoint_manager.IncrementSyncpointMaxExt(channel_syncpoint, increment);
gpu.PushGPUEntries(bind_id, std::move(entries));
+ if (g_nx2_driver && !flags.fence_increment.Value() && increment > 0) {
+ gpu.PushGPUEntries(bind_id,
+ Tegra::CommandList{BuildIncrementCountCommandList(params.fence, increment)});
+ }
+
if (flags.fence_increment.Value()) {
if (flags.suppress_wfi.Value()) {
gpu.PushGPUEntries(bind_id,
@@ -382,6 +470,12 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL
}
}
+ if (g_nx2_driver) {
+ LOG_TRACE(Service_NVDRV,
+ "nx2 SubmitGPFIFO entries={} flags={:#x} fence_out=({}, {}) guest_value={}",
+ params.num_entries, flags.raw, params.fence.id, params.fence.value,
+ system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(params.fence.id));
+ }
flags.raw = 0;
return NvResult::Success;
@@ -464,6 +558,16 @@ Kernel::KEvent* nvhost_gpu::QueryEvent(u32 event_id) {
case 3:
return error_notifier_event;
default:
+ if (g_nx2_driver) {
+ // Switch 2 NVN also asks for event 7 and multi-waits on it with its syncpoint events.
+ // Nothing signals it yet.
+ auto*& event = nx2_events[event_id];
+ if (event == nullptr) {
+ event = events_interface.CreateEvent(fmt::format("GpuChannelNx2Event{}", event_id));
+ }
+ LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 channel event {}", event_id);
+ return event;
+ }
LOG_CRITICAL(Service_NVDRV, "Unknown Ctrl GPU Event {}", event_id);
return nullptr;
}
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h
index 70a6b1bd68..ab968d6c7e 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h
+++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h
@@ -6,6 +6,7 @@
#pragma once
+#include <map>
#include <memory>
#include <vector>
#include "common/bit_field.h"
@@ -225,6 +226,7 @@ private:
Kernel::KEvent* sm_exception_breakpoint_int_report_event;
Kernel::KEvent* sm_exception_breakpoint_pause_report_event;
Kernel::KEvent* error_notifier_event;
+ std::map<u32, Kernel::KEvent*> nx2_events;
};
} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h
new file mode 100644
index 0000000000..b91a929f68
--- /dev/null
+++ b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h
@@ -0,0 +1,204 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <algorithm>
+#include <atomic>
+#include <functional>
+#include <map>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "common/logging.h"
+#include "core/hle/service/nvdrv/devices/nvdevice.h"
+
+namespace Service::Nvidia::Devices {
+
+// Set once a Switch 2 NvRm has identified itself (0xD0-byte GetCharacteristics). Devices then
+// answer ioctls they don't know with a logged, input-echoing success instead of NotImplemented.
+inline std::atomic<bool> g_nx2_driver{false};
+
+inline NvResult Nx2UnknownIoctl(const char* device, Ioctl command, std::span<const u8> input,
+ std::span<u8> output) {
+ LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 {} ioctl={:08X} group={:01X} command={:02X} in={:#x}",
+ device, command.raw, command.group, command.cmd, input.size());
+ std::fill(output.begin(), output.end(), u8{0});
+ std::copy_n(input.begin(), std::min(input.size(), output.size()), output.begin());
+ return NvResult::Success;
+}
+
+// GPU mappings a Switch 2 process made through nx2:gpu: CreateGpuAccessibleMemory (sub 101) maps
+// the memory at GPU address == CPU address, and NormalMapGpuMemory4K (211) maps a memory handle at
+// a chosen GPU address (also in sparse regions). Every nvhost-as-gpu address space applies each
+// mapping, including ones made before the address space existed.
+class Nx2GpuMemoryRegistry {
+public:
+ using Listener = std::function<void(u64 gpu_address, u64 cpu_address, u64 size)>;
+ using UnmapListener = std::function<void(u64 gpu_address, u64 size)>;
+ using ReleaseListener = std::function<void(u64 cpu_address, u64 size)>;
+
+ static Nx2GpuMemoryRegistry& Get() {
+ static Nx2GpuMemoryRegistry registry;
+ return registry;
+ }
+
+ void Register(u64 address, u64 size) {
+ MapAt(address, address, size);
+ }
+
+ void MapAt(u64 gpu_address, u64 cpu_address, u64 size) {
+ std::scoped_lock lk{mutex};
+ TrimMappings(gpu_address, size);
+ mappings.push_back({gpu_address, cpu_address, size});
+ for (auto& [owner, listener] : listeners) {
+ listener.map(gpu_address, cpu_address, size);
+ }
+ }
+
+ // NormalUnmapGpuMemory4K, or destroying the memory: the GPU range stops reaching it.
+ void Unmap(u64 gpu_address, u64 size) {
+ std::scoped_lock lk{mutex};
+ TrimMappings(gpu_address, size);
+ for (auto& [owner, listener] : listeners) {
+ listener.unmap(gpu_address, size);
+ }
+ }
+
+ // Backing memory [cpu_address, +size) is about to be freed (its GPU mappings are gone): each
+ // address space drops its SMMU mapping of it, in GPU-thread order.
+ void ReleaseBacking(u64 cpu_address, u64 size) {
+ {
+ std::scoped_lock lk{device_mutex};
+ device_ranges.erase(cpu_address);
+ }
+ std::scoped_lock lk{mutex};
+ for (auto& [owner, listener] : listeners) {
+ if (listener.release) {
+ listener.release(cpu_address, size);
+ }
+ }
+ }
+
+ void AddListener(const void* owner, Listener listener, UnmapListener unmap_listener,
+ ReleaseListener release_listener = {}) {
+ std::scoped_lock lk{mutex};
+ for (const auto& m : mappings) {
+ listener(m.gpu_address, m.cpu_address, m.size);
+ }
+ listeners.insert_or_assign(owner, Listeners{std::move(listener), std::move(unmap_listener),
+ std::move(release_listener)});
+ }
+
+ void RemoveListener(const void* owner) {
+ std::scoped_lock lk{mutex};
+ listeners.erase(owner);
+ }
+
+ // Device (SMMU) address of a CPU range an address space mapped; used to present images.
+ void RecordDeviceRange(u64 cpu_address, u64 size, u64 device_address) {
+ std::scoped_lock lk{device_mutex};
+ device_ranges[cpu_address] = {size, device_address};
+ }
+
+ std::optional<u64> DeviceAddressOf(u64 cpu_address) {
+ std::scoped_lock lk{device_mutex};
+ auto it = device_ranges.upper_bound(cpu_address);
+ if (it == device_ranges.begin()) {
+ return std::nullopt;
+ }
+ --it;
+ const auto& [start, range] = *it;
+ if (cpu_address - start >= range.first) {
+ return std::nullopt;
+ }
+ return range.second + (cpu_address - start);
+ }
+
+private:
+ // Separate lock: ranges are recorded from listeners that run under `mutex`.
+ std::mutex device_mutex;
+ std::map<u64, std::pair<u64, u64>> device_ranges; // cpu start -> (size, device address)
+
+ struct Mapping {
+ u64 gpu_address;
+ u64 cpu_address;
+ u64 size;
+ };
+
+ struct Listeners {
+ Listener map;
+ UnmapListener unmap;
+ ReleaseListener release;
+ };
+
+ // Drops [gpu_address, +size) from the recorded mappings, keeping the parts outside it.
+ void TrimMappings(u64 gpu_address, u64 size) {
+ const u64 end = gpu_address + size;
+ std::vector<Mapping> kept;
+ kept.reserve(mappings.size() + 1);
+ for (const auto& m : mappings) {
+ const u64 m_end = m.gpu_address + m.size;
+ if (m_end <= gpu_address || m.gpu_address >= end) {
+ kept.push_back(m);
+ continue;
+ }
+ if (m.gpu_address < gpu_address) {
+ kept.push_back({m.gpu_address, m.cpu_address, gpu_address - m.gpu_address});
+ }
+ if (m_end > end) {
+ kept.push_back({end, m.cpu_address + (end - m.gpu_address), m_end - end});
+ }
+ }
+ mappings = std::move(kept);
+ }
+
+ std::mutex mutex;
+ std::vector<Mapping> mappings;
+ std::map<const void*, Listeners> listeners;
+};
+
+// Device node that Switch 2 nvgpu drivers open but Switch 1 never had (e.g. /dev/nvhost-tsg-gpu).
+// Every ioctl succeeds with a zeroed reply and is logged, so the driver's use of it shows up.
+class nx2_stub_device final : public nvdevice {
+public:
+ explicit nx2_stub_device(Core::System& system_, std::string name_)
+ : nvdevice{system_}, name{std::move(name_)} {}
+
+ NvResult Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input,
+ std::span<u8> output) override {
+ return Stub(fd, command, input, output, {});
+ }
+ NvResult Ioctl2(DeviceFD fd, Ioctl command, std::span<const u8> input,
+ std::span<const u8> inline_input, std::span<u8> output) override {
+ return Stub(fd, command, input, output, {});
+ }
+ NvResult Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> input, std::span<u8> output,
+ std::span<u8> inline_output) override {
+ return Stub(fd, command, input, output, inline_output);
+ }
+
+ void OnOpen(NvCore::SessionId session_id, DeviceFD fd) override {
+ LOG_WARNING(Service_NVDRV, "(STUBBED) {} opened as fd {}", name, fd);
+ }
+ void OnClose(DeviceFD fd) override {}
+
+private:
+ NvResult Stub(DeviceFD fd, Ioctl command, std::span<const u8> input, std::span<u8> output,
+ std::span<u8> inline_output) {
+ LOG_WARNING(Service_NVDRV, "(STUBBED) {} fd={} ioctl={:08X} group={:01X} command={:02X} in={:#x}",
+ name, fd, command.raw, command.group, command.cmd, input.size());
+ // Echo the input (in/out ioctls keep their fields), zero the rest.
+ std::fill(output.begin(), output.end(), u8{0});
+ std::copy_n(input.begin(), std::min(input.size(), output.size()), output.begin());
+ std::fill(inline_output.begin(), inline_output.end(), u8{0});
+ return NvResult::Success;
+ }
+
+ std::string name;
+};
+
+} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/nvdrv.cpp b/src/core/hle/service/nvdrv/nvdrv.cpp
index b252a2efa5..5cac2693c5 100644
--- a/src/core/hle/service/nvdrv/nvdrv.cpp
+++ b/src/core/hle/service/nvdrv/nvdrv.cpp
@@ -23,6 +23,7 @@
#include "core/hle/service/nvdrv/devices/nvhost_nvjpg.h"
#include "core/hle/service/nvdrv/devices/nvhost_vic.h"
#include "core/hle/service/nvdrv/devices/nvmap.h"
+#include "core/hle/service/nvdrv/devices/nx2_stub_device.h"
#include "core/hle/service/nvdrv/nvdrv.h"
#include "core/hle/service/nvdrv/nvdrv_interface.h"
#include "core/hle/service/nvdrv/nvmemp.h"
@@ -125,6 +126,11 @@ void LoopProcess(Core::System& system) {
Module::Module(Core::System& system)
: container{system.Host1x()}, service_context{system, "nvdrv"}, events_interface{*this} {
+ nx2_stub_builder = [this, &system](DeviceFD fd, const std::string& name) {
+ auto device = std::make_shared<Devices::nx2_stub_device>(system, name);
+ open_files.emplace(fd, device);
+ return std::static_pointer_cast<Devices::nvdevice>(device);
+ };
builders["/dev/nvhost-as-gpu"] = [this, &system](DeviceFD fd) {
auto device = std::make_shared<Devices::nvhost_as_gpu>(system, *this, container);
return open_files.emplace(fd, std::move(device)).first;
@@ -181,6 +187,15 @@ NvResult Module::VerifyFD(DeviceFD fd) const {
DeviceFD Module::Open(const std::string& device_name, NvCore::SessionId session_id) {
auto it = builders.find(device_name);
+ if (it == builders.end() && device_name.starts_with("/dev/nvhost-")) {
+ // Newer nvgpu nodes used by Switch 2 drivers (e.g. /dev/nvhost-tsg-gpu): serve them with
+ // a logged stub instead of failing the open.
+ LOG_WARNING(Service_NVDRV, "Opening unknown device {} as a Switch 2 stub", device_name);
+ const DeviceFD fd = next_fd++;
+ auto device = nx2_stub_builder(fd, device_name);
+ device->OnOpen(session_id, fd);
+ return fd;
+ }
if (it == builders.end()) {
LOG_ERROR(Service_NVDRV, "Trying to open unknown device {}", device_name);
return INVALID_NVDRV_FD;
diff --git a/src/core/hle/service/nvdrv/nvdrv.h b/src/core/hle/service/nvdrv/nvdrv.h
index f503f108c3..ba8ab6a40e 100644
--- a/src/core/hle/service/nvdrv/nvdrv.h
+++ b/src/core/hle/service/nvdrv/nvdrv.h
@@ -98,6 +98,8 @@ private:
friend class EventInterface;
/// Manages syncpoints on the host
+ /// Builds a logged stub for nvgpu device nodes only Switch 2 drivers use.
+ std::function<std::shared_ptr<Devices::nvdevice>(DeviceFD, const std::string&)> nx2_stub_builder;
NvCore::Container container;
/// Id to use for the next open file descriptor.
diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.cpp b/src/core/hle/service/nvdrv/nvdrv_interface.cpp
index 5fae68adbb..020036d74b 100644
--- a/src/core/hle/service/nvdrv/nvdrv_interface.cpp
+++ b/src/core/hle/service/nvdrv/nvdrv_interface.cpp
@@ -34,6 +34,7 @@ void NVDRV::Open(HLERequestContext& ctx) {
const auto& buffer = ctx.ReadBuffer();
const std::string device_name(Common::StringFromBuffer(buffer));
+ LOG_DEBUG(Service_NVDRV, "device_name={}", device_name);
if (device_name == "/dev/nvhost-prof-gpu") {
rb.Push<DeviceFD>(0);
@@ -181,6 +182,14 @@ void NVDRV::Initialize(HLERequestContext& ctx) {
is_initialized = true;
}
+void NVDRV::InitializeForProcess(Kernel::KProcess* process) {
+ if (is_initialized) {
+ return;
+ }
+ session_id = nvdrv->GetContainer().OpenSession(process);
+ is_initialized = true;
+}
+
void NVDRV::QueryEvent(HLERequestContext& ctx) {
IPC::RequestParser rp{ctx};
const auto fd = rp.Pop<DeviceFD>();
diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.h b/src/core/hle/service/nvdrv/nvdrv_interface.h
index f628fca7df..8fe3196176 100644
--- a/src/core/hle/service/nvdrv/nvdrv_interface.h
+++ b/src/core/hle/service/nvdrv/nvdrv_interface.h
@@ -23,6 +23,9 @@ public:
return nvdrv;
}
+ // For Switch 2 nvsrv sessions, which are set up through gsv instead of Initialize.
+ void InitializeForProcess(Kernel::KProcess* process);
+
private:
void Open(HLERequestContext& ctx);
void Ioctl1(HLERequestContext& ctx);
diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h
new file mode 100644
index 0000000000..740b5627af
--- /dev/null
+++ b/src/core/hle/service/nx2_gsv.h
@@ -0,0 +1,1647 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <thread>
+#include <chrono>
+#include <string>
+#include <cstdlib>
+#include <cstdio>
+#include <initializer_list>
+#include <atomic>
+#include <deque>
+#include <map>
+#include <optional>
+#include <memory>
+#include <mutex>
+#include <vector>
+
+#include "common/alignment.h"
+#include "common/settings.h"
+#include "core/core.h"
+#include "video_core/rasterizer_interface.h"
+#include "video_core/renderer_base.h"
+#include "core/frontend/framebuffer_layout.h"
+#include "core/perf_stats.h"
+#include "video_core/gpu.h"
+#include "video_core/framebuffer_config.h"
+#include "core/core_timing.h"
+#include "core/hle/kernel/k_client_session.h"
+#include "core/hle/kernel/k_process.h"
+#include "core/hle/kernel/kernel.h"
+#include "core/hle/kernel/k_system_resource.h"
+#include "core/hle/kernel/k_memory_manager.h"
+#include "core/hle/kernel/k_page_group.h"
+#include "core/hle/kernel/k_resource_limit.h"
+#include "core/hle/kernel/k_server_session.h"
+#include "core/hle/kernel/k_event.h"
+#include "core/hle/kernel/k_readable_event.h"
+#include "core/hle/kernel/k_session.h"
+#include "core/hle/kernel/k_transfer_memory.h"
+#include "core/memory.h"
+#include "video_core/host1x/host1x.h"
+#include "video_core/nx2.h"
+#include "core/hle/service/hle_ipc.h"
+#include "core/hle/service/ipc_helpers.h"
+#include "core/hle/service/kernel_helpers.h"
+#include "core/hle/service/nvdrv/devices/nx2_stub_device.h"
+#include "core/hle/service/nvdrv/nvdrv.h"
+#include "core/hle/service/nvdrv/nvdrv_interface.h"
+#include "core/hle/service/nx2_stub_interface.h"
+#include "core/hle/service/sm/sm.h"
+#include "core/hle/service/server_manager.h"
+#include "core/hle/service/service.h"
+
+namespace Service {
+
+// Creates a session served by `iface` and returns its client end. Unlike PushIpcInterface this
+// works from a domain too, for callers that want a raw session handle.
+template <class T>
+inline Kernel::KClientSession* CreateRawSession(Core::System& system, HLERequestContext& ctx,
+ std::shared_ptr<T> iface) {
+ auto& kernel = system.Kernel();
+ ASSERT(Kernel::GetCurrentProcess(kernel).GetResourceLimit()->Reserve(
+ kernel, Kernel::LimitableResource::SessionCountMax, 1));
+
+ auto* session = Kernel::KSession::Create(kernel);
+ session->Initialize(kernel, nullptr, 0);
+ Kernel::KSession::Register(kernel, session);
+
+ auto manager =
+ std::make_shared<SessionRequestManager>(kernel, ctx.GetManager()->GetServerManager());
+ manager->SetSessionHandler(std::move(iface));
+ ctx.GetManager()->GetServerManager().RegisterSession(&session->GetServerSession(), manager);
+ return &session->GetClientSession();
+}
+
+// Replies to a TIPC request with success and raw u32 data words. ResponseBuilder::Push(Result)
+// writes the CMIF two-word result, which shifts TIPC data one word past where the client reads
+// it (and past the copied size); TIPC has a one-word result followed directly by the data.
+inline void TipcReplyWords(HLERequestContext& ctx, std::initializer_list<u32> words) {
+ IPC::ResponseBuilder rb{ctx, static_cast<u32>(2 + words.size())};
+ rb.PushRaw<u32>(ResultSuccess.raw);
+ for (const u32 w : words) {
+ rb.Push<u32>(w);
+ }
+}
+
+// Replies with success and a copied handle to a new session served by `iface`.
+template <class T>
+inline void ReturnRawSession(Core::System& system, HLERequestContext& ctx, std::shared_ptr<T> iface) {
+ auto* client = CreateRawSession(system, ctx, std::move(iface));
+ IPC::ResponseBuilder rb{ctx, 2, 1, 0};
+ rb.Push(ResultSuccess);
+ rb.PushCopyObjects(ctx, *client);
+}
+
+// Placeholder for the Switch 2 NVIDIA driver server (nn::nvsrv). It has no handlers, so every
+// command (CMIF or TIPC) gets an empty success and is logged through the nx2: prefix.
+class INx2NvsrvSession final : public ServiceFramework<INx2NvsrvSession> {
+public:
+ explicit INx2NvsrvSession(Core::System& system_) : ServiceFramework{system_, "nx2:nvsrv"} {}
+};
+
+// Placeholder for the Switch 2 syncpoint server (nn::syncpt), handed out by gsv 202. TIPC 1
+// returns a u32 the SDK requires to be < 0x400 and then passes min(value, 63) to TIPC 21.
+// Everything else is auto-stubbed through the nx2: prefix.
+// - TIPC 0(u64): handshake, echo the token (newer SDKs).
+// - TIPC 22(u32 slot): an event handle for that slot (copy handle at word 3, result at word 4).
+// NVN's NvOsEventPool asks for slots and multi-waits on them.
+// - TIPC 201(slot, counter, u64 value, mode): signal the slot's event once the counter reaches
+// value. The counter is a SyncCounterId, the bitwise NOT of the host1x syncpoint id; mode 0
+// compares 32-bit wrapping, mode 1 64-bit. Eden's guest syncpoints back it.
+// - TIPC 202(slot): cancel the slot's request.
+class INx2SyncptSession final : public ServiceFramework<INx2SyncptSession> {
+public:
+ explicit INx2SyncptSession(Core::System& system_)
+ : ServiceFramework{system_, "nx2:syncpt"}, m_context{system_, "nx2:syncpt"} {
+ RegisterHandlersTipc({
+ {0, &INx2SyncptSession::Handshake, "Handshake"},
+ {1, &INx2SyncptSession::GetCount, "GetCount (guessed)"},
+ {22, &INx2SyncptSession::GetSlotEvent, "GetSlotEvent (guessed)"},
+ {103, &INx2SyncptSession::QueryCounter, "QueryCounter"},
+ {201, &INx2SyncptSession::RequestSlotEvent, "RequestSlotEvent (guessed)"},
+ {202, &INx2SyncptSession::CancelSlotEvent, "CancelSlotEvent (guessed)"},
+ });
+ }
+
+ ~INx2SyncptSession() override {
+ // A registered action may be running on the GPU thread right now: mark the session dead
+ // under the lock it signals under, so it can't touch a request or event closed below.
+ {
+ std::scoped_lock lk{m_shared->mutex};
+ m_shared->alive = false;
+ }
+ for (auto& [slot, request] : m_requests) {
+ Cancel(request);
+ }
+ for (auto& [slot, event] : m_events) {
+ m_context.CloseEvent(event);
+ }
+ }
+
+private:
+ // Newer SDKs (title 1) open with TIPC 0(u64 token) and abort unless the reply echoes it.
+ void Handshake(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ LOG_INFO(Service, "nx2:syncpt TIPC 0 handshake {:#x}", u64{w[2]} | (u64{w[3]} << 32));
+ TipcReplyWords(ctx, {w[2], w[3]});
+ }
+
+ void GetCount(HLERequestContext& ctx) {
+ LOG_WARNING(Service, "(STUBBED) nx2:syncpt TIPC 1, returning 32");
+ TipcReplyWords(ctx, {32});
+ }
+
+ // From changes.patch: the counter is the bitwise NOT of the host1x syncpoint id.
+ void QueryCounter(HLERequestContext& ctx) {
+ const u32 syncpoint = ~ctx.CommandBuffer()[2];
+ const u64 value = syncpoint < HostSyncpoints
+ ? system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(syncpoint)
+ : 0;
+ LOG_DEBUG(Service, "nx2:syncpt 103 syncpoint={} value={:#x}", syncpoint, value);
+ TipcReplyWords(ctx, {static_cast<u32>(value), static_cast<u32>(value >> 32), 0});
+ }
+
+ void GetSlotEvent(HLERequestContext& ctx) {
+ const u32 slot = ctx.CommandBuffer()[2];
+ LOG_WARNING(Service, "(STUBBED) nx2:syncpt TIPC 22 slot={:#x}, returning an event", slot);
+ auto*& event = m_events[slot];
+ if (event == nullptr) {
+ event = m_context.CreateEvent(fmt::format("nx2:syncpt:Slot{:x}", slot));
+ }
+ IPC::ResponseBuilder rb{ctx, 2, 1, 0};
+ rb.Push(ResultSuccess);
+ rb.PushCopyObjects(ctx, event->GetReadableEvent());
+ }
+
+ // Map nodes are stable, so a registered action can hold a reference to its request.
+ struct SlotRequest {
+ u32 syncpoint{};
+ std::atomic<bool> pending{};
+ Tegra::Host1x::SyncpointManager::ActionHandle handle{};
+ };
+
+ // Outlives the session (actions hold a reference): signalling happens under `mutex` and only
+ // while `alive`.
+ struct SharedState {
+ std::mutex mutex;
+ bool alive{true};
+ };
+
+ void RequestSlotEvent(HLERequestContext& ctx) {
+ const auto* cmd_buf = ctx.CommandBuffer();
+ const u32 slot = cmd_buf[2];
+ const u32 syncpoint = ~cmd_buf[3];
+ const u64 value = u64{cmd_buf[4]} | (u64{cmd_buf[5]} << 32);
+ const u32 mode = cmd_buf[6];
+ LOG_INFO(Service, "nx2:syncpt TIPC 201 slot={:#x} syncpoint={} value={:#x} mode={}", slot,
+ syncpoint, value, mode);
+ auto*& event = m_events[slot];
+ if (event == nullptr) {
+ event = m_context.CreateEvent(fmt::format("nx2:syncpt:Slot{:x}", slot));
+ }
+ if (syncpoint >= HostSyncpoints) {
+ LOG_WARNING(Service, "nx2:syncpt TIPC 201 syncpoint {} out of range, signalling",
+ syncpoint);
+ event->Signal(system.Kernel());
+ TipcReplyWords(ctx, {});
+ return;
+ }
+ auto& request = m_requests[slot];
+ Cancel(request);
+ request.syncpoint = syncpoint;
+ request.pending = true;
+ auto& manager = system.Host1x().GetSyncpointManager();
+ // The action runs on the GPU thread under the manager's lock, or inline if the value is
+ // already reached.
+ request.handle = manager.RegisterGuestAction(
+ syncpoint, static_cast<u32>(value),
+ [shared = m_shared, &kernel = system.Kernel(), &request, event] {
+ std::scoped_lock lk{shared->mutex};
+ if (!shared->alive) {
+ return;
+ }
+ request.pending = false;
+ event->Signal(kernel);
+ });
+ TipcReplyWords(ctx, {});
+ }
+
+ void CancelSlotEvent(HLERequestContext& ctx) {
+ const u32 slot = ctx.CommandBuffer()[2];
+ LOG_INFO(Service, "nx2:syncpt TIPC 202 slot={:#x}", slot);
+ if (auto it = m_requests.find(slot); it != m_requests.end() && Cancel(it->second)) {
+ // The client waits on the slot's event after a cancel (NvOsEventPool, after its
+ // ~100 ms wait times out), so a cancel must signal it like a completion; it rechecks
+ // the counter (TIPC 103) itself. Title 1 hung forever there when the GPU thread ran
+ // a frame behind (EDEN_NX2_FRAMES_AHEAD) and a mid-frame wait outlasted 100 ms.
+ if (const auto event = m_events.find(slot); event != m_events.end()) {
+ event->second->Signal(system.Kernel());
+ }
+ }
+ TipcReplyWords(ctx, {});
+ }
+
+ /// Returns true if the request was still pending.
+ bool Cancel(SlotRequest& request) {
+ if (request.pending.exchange(false)) {
+ system.Host1x().GetSyncpointManager().DeregisterGuestAction(request.syncpoint,
+ request.handle);
+ return true;
+ }
+ return false;
+ }
+
+ static constexpr u32 HostSyncpoints = 192; // Tegra::Host1x::SyncpointManager
+
+ KernelHelpers::ServiceContext m_context;
+ std::map<u32, Kernel::KEvent*> m_events;
+ std::map<u32, SlotRequest> m_requests;
+ std::shared_ptr<SharedState> m_shared = std::make_shared<SharedState>();
+};
+
+// Placeholder for the Switch 2 GPU session (nn::gpu::sdk), handed out by gsv 497.
+// - TIPC 11: [first, last] id range (first <= min(last, 64)); ids outside it are masked out.
+// - TIPC 12(id): event handle for that id (handle at word 3, result at word 4); the SDK attaches
+// it as a SystemEvent.
+// - TIPC 21(id): starts an operation; the SDK then waits on the id's event. Nothing runs here, so
+// the event is signalled at once.
+// - TIPC 22: another handle the SDK keeps like a session; a placeholder session is returned.
+class INx2GpuSession final : public ServiceFramework<INx2GpuSession> {
+public:
+ explicit INx2GpuSession(Core::System& system_)
+ : ServiceFramework{system_, "nx2:gpu"}, m_context{system_, "nx2:gpu"} {
+ RegisterHandlersTipc({
+ {11, &INx2GpuSession::GetIdRange, "GetIdRange (guessed)"},
+ {12, &INx2GpuSession::GetIdEvent, "GetIdEvent (guessed)"},
+ {21, &INx2GpuSession::Start, "Start (guessed)"},
+ {22, &INx2GpuSession::OpenSubSession, "OpenSubSession (guessed)"},
+ {33, &INx2GpuSession::GetCount33, "Unknown33 (u32 count)"},
+ {201, &INx2GpuSession::GetUserPartition, "GetUserPartition (newer SDKs)"},
+ {202, &INx2GpuSession::GetGsvPartition, "GetGsvPartition (newer SDKs)"},
+ {203, &INx2GpuSession::GetVaRange, "GetVaRange (guessed)"},
+ // Sub-session commands (gpu 22 hands out a second session to this same object).
+ {101, &INx2GpuSession::SubRegisterMemory, "CreateGpuAccessibleMemory"},
+ {102, &INx2GpuSession::SubDestroyMemory, "DestroyGpuAccessibleMemory (guessed)"},
+ {211, &INx2GpuSession::SubSubmitJob, "NormalMapGpuMemory4K"},
+ {213, &INx2GpuSession::SubUnmapJob, "NormalUnmapGpuMemory4K"},
+ {214, &INx2GpuSession::SubCreateGpuOnlyRegion, "CreateSparseRegion"},
+ {215, &INx2GpuSession::SubDestroySparseRegion, "DestroySparseRegion"},
+ });
+ }
+
+ ~INx2GpuSession() override {
+ for (auto* event : m_events) {
+ if (event != nullptr) {
+ m_context.CloseEvent(event);
+ }
+ }
+ }
+
+private:
+ static constexpr u32 NumIds = 32;
+
+ Kernel::KEvent* EventFor(u32 id) {
+ id %= NumIds;
+ if (m_events[id] == nullptr) {
+ m_events[id] = m_context.CreateEvent(fmt::format("nx2:gpu:Event{}", id));
+ }
+ return m_events[id];
+ }
+
+ void GetIdRange(HLERequestContext& ctx) {
+ LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 11, returning [0, {}]", NumIds - 1);
+ TipcReplyWords(ctx, {0, NumIds - 1});
+ }
+
+ void GetIdEvent(HLERequestContext& ctx) {
+ const u32 id = ctx.CommandBuffer()[2];
+ LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 12 id={}", id);
+ IPC::ResponseBuilder rb{ctx, 2, 1, 0};
+ rb.Push(ResultSuccess);
+ rb.PushCopyObjects(ctx, EventFor(id)->GetReadableEvent());
+ }
+
+ void Start(HLERequestContext& ctx) {
+ const u32 id = ctx.CommandBuffer()[2];
+ LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 21 id={}, signalling completion", id);
+ EventFor(id)->Signal(system.Kernel());
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(ResultSuccess);
+ }
+
+ void GetCount33(HLERequestContext& ctx) {
+ LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 33 [2]={:#x}, returning 1", ctx.CommandBuffer()[2]);
+ TipcReplyWords(ctx, {1});
+ }
+
+ // Used by nn::gpu::sdk::CreateGpuAddressSpace: three u64s after the result, a kind (< 5, 0 =
+ // none) and the GPU VA range of the in-process VirtualMachine that serves AllocSpace
+ // (nn::gpu::sdk::util::ExecuteVirtualMachineAllocSpace). NVN reserves fixed VA from 0x8000000
+ // up to 512 GiB, so report the same range NvRm gives AllocAsEx: [0x8000000, 0xC000000000).
+ void GetVaRange(HLERequestContext& ctx) {
+ LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 203 [2]={:#x}", ctx.CommandBuffer()[2]);
+ constexpr u64 kind = 1;
+ constexpr u64 start = 0x8000000ULL;
+ constexpr u64 size = 0xC000000000ULL - start;
+ TipcReplyWords(ctx, {static_cast<u32>(kind), static_cast<u32>(kind >> 32),
+ static_cast<u32>(start), static_cast<u32>(start >> 32),
+ static_cast<u32>(size), static_cast<u32>(size >> 32)});
+ }
+
+ // Newer SDKs (title 1) query the address space's user and gsv partitions with 201/202(id):
+ // two u64s each, stored as the partition's min/max. The user value works whether it is read
+ // as (start, size) or (min, max). Title 4 never needed the gsv partition, so it stays empty.
+ void GetUserPartition(HLERequestContext& ctx) {
+ LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 201 [2]={:#x}", ctx.CommandBuffer()[2]);
+ constexpr u64 first = 0x8000000ULL;
+ constexpr u64 second = 0xC000000000ULL - first;
+ TipcReplyWords(ctx, {static_cast<u32>(first), static_cast<u32>(first >> 32),
+ static_cast<u32>(second), static_cast<u32>(second >> 32)});
+ }
+
+ void GetGsvPartition(HLERequestContext& ctx) {
+ LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 202 [2]={:#x}, empty", ctx.CommandBuffer()[2]);
+ TipcReplyWords(ctx, {0, 0, 0, 0});
+ }
+
+ void OpenSubSession(HLERequestContext& ctx) {
+ // Served by this object so sub-session jobs can signal the per-id events.
+ LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 22, returning a sub-session on nx2:gpu");
+ ReturnRawSession(system, ctx, std::static_pointer_cast<INx2GpuSession>(shared_from_this()));
+ }
+
+ // Sub 101: (start u64, last u64, flags...) after CreateGpuAccessibleMemory's svc 0x80.
+ void SubRegisterMemory(HLERequestContext& ctx) {
+ const auto* b = ctx.CommandBuffer();
+ const u64 start = u64{b[2]} | (u64{b[3]} << 32);
+ const u64 last = u64{b[4]} | (u64{b[5]} << 32);
+ LOG_INFO(Service, "nx2:gpu sub 101 start={:#x} last={:#x} [6]={:#x} [7]={:#x}", start,
+ last, b[6], b[7]);
+ if (last > start) {
+ Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Register(start, last - start + 1);
+ // Two-page regions hold queue semaphores: GPU releases go to the second page, waits
+ // read the first (see video_core/nx2.h).
+ if (last - start + 1 == 0x2000) {
+ Tegra::Nx2AddSemaphoreRegion(start);
+ LOG_INFO(Service, "nx2:gpu semaphore page {:#x} mirrored to {:#x}", start + 0x1000,
+ start);
+ }
+ }
+ // nn::gpu::sdk::CreateGpuAccessibleMemory stores word 3 of the reply as the
+ // GpuMemoryHandle that later map jobs (211) name.
+ const u32 handle = m_next_memory_handle++;
+ m_memory[handle] = {start, last - start + 1};
+ TipcReplyWords(ctx, {handle});
+ }
+
+ // Jobs that complete through a job-slot event (word Slot of the request), like 211.
+ template <size_t Slot>
+ void SubSlotJob(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ LOG_INFO(Service, "(STUBBED) nx2:gpu {} words {:08x} slot={}", ctx.GetCommand(),
+ fmt::join(std::span<const u32>{w + 2, 18}, " "), w[Slot]);
+ EventFor(w[Slot])->Signal(system.Kernel());
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(ResultSuccess);
+ }
+
+ // Sub 211: job submission (nn::gpu::sdk job tracking). Word 2 is a constant id; word 17 is
+ // the job slot whose event (gpu TIPC 12) the SDK then waits on. Nothing runs here, so
+ // complete it at once.
+ void SubSubmitJob(HLERequestContext& ctx) {
+ const auto* cmd_buf = ctx.CommandBuffer();
+ const u32 slot = cmd_buf[17];
+ const u64 a = u64{cmd_buf[3]} | (u64{cmd_buf[4]} << 32);
+ const u64 b = u64{cmd_buf[5]} | (u64{cmd_buf[6]} << 32);
+ const u64 c = u64{cmd_buf[8]} | (u64{cmd_buf[9]} << 32);
+ // Words 3-4: first GPU address, 5-6: last, 7: memory handle, 8-9: offset into it.
+ const u32 handle = cmd_buf[7];
+ LOG_INFO(Service, "nx2:gpu 211 map [{:#x}, {:#x}] handle={} offset={:#x} slot={}", a, b,
+ handle, c, slot);
+ if (const auto it = m_memory.find(handle); it != m_memory.end() && b > a) {
+ // Also at the memory's own address: an earlier 213 may have unmapped the mapping 101
+ // made there.
+ const u64 cpu = it->second.address + c;
+ Nvidia::Devices::Nx2GpuMemoryRegistry::Get().MapAt(a, cpu, b - a + 1);
+ } else {
+ LOG_WARNING(Service, "nx2:gpu 211 map of unknown handle {}", handle);
+ }
+ EventFor(slot)->Signal(system.Kernel());
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(ResultSuccess);
+ }
+
+ // Sub 213: words 3-4 first GPU address, 5-6 last, job slot in word 8.
+ void SubUnmapJob(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ const u64 first = u64{w[3]} | (u64{w[4]} << 32);
+ const u64 last = u64{w[5]} | (u64{w[6]} << 32);
+ LOG_DEBUG(Service, "nx2:gpu 213 unmap [{:#x}, {:#x}] slot={}", first, last, w[8]);
+ if (last > first) {
+ Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Unmap(first, last - first + 1);
+ }
+ SignalAfterGpu(w[8]);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(ResultSuccess);
+ }
+
+ // Signals a job slot's event once the GPU thread has run everything queued so far, as the
+ // console's graphics server completes unmap jobs behind the submitted work.
+ void SignalAfterGpu(u32 slot) {
+ auto* const event = EventFor(slot);
+ system.GPU().RunInOrder([this, event] { event->Signal(system.Kernel()); });
+ }
+
+ // Sub 102: word 2 is the memory handle 101 returned, word 3 the job slot. The game frees the
+ // memory (svc 0x80 clears GPU access) and reuses it as ordinary heap right after, so nothing
+ // on the GPU side may keep it: drop the mapping 101 made at its own address.
+ void SubDestroyMemory(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ const u32 handle = w[2];
+ if (const auto it = m_memory.find(handle); it != m_memory.end()) {
+ const auto [address, size] = it->second;
+ LOG_DEBUG(Service, "nx2:gpu 102 destroy handle={} [{:#x}, +{:#x}] slot={}", handle,
+ address, size, w[3]);
+ if (size != 0) {
+ Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Unmap(address, size);
+ }
+ if (size == 0x2000) {
+ Tegra::Nx2RemoveSemaphoreRegion(address);
+ }
+ m_memory.erase(it);
+ } else {
+ LOG_WARNING(Service, "nx2:gpu 102 destroy of unknown handle {}", handle);
+ }
+ SignalAfterGpu(w[3]);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(ResultSuccess);
+ }
+
+ // Sub 214: GPU-only memory [start, last] (words 3-6, job slot in word 11). The graphics
+ // server backs it itself: the client never maps anything there but renders into it (title 4
+ // keeps its render targets in six 300 MB regions at 0x81_0000_0000). Back each region with
+ // kernel memory placed in the process's alias code region and map that for the GPU.
+ void SubCreateGpuOnlyRegion(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ const u64 start = u64{w[3]} | (u64{w[4]} << 32);
+ const u64 last = u64{w[5]} | (u64{w[6]} << 32);
+ const u32 slot = w[11];
+ LOG_INFO(Service, "nx2:gpu 214 words {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}",
+ w[1], w[2], w[3], w[4], w[5], w[6], w[7], w[8], w[9], w[10], w[11], w[12]);
+ if (last > start && ((last + 1 - start) % Core::Memory::YUZU_PAGESIZE) == 0) {
+ const u64 size = last + 1 - start;
+ FreeReleasedRegions();
+ if (const auto address = BackGpuOnlyRegion(start, size)) {
+ Nvidia::Devices::Nx2GpuMemoryRegistry::Get().MapAt(start, *address, size);
+ LOG_INFO(Service, "nx2:gpu 214 GPU-only region [{:#x}, {:#x}] backed at {:#x}",
+ start, last, *address);
+ }
+ } else {
+ LOG_WARNING(Service, "nx2:gpu 214 unexpected region [{:#x}, {:#x}]", start, last);
+ }
+ EventFor(slot)->Signal(system.Kernel());
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(ResultSuccess);
+ }
+
+ // Sub 215 (nn::gpu::sdk::DestroySparseRegion): words 3-6 the range, word 8 the job slot, like
+ // 213. Drop the GPU mapping, then the backing 214 added: its SMMU mapping (in GPU-thread
+ // order) and, once the GPU thread is past that, its pages. Kept allocated, title 1 ran the
+ // pool dry within minutes and later regions had no memory at all (GPU writes to them lost).
+ void SubDestroySparseRegion(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ const u64 first = u64{w[3]} | (u64{w[4]} << 32);
+ const u64 last = u64{w[5]} | (u64{w[6]} << 32);
+ LOG_INFO(Service, "nx2:gpu 215 destroy sparse region [{:#x}, {:#x}] words {:#x} {:#x} {:#x} {:#x}",
+ first, last, w[1], w[2], w[7], w[8]);
+ FreeReleasedRegions();
+ if (last > first) {
+ Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Unmap(first, last - first + 1);
+ const auto region = m_gpu_only_regions.find(first);
+ if (region != m_gpu_only_regions.end() && region->second.size == last - first + 1) {
+ Nvidia::Devices::Nx2GpuMemoryRegistry::Get().ReleaseBacking(
+ region->second.cpu_address, region->second.size);
+ auto done = std::make_shared<std::atomic<bool>>(false);
+ system.GPU().RunInOrder([done] { done->store(true, std::memory_order_release); });
+ m_released_regions.push_back({std::move(region->second), std::move(done)});
+ m_gpu_only_regions.erase(region);
+ }
+ }
+ SignalAfterGpu(w[8]);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(ResultSuccess);
+ }
+
+ // Returns the pages of destroyed regions the GPU thread is done with to the kernel.
+ void FreeReleasedRegions() {
+ auto* const process = system.ApplicationProcess();
+ std::erase_if(m_released_regions, [&](ReleasedRegion& released) {
+ if (!released.done->load(std::memory_order_acquire)) {
+ return false;
+ }
+ auto& region = released.region;
+ if (process != nullptr &&
+ R_FAILED(process->GetPageTable().UnmapPageGroup(
+ region.cpu_address, *region.pages, Kernel::KMemoryState::IoMemory))) {
+ LOG_ERROR(Service, "nx2:gpu could not unmap GPU-only backing {:#x}+{:#x}",
+ region.cpu_address, region.size);
+ return true; // still mapped: keep its pages
+ }
+ region.pages->Close(system.Kernel());
+ return true;
+ });
+ }
+
+ std::optional<u64> BackGpuOnlyRegion(u64 gpu_address, u64 size) {
+ using Core::Memory::YUZU_PAGESIZE;
+ auto& kernel_core = system.Kernel();
+ auto* const process = system.ApplicationProcess();
+ if (process == nullptr) {
+ return std::nullopt;
+ }
+ auto pg = std::make_unique<Kernel::KPageGroup>(
+ kernel_core, std::addressof(kernel_core.GetSystemSystemResource().GetBlockInfoManager()));
+ // The graphics server's own memory, not the application's: taken from the application
+ // pool, title 4's six regions left it 1.8 GB less than the kernel reports to it. Not
+ // cleared: the GPU writes these before reading them, and touching 300 MB would commit it
+ // on the host.
+ if (R_FAILED(kernel_core.MemoryManager().AllocateAndOpen(
+ pg.get(), size / YUZU_PAGESIZE,
+ Kernel::KMemoryManager::EncodeOption(Kernel::KMemoryManager::Pool::SystemNonSecure,
+ Kernel::KMemoryManager::Direction::FromBack)))) {
+ LOG_ERROR(Service, "nx2:gpu 214 could not allocate {:#x} bytes", size);
+ return std::nullopt;
+ }
+ auto& page_table = process->GetPageTable();
+ const u64 region_start = GetInteger(page_table.GetAliasCodeRegionStart());
+ const u64 region_size = page_table.GetAliasCodeRegionSize();
+ constexpr u64 Alignment = 0x200000; // 2 MiB
+ for (u64 offset = 0; offset + size <= region_size; offset += Alignment) {
+ const Common::ProcessAddress address{region_start + offset};
+ if (R_SUCCEEDED(page_table.MapPageGroup(address, *pg, Kernel::KMemoryState::IoMemory,
+ Kernel::KMemoryPermission::UserReadWrite))) {
+ m_gpu_only_regions[gpu_address] = {GetInteger(address), size, std::move(pg)};
+ return GetInteger(address);
+ }
+ }
+ LOG_ERROR(Service, "nx2:gpu 214 no room for {:#x} bytes in the alias code region "
+ "[{:#x}, +{:#x})", size, region_start, region_size);
+ pg->Close(kernel_core);
+ return std::nullopt;
+ }
+
+ struct GpuOnlyRegion {
+ u64 cpu_address;
+ u64 size;
+ std::unique_ptr<Kernel::KPageGroup> pages;
+ };
+ struct ReleasedRegion {
+ GpuOnlyRegion region;
+ std::shared_ptr<std::atomic<bool>> done; ///< Set by the GPU thread past the region's last use
+ };
+ std::map<u64, GpuOnlyRegion> m_gpu_only_regions; ///< By GPU address
+ std::vector<ReleasedRegion> m_released_regions;
+
+ struct GpuMemory {
+ u64 address;
+ u64 size;
+ };
+
+ KernelHelpers::ServiceContext m_context;
+ std::array<Kernel::KEvent*, NumIds> m_events{};
+ std::map<u32, GpuMemory> m_memory;
+ u32 m_next_memory_handle{1};
+};
+
+// Native memory and images handed out by the Switch 2 graphics server (gsv 111/114/423/429/315),
+// shared with the vi2 session so presents can find an image's memory. Command meanings follow
+// the shared changes.patch (IGraphicsSupervisor), re-checked against title 4's SDK.
+struct Nx2NativeObjects {
+ struct Memory {
+ u64 size{};
+ u64 address{}; // CPU (== GPU) address once bound by 114/423
+ };
+ struct Image {
+ u32 memory{};
+ std::array<u32, 16> preference{};
+ u64 size{};
+ bool window_pool{};
+ };
+ std::mutex mutex;
+ std::map<u32, Memory> memories;
+ std::map<u32, Image> images;
+ u32 next_memory{1};
+ u32 next_image{1};
+};
+
+// Block-linear layout of an InterModuleUserImagePreference (16 words): word 2 holds
+// (width - 1) | (height - 1) << 16, word 4 the block height (log2 of GOB rows), word 5 the row
+// pitch in 64-byte GOBs. Formula from changes.patch (native_image_layout.h); preferences outside
+// it fall back to 4 bytes per pixel and 128-row blocks.
+struct Nx2ImageLayout {
+ u32 width;
+ u32 height;
+ u64 pitch;
+ u32 block_height_log2;
+ u64 size;
+ u64 alignment;
+};
+
+inline Nx2ImageLayout Nx2CalculateImageLayout(const u32* preference) {
+ Nx2ImageLayout layout{};
+ layout.width = (preference[2] & 0xFFFF) + 1;
+ layout.height = (preference[2] >> 16) + 1;
+ const u64 pitch_gobs = preference[5];
+ const u32 block_height_log2 = preference[4];
+ if (block_height_log2 <= 8 && pitch_gobs >= (u64{layout.width} * 4 + 63) / 64 &&
+ pitch_gobs <= 4096) {
+ layout.pitch = pitch_gobs * 64;
+ layout.block_height_log2 = block_height_log2;
+ } else {
+ layout.pitch = Common::AlignUp(u64{layout.width} * 4, 64);
+ layout.block_height_log2 = 7;
+ }
+ const u64 block_rows = u64{1} << layout.block_height_log2;
+ layout.size = layout.pitch * Common::AlignUp(u64{layout.height}, block_rows);
+ layout.alignment = std::max<u64>(64 * block_rows, 0x1000);
+ return layout;
+}
+
+// Placeholder for the Switch 2 display session (nn::vi2::sdk), handed out by gsv 305. Reply
+// shapes read from the SDK: TIPC 1 -> u32 < 0x400, TIPC 3 -> u32 < 8, TIPC 11/31(u32) and
+// 32(handle, u32) -> one handle at word 3 then the result. Handles returned here are events (a
+// guess); everything else is auto-stubbed through the nx2: prefix.
+class INx2Vi2Session final : public ServiceFramework<INx2Vi2Session> {
+public:
+ explicit INx2Vi2Session(Core::System& system_, std::shared_ptr<Nx2NativeObjects> objects_)
+ : ServiceFramework{system_, "nx2:vi2"}, m_context{system_, "nx2:vi2"},
+ m_objects{std::move(objects_)} {
+ RegisterHandlersTipc({
+ {1, &INx2Vi2Session::ReturnOne, "Unknown1"},
+ {3, &INx2Vi2Session::ReturnOne, "GetDisplayCount (guessed)"},
+ {5, &INx2Vi2Session::GetTimestampRecorderCount, "GetTimestampRecorderCount (guessed)"},
+ {11, &INx2Vi2Session::ReturnEvent, "Unknown11"},
+ {31, &INx2Vi2Session::GetVsyncEvent, "GetVsyncEvent (guessed)"},
+ {32, &INx2Vi2Session::GetVsyncEvent, "GetDisplayVsyncEvent (guessed)"},
+ // Title 1 calls these ~75 times a second with the display name "default"; the
+ // auto stub's empty success reply works, but logged two lines per call (a 50 MB log
+ // in an 18 minute session).
+ {104, &INx2Vi2Session::SucceedQuietly, "Unknown104 (display \"default\")"},
+ {105, &INx2Vi2Session::SucceedQuietly, "Unknown105 (display \"default\")"},
+ {228, &INx2Vi2Session::CancelClientBuffer, "CancelClientBuffer"},
+ {229, &INx2Vi2Session::PresentClientBuffer, "PresentClientBuffer"},
+ });
+ }
+
+ ~INx2Vi2Session() override {
+ // The vsync callback may be running on the HostTiming thread: empty its event list under
+ // its lock first (it holds the state, not this session), then close the events.
+ {
+ std::scoped_lock lk{m_vsync->mutex};
+ m_vsync->events.clear();
+ }
+ if (m_vsync_timer) {
+ system.CoreTiming().UnscheduleEvent(m_vsync_timer);
+ }
+ for (auto* event : m_events) {
+ m_context.CloseEvent(event);
+ }
+ }
+
+private:
+ // 229(u64 client token, u32 slot, u32 image, ...fences): result, u64 mask of buffer slots the
+ // client may acquire next, four zero words. Nothing is shown yet; every slot is handed back
+ // at once (changes.patch's default mask is 0x7fff).
+ void PresentClientBuffer(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ ++m_presents;
+ Core::g_nx2_present_count.fetch_add(1, std::memory_order_relaxed);
+ if (m_presents <= 4 || (m_presents % 600) == 0) {
+ LOG_INFO(Service, "nx2:vi2 229 present #{} token={:#x} slot={} image={}", m_presents,
+ u64{w[2]} | (u64{w[3]} << 32), w[4], w[5]);
+ }
+ // Wait for the previous frame's fence instead of this one, so the game prepares frame N+1
+ // while the GPU thread runs frame N (the console keeps several frames in flight). Waiting
+ // for this frame serialized the two: 8-56 ms of every 34-105 ms frame on title 1.
+ // EDEN_NX2_FRAMES_AHEAD=0 restores the wait for the current frame.
+ static const bool frames_ahead = [] {
+ const char* value = std::getenv("EDEN_NX2_FRAMES_AHEAD");
+ return value == nullptr || value[0] != '0';
+ }();
+ const auto entered = std::chrono::steady_clock::now();
+ if (frames_ahead) {
+ LOG_DEBUG(Service, "nx2:vi2 present #{} fence ({}, {:#x}), waiting for ({}, {:#x}); now {:#x}",
+ m_presents, ~w[22], w[24], m_previous_fence_id, m_previous_fence_value,
+ ~w[22] < 192 ? system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(~w[22]) : 0U);
+ WaitForFrame(m_previous_fence_id, m_previous_fence_value);
+ m_previous_fence_id = ~w[22];
+ m_previous_fence_value = w[24];
+ } else {
+ WaitForFrame(~w[22], w[24]);
+ }
+ const auto waited = std::chrono::steady_clock::now();
+ Composite(w[5], ~w[22], w[24]);
+ const auto composited = std::chrono::steady_clock::now();
+ PaceToVblank();
+ const auto paced = std::chrono::steady_clock::now();
+ // Where each frame's time goes on the present thread, averaged per 600 presents: the
+ // game is blocked here while Eden's GPU thread catches up (fence) and composites.
+ {
+ using us = std::chrono::microseconds;
+ const auto to_us = [](auto d) { return std::chrono::duration_cast<us>(d).count(); };
+ m_stats.fence_us += to_us(waited - entered);
+ m_stats.composite_us += to_us(composited - waited);
+ m_stats.pace_us += to_us(paced - composited);
+ m_stats.max_fence_us = (std::max)(m_stats.max_fence_us, to_us(waited - entered));
+ if (m_stats.last_present != std::chrono::steady_clock::time_point{}) {
+ const long long frame = to_us(entered - m_stats.last_present);
+ m_stats.frame_us += frame;
+ m_stats.max_frame_us = (std::max)(m_stats.max_frame_us, frame);
+ m_stats.hitches += frame > 100'000 ? 1 : 0;
+ }
+ m_stats.last_present = entered;
+ if (++m_stats.presents == 600) {
+ LOG_INFO(Service,
+ "nx2:vi2 last 600 presents: frame {} us (max {}, {} over 100 ms), "
+ "blocked on the GPU fence {} us (max {}), composite {} us, vblank "
+ "pacing {} us",
+ m_stats.frame_us / 599, m_stats.max_frame_us, m_stats.hitches,
+ m_stats.fence_us / 600, m_stats.max_fence_us, m_stats.composite_us / 600,
+ m_stats.pace_us / 600);
+ m_stats = {.last_present = entered};
+ }
+ }
+ constexpr u64 AvailableSlots = 0x7fff;
+ TipcReplyWords(ctx, {static_cast<u32>(AvailableSlots),
+ static_cast<u32>(AvailableSlots >> 32), 0, 0, 0, 0});
+ }
+
+ // The console's compositor frees a buffer only once its frame's GPU fence has passed, which
+ // is what keeps a game from running ahead of the GPU. This reply always reports every buffer
+ // free, so pace the game here instead: wait (bounded, in case the GPU waits on the CPU) until
+ // Eden's GPU thread has processed the frame. Otherwise the game laps the GPU thread and
+ // rewrites command memory it hasn't read yet.
+ u32 m_wait_timeouts{};
+ struct PresentStats {
+ long long frame_us{}, fence_us{}, composite_us{}, pace_us{}, max_fence_us{};
+ long long max_frame_us{}, hitches{};
+ u32 presents{};
+ std::chrono::steady_clock::time_point last_present{};
+ } m_stats{};
+ u32 m_previous_fence_id{~0U}; // none yet (WaitForFrame ignores ids >= 192)
+ u32 m_previous_fence_value{};
+
+ // The console flips one frame per 60 Hz vblank and keeps the rest queued, so a game can't
+ // present faster than that. Every slot is reported free here: hold each present until the
+ // next vblank instead (title 4 ran at up to 160 fps, its timing and audio with it).
+ std::chrono::steady_clock::time_point m_next_vblank{};
+
+ void PaceToVblank() {
+ if (!Settings::values.use_speed_limit.GetValue() || Settings::SpeedLimit() == 0) {
+ return;
+ }
+ const auto interval = std::chrono::nanoseconds{100'000'000'000LL / 60 / Settings::SpeedLimit()};
+ const auto now = std::chrono::steady_clock::now();
+ if (now > m_next_vblank + interval) {
+ // Running slower than 60 fps: show at once and don't bank the lost time.
+ m_next_vblank = now;
+ } else if (now < m_next_vblank) {
+ std::this_thread::sleep_until(m_next_vblank);
+ }
+ m_next_vblank += interval;
+ }
+
+ void WaitForFrame(u32 fence_id, u32 fence_value) {
+ if (fence_id >= 192) {
+ return;
+ }
+ auto& syncpoints = system.Host1x().GetSyncpointManager();
+ const auto start = std::chrono::steady_clock::now();
+ // Long enough to cover shader/pipeline compile stalls: a game that gets more than a few
+ // command buffers ahead misses the equal-value gates its other channels acquire.
+ const auto deadline = start + std::chrono::milliseconds{4000};
+ if (!syncpoints.IsReadyGuest(fence_id, fence_value)) {
+ // Woken by the GPU thread as it reaches the fence. Polling with 200 us sleeps slept
+ // about 1 ms per check on Windows, added to every frame. An action left registered
+ // after a timeout fires later on a state nobody waits on.
+ struct Wake {
+ std::mutex mutex;
+ std::condition_variable cv;
+ bool done{};
+ };
+ auto wake = std::make_shared<Wake>();
+ syncpoints.RegisterGuestAction(fence_id, fence_value, [wake] {
+ {
+ std::scoped_lock lk{wake->mutex};
+ wake->done = true;
+ }
+ wake->cv.notify_all();
+ });
+ std::unique_lock lk{wake->mutex};
+ while (!wake->done && system.IsPoweredOn() &&
+ std::chrono::steady_clock::now() < deadline) {
+ wake->cv.wait_for(lk, std::chrono::milliseconds{50});
+ }
+ }
+ if (!syncpoints.IsReadyGuest(fence_id, fence_value) && m_wait_timeouts++ < 64) {
+ LOG_WARNING(Service, "nx2:vi2 present #{} fence ({}, {}) not reached in 4 s (now {})",
+ m_presents, fence_id, fence_value, syncpoints.GetGuestSyncpointValue(fence_id));
+ }
+ if (m_presents <= 8 || (m_presents % 600) == 0) {
+ LOG_INFO(Service, "nx2:vi2 present #{} fence ({}, {}) now {}, waited {} us",
+ m_presents, fence_id, fence_value, syncpoints.GetGuestSyncpointValue(fence_id),
+ std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::steady_clock::now() - start)
+ .count());
+ }
+ }
+
+ // Shows a presented native image: word 5 is the image id, ~word 22 / word 24 the GPU fence
+ // (syncpoint, value) to wait for (layout from changes.patch). The image is a block-linear
+ // RGBA8 surface in the console's native GOB layout at GPU address == CPU address.
+ void Composite(u32 image_id, u32 fence_id, u32 fence_value) {
+ u64 address{};
+ Nx2ImageLayout layout{};
+ {
+ std::scoped_lock lk{m_objects->mutex};
+ const auto image = m_objects->images.find(image_id);
+ if (image == m_objects->images.end()) {
+ return;
+ }
+ const auto memory = m_objects->memories.find(image->second.memory);
+ if (memory == m_objects->memories.end() || memory->second.address == 0) {
+ return;
+ }
+ address = memory->second.address;
+ layout = Nx2CalculateImageLayout(image->second.preference.data());
+ }
+ const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address);
+ if (!device) {
+ LOG_WARNING(Service, "nx2:vi2 present: image {} at {:#x} has no device mapping",
+ image_id, address);
+ return;
+ }
+ std::vector<Tegra::FramebufferConfig> layers{Tegra::FramebufferConfig{
+ .address = *device,
+ .offset = 0,
+ .width = layout.width,
+ .height = layout.height,
+ .stride = static_cast<u32>(layout.pitch / 4),
+ .pixel_format = android::PixelFormat::Rgba8888,
+ .transform_flags = android::BufferTransformFlags::Unset,
+ .crop_rect = {0, 0, static_cast<int>(layout.width), static_cast<int>(layout.height)},
+ .blending = Tegra::BlendMode::Opaque,
+ .native_tiling = true,
+ }};
+ std::vector<Nvidia::NvFence> fences;
+ if (fence_id < 192) {
+ fences.push_back({static_cast<s32>(fence_id), fence_value});
+ }
+ system.GPU().RequestComposite(std::move(layers), std::move(fences));
+ CaptureFrame();
+ DumpRanges();
+ system.SpeedLimiter().DoSpeedLimiting(system.CoreTiming().GetGlobalTimeUs());
+ system.GetPerfStats().EndSystemFrame();
+ system.GetPerfStats().BeginSystemFrame();
+ }
+
+ // EDEN_NX2_FRAME_CAPTURE=<dir>: every 120th present, save what the renderer shows as
+ // nx2-frame-<n>.tga in <dir> (lets hidden test runs check the output; idea from changes.patch).
+ void CaptureFrame() {
+ static const char* const directory = std::getenv("EDEN_NX2_FRAME_CAPTURE");
+ if (directory == nullptr || (m_presents % 120) != 1 || m_capture_pending->exchange(true)) {
+ return;
+ }
+ const auto layout = Layout::DefaultFrameLayout(1280, 720);
+ auto pixels = std::make_shared<std::vector<u8>>(layout.width * layout.height * 4);
+ const std::string path =
+ fmt::format("{}/nx2-frame-{:05}.tga", directory, m_presents);
+ auto pending = m_capture_pending; // the screenshot callback can outlive the session
+ system.GPU().Renderer().RequestScreenshot(
+ pixels->data(),
+ [pixels, layout, path, pending](bool invert_y) {
+ std::array<u8, 18> header{};
+ header[2] = 2; // uncompressed true colour
+ header[12] = static_cast<u8>(layout.width);
+ header[13] = static_cast<u8>(layout.width >> 8);
+ header[14] = static_cast<u8>(layout.height);
+ header[15] = static_cast<u8>(layout.height >> 8);
+ header[16] = 32;
+ // Descriptor bit 5: rows top to bottom. invert_y means the data is bottom-up.
+ header[17] = invert_y ? 0x08 : 0x28;
+ if (std::FILE* file = std::fopen(path.c_str(), "wb")) {
+ std::fwrite(header.data(), 1, header.size(), file);
+ std::fwrite(pixels->data(), 1, pixels->size(), file);
+ std::fclose(file);
+ }
+ pending->store(false);
+ },
+ layout);
+ }
+
+ // EDEN_NX2_DUMP=<present>:<hex address>:<hex size>[,<hex address>:<hex size>...]: at that
+ // present, flush those GPU ranges back from the host GPU and save their bytes as
+ // nx2-dump-<address>.bin in the EDEN_NX2_FRAME_CAPTURE directory (to inspect what a pass
+ // rendered, e.g. a 3D volume, offline).
+ void DumpRanges() {
+ static const std::string spec = [] {
+ const char* text = std::getenv("EDEN_NX2_DUMP");
+ return text ? std::string{text} : std::string{};
+ }();
+ if (spec.empty() || m_presents != std::strtoull(spec.c_str(), nullptr, 10)) {
+ return;
+ }
+ const char* const directory = std::getenv("EDEN_NX2_FRAME_CAPTURE");
+ size_t position = spec.find(':');
+ while (position != std::string::npos) {
+ char* end{};
+ const u64 address = std::strtoull(spec.c_str() + position + 1, &end, 16);
+ const u64 size = *end == ':' ? std::strtoull(end + 1, &end, 16) : 0;
+ position = spec.find(',', position + 1); // the next "address:size"
+ const std::string path = fmt::format("{}/nx2-dump-{:x}.bin", directory ? directory : ".", address);
+ // On the GPU thread: waiting for it here can deadlock with a GPU wait on this present.
+ static_cast<void>(system.GPU().RunOnGpuThread([&system = system, address, size, path] {
+ std::vector<u8> bytes(size);
+ for (u64 current = address; current < address + size;) {
+ const u64 chunk = std::min((current | 0xfff) + 1, address + size) - current;
+ if (const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(current)) {
+ system.GPU().Renderer().ReadRasterizer()->FlushRegion(*device, chunk);
+ system.Host1x().MemoryManager().ReadBlock(*device, bytes.data() + (current - address), chunk);
+ }
+ current += chunk;
+ }
+ if (std::FILE* file = std::fopen(path.c_str(), "wb")) {
+ std::fwrite(bytes.data(), 1, bytes.size(), file);
+ std::fclose(file);
+ }
+ LOG_INFO(Service, "nx2: dumped {:#x} bytes at {:#x} to {}", size, address, path);
+ }));
+ }
+ }
+
+ // 228(u64 token, u32 slot): return a buffer without presenting it.
+ void CancelClientBuffer(HLERequestContext& ctx) {
+ constexpr u64 AvailableSlots = 0x7fff;
+ TipcReplyWords(ctx, {static_cast<u32>(AvailableSlots),
+ static_cast<u32>(AvailableSlots >> 32), 0, 0, 0, 0});
+ }
+
+ void ReturnOne(HLERequestContext& ctx) {
+ LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC {}, returning 1", ctx.GetCommand());
+ TipcReplyWords(ctx, {1});
+ }
+
+ // The same reply as the auto stub (success, no data), without its log lines.
+ void SucceedQuietly(HLERequestContext& ctx) {
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(ResultSuccess);
+ }
+
+ // Must be < 8; nn::vi::detail::LayerHistory creates 7 timestamp recorders per layer.
+ void GetTimestampRecorderCount(HLERequestContext& ctx) {
+ LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC 5, returning 7");
+ TipcReplyWords(ctx, {7});
+ }
+
+ void ReturnEvent(HLERequestContext& ctx) {
+ LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC {} [2]={:#x} [3]={:#x}, returning an event",
+ ctx.GetCommand(), ctx.CommandBuffer()[2], ctx.CommandBuffer()[3]);
+ auto* event = m_context.CreateEvent(fmt::format("nx2:vi2:Event{}", m_events.size()));
+ m_events.push_back(event);
+ IPC::ResponseBuilder rb{ctx, 2, 1, 0};
+ rb.Push(ResultSuccess);
+ rb.PushCopyObjects(ctx, event->GetReadableEvent());
+ }
+
+ // 31(u32 kind) and 32(u32 display): events signalled on every vblank. Title 1's main thread
+ // waits on the 32 event until it has fired twice before it submits any GPU work, and hung
+ // there while it never fired.
+ void GetVsyncEvent(HLERequestContext& ctx) {
+ LOG_INFO(Service, "nx2:vi2 {} vsync event, [2]={:#x}", ctx.GetCommand(), ctx.CommandBuffer()[2]);
+ auto* event = m_context.CreateEvent(fmt::format("nx2:vi2:Vsync{}", m_events.size()));
+ m_events.push_back(event);
+ {
+ std::scoped_lock lk{m_vsync->mutex};
+ m_vsync->events.push_back(event);
+ }
+ if (!m_vsync_timer) {
+ m_vsync_timer = Core::Timing::CreateEvent(
+ "nx2:vi2 vsync",
+ [vsync_state = m_vsync, &kernel = system.Kernel()](
+ s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> {
+ std::scoped_lock lk{vsync_state->mutex};
+ for (auto* vsync : vsync_state->events) {
+ vsync->Signal(kernel);
+ }
+ return std::nullopt;
+ });
+ // EDEN_NX2_VSYNC_HZ=<rate> (experiment): the game starts frames on this event, so a
+ // frame that misses a 60 Hz tick idles until the next one.
+ static const std::chrono::nanoseconds VblankInterval{[] {
+ const char* text = std::getenv("EDEN_NX2_VSYNC_HZ");
+ const long long hz = text ? (std::max)(std::strtoll(text, nullptr, 10), 1LL) : 60LL;
+ return 1'000'000'000LL / hz;
+ }()};
+ system.CoreTiming().ScheduleLoopingEvent(VblankInterval, VblankInterval, m_vsync_timer);
+ }
+ IPC::ResponseBuilder rb{ctx, 2, 1, 0};
+ rb.Push(ResultSuccess);
+ rb.PushCopyObjects(ctx, event->GetReadableEvent());
+ }
+
+ KernelHelpers::ServiceContext m_context;
+ std::shared_ptr<Nx2NativeObjects> m_objects;
+ std::vector<Kernel::KEvent*> m_events;
+ // Shared with the vsync timer callback, which can outlive the session by one call.
+ struct VsyncState {
+ std::mutex mutex;
+ std::vector<Kernel::KEvent*> events;
+ };
+ std::shared_ptr<VsyncState> m_vsync = std::make_shared<VsyncState>();
+ std::shared_ptr<Core::Timing::EventType> m_vsync_timer;
+ u64 m_presents{};
+ std::shared_ptr<std::atomic<bool>> m_capture_pending = std::make_shared<std::atomic<bool>>(false);
+};
+
+// The Switch 2 video decoder (nn::codec::video::detail::ipc). gsv 902/1002 hand out a session
+// whose TIPC 1 (CreateVideoDecoderService: handle at word 3, result at word 4) creates a decoder
+// service, another session of this class. Layouts read from the SDK's nn::codec VP9 decoder:
+// - 200(copy handle: work buffer transfer memory, u64 size, u32, u32, u32): initialize.
+// - 101(X: codec configuration, u32 0, u32 width, u32 height, u32 rate, u32 max input): configure.
+// - 3(): an event (copy handle at word 3, result at word 4) the client attaches as its
+// submission-ready event.
+// - 6 / 207 / 8: start / stop / flush; 115: finalize.
+// - 311(u32 image id): register an output image (gsv 911 creates it); reply a 16-byte
+// NativeSyncHandle, all zero reading as already signalled. The client acquires output images
+// in registration/release order from a ring of its own (no IPC).
+// - 314(u32 image id, u32, NativeSyncHandle): give an output image back; reply a new handle.
+// - 212(u32 image id): unregister it.
+// - 109(A: one compressed frame, u64 pts in us, u8 end of stream): submit input; reply a u64.
+// - 213(): the next finished frame (16 words): u32 image id, u32 width, u32 height, u32 width,
+// u32 height, u32 colour space, u64 pts, u8 end of stream.
+// Frames are not decoded yet: each submitted frame completes at once into the next free output
+// image, so movies run to their end (with no picture) instead of stalling.
+class INx2VideoSession final : public ServiceFramework<INx2VideoSession> {
+public:
+ explicit INx2VideoSession(Core::System& system_, std::shared_ptr<Nx2NativeObjects> objects_,
+ const char* name)
+ : ServiceFramework{system_, name}, m_context{system_, name},
+ m_objects{std::move(objects_)} {
+ RegisterHandlersTipc({
+ {1, &INx2VideoSession::CreateDecoderService, "CreateVideoDecoderService"},
+ {3, &INx2VideoSession::GetEvent, "GetEvent"},
+ {6, &INx2VideoSession::Succeed, "Start"},
+ {8, &INx2VideoSession::Succeed, "Flush"},
+ {101, &INx2VideoSession::Configure, "Configure"},
+ {109, &INx2VideoSession::SubmitInput, "SubmitInputBuffer"},
+ {115, &INx2VideoSession::Succeed, "Finalize"},
+ {200, &INx2VideoSession::Succeed, "Initialize"},
+ {207, &INx2VideoSession::Succeed, "Stop"},
+ {212, &INx2VideoSession::UnregisterBuffer, "UnregisterBuffer"},
+ {213, &INx2VideoSession::GetOutputInfo, "GetOutputInfo"},
+ {311, &INx2VideoSession::RegisterBuffer, "RegisterBuffer"},
+ {314, &INx2VideoSession::ReleaseBuffer, "ReleaseBuffer"},
+ });
+ }
+
+ ~INx2VideoSession() override {
+ // The ready timer runs on the HostTiming thread: clear its event under its lock first.
+ {
+ std::scoped_lock lk{m_ready->mutex};
+ m_ready->event = nullptr;
+ }
+ if (m_ready_timer) {
+ system.CoreTiming().UnscheduleEvent(m_ready_timer);
+ }
+ if (m_event != nullptr) {
+ m_context.CloseEvent(m_event);
+ }
+ }
+
+private:
+ struct Output {
+ u32 image;
+ u64 pts;
+ bool end;
+ };
+
+ void Log(HLERequestContext& ctx, std::string_view what) {
+ if (m_logged++ < 64) {
+ const auto* w = ctx.CommandBuffer();
+ LOG_INFO(Service, "{} TIPC {} {}: [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x} [6]={:#x} [7]={:#x}",
+ GetServiceName(), ctx.GetCommand(), what, w[2], w[3], w[4], w[5], w[6], w[7]);
+ }
+ }
+
+ void CreateDecoderService(HLERequestContext& ctx) {
+ Log(ctx, "create decoder service");
+ ReturnRawSession(system, ctx,
+ std::make_shared<INx2VideoSession>(system, m_objects, "nx2:vdecsvc"));
+ }
+
+ void Succeed(HLERequestContext& ctx) {
+ Log(ctx, "");
+ TipcReplyWords(ctx, {});
+ }
+
+ void Configure(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ // Words 2-3 are the X descriptor of the configuration data.
+ m_width = w[5];
+ m_height = w[6];
+ Log(ctx, "configure");
+ TipcReplyWords(ctx, {});
+ }
+
+ void GetEvent(HLERequestContext& ctx) {
+ Log(ctx, "event");
+ if (m_event == nullptr) {
+ m_event = m_context.CreateEvent(fmt::format("{}:Ready", GetServiceName()));
+ // Ready for input at all times (frames complete as they are submitted). The client
+ // attaches it auto-clearing, so one signal is consumed by the first wait: re-signal
+ // every 4 ms; a spurious wake only makes it check again.
+ {
+ std::scoped_lock lk{m_ready->mutex};
+ m_ready->event = m_event;
+ }
+ m_ready_timer = Core::Timing::CreateEvent(
+ "nx2:vdec ready",
+ [ready = m_ready, &kernel = system.Kernel()](
+ s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> {
+ std::scoped_lock lk{ready->mutex};
+ if (ready->event != nullptr) {
+ ready->event->Signal(kernel);
+ }
+ return std::nullopt;
+ });
+ constexpr std::chrono::nanoseconds ReadyInterval{std::chrono::milliseconds{4}};
+ system.CoreTiming().ScheduleLoopingEvent(ReadyInterval, ReadyInterval, m_ready_timer);
+ }
+ m_event->Signal(system.Kernel());
+ IPC::ResponseBuilder rb{ctx, 2, 1, 0};
+ rb.Push(ResultSuccess);
+ rb.PushCopyObjects(ctx, m_event->GetReadableEvent());
+ }
+
+ void RegisterBuffer(HLERequestContext& ctx) {
+ Log(ctx, "register output");
+ GiveBack(ctx.CommandBuffer()[2]);
+ TipcReplyWords(ctx, {0, 0, 0, 0});
+ }
+
+ void ReleaseBuffer(HLERequestContext& ctx) {
+ Log(ctx, "release output");
+ GiveBack(ctx.CommandBuffer()[2]);
+ TipcReplyWords(ctx, {0, 0, 0, 0});
+ }
+
+ void UnregisterBuffer(HLERequestContext& ctx) {
+ Log(ctx, "unregister output");
+ std::erase(m_free, ctx.CommandBuffer()[2]);
+ TipcReplyWords(ctx, {});
+ }
+
+ // An output image is free again: a frame waiting for one completes into it.
+ void GiveBack(u32 image) {
+ if (!m_waiting.empty()) {
+ m_outputs.push_back({image, m_waiting.front().pts, m_waiting.front().end});
+ m_waiting.pop_front();
+ return;
+ }
+ m_free.push_back(image);
+ }
+
+ void SubmitInput(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ // Words 2-4 are the A descriptor of the compressed frame.
+ const u64 pts = u64{w[5]} | (u64{w[6]} << 32);
+ const bool end = (w[7] & 0xFF) != 0;
+ Log(ctx, "submit");
+ if (!m_free.empty()) {
+ m_outputs.push_back({m_free.front(), pts, end});
+ m_free.pop_front();
+ } else {
+ m_waiting.push_back({0, pts, end});
+ }
+ const u64 id = ++m_submitted;
+ if (m_event != nullptr) {
+ m_event->Signal(system.Kernel());
+ }
+ TipcReplyWords(ctx, {static_cast<u32>(id), static_cast<u32>(id >> 32)});
+ }
+
+ void GetOutputInfo(HLERequestContext& ctx) {
+ Log(ctx, "output info");
+ if (m_outputs.empty()) {
+ // Nothing finished: the code the client's own Acquire uses for an empty ring.
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.PushRaw<u32>(0x1c286f);
+ return;
+ }
+ const Output output = m_outputs.front();
+ m_outputs.pop_front();
+ TipcReplyWords(ctx, {output.image, m_width, m_height, m_width, m_height, 0,
+ static_cast<u32>(output.pts), static_cast<u32>(output.pts >> 32),
+ output.end ? 1U : 0U, 0, 0, 0, 0, 0, 0, 0});
+ }
+
+ struct ReadyState {
+ std::mutex mutex;
+ Kernel::KEvent* event{};
+ };
+
+ KernelHelpers::ServiceContext m_context;
+ std::shared_ptr<Nx2NativeObjects> m_objects;
+ Kernel::KEvent* m_event{};
+ std::shared_ptr<ReadyState> m_ready = std::make_shared<ReadyState>();
+ std::shared_ptr<Core::Timing::EventType> m_ready_timer;
+ u32 m_width{};
+ u32 m_height{};
+ u64 m_submitted{};
+ u32 m_logged{};
+ std::deque<u32> m_free;
+ std::deque<Output> m_waiting;
+ std::deque<Output> m_outputs;
+};
+
+// Placeholder for the Switch 2 graphics-system server that nn::gsv::BootWithRemoteMemory talks
+// to over TIPC. Nothing behind it is emulated: commands succeed and are logged, so boot can show
+// what comes next. Commands 202 and 1402 hand out the syncpt and nvsrv sessions the SDK uses.
+class INx2GraphicsSystemSession final : public ServiceFramework<INx2GraphicsSystemSession> {
+public:
+ explicit INx2GraphicsSystemSession(Core::System& system_)
+ : ServiceFramework{system_, "nx2:gsv"}, m_context{system_, "nx2:gsv"} {
+ RegisterHandlersTipc({
+ {4, &INx2GraphicsSystemSession::Stub, "Boot (Switch 2, guessed)"},
+ {5, &INx2GraphicsSystemSession::Stub, "Unknown5"},
+ {101, &INx2GraphicsSystemSession::Stub, "Unknown101"},
+ {202, &INx2GraphicsSystemSession::OpenSyncptSession, "OpenSyncptSession (guessed)"},
+ {204, &INx2GraphicsSystemSession::AttachSyncptMemory, "AttachSyncptMemory (guessed)"},
+ {305, &INx2GraphicsSystemSession::OpenViSession, "OpenViSession (guessed)"},
+ {497, &INx2GraphicsSystemSession::OpenGpuSession, "OpenGpuSession (guessed)"},
+ {111, &INx2GraphicsSystemSession::CreateNativeMemory, "CreateNativeMemory"},
+ {112, &INx2GraphicsSystemSession::DestroyNativeMemory, "DestroyNativeMemory"},
+ {114, &INx2GraphicsSystemSession::BindNativeMemory, "BindNativeMemory"},
+ {314, &INx2GraphicsSystemSession::ConfigureWindowImagePool, "DestroyNativeWindowAliasImagePool"},
+ {315, &INx2GraphicsSystemSession::ConfigureWindowImagePool, "CreateNativeWindowAliasImagePool"},
+ {423, &INx2GraphicsSystemSession::BindNativeGpuMemory, "BindNativeGpuMemory"},
+ {429, &INx2GraphicsSystemSession::CreateNativeImage, "CreateNativeImage"},
+ {430, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyNativeImage"},
+ {435, &INx2GraphicsSystemSession::CalculateNativeImageSize, "CalculateNativeImageSize"},
+ {1402, &INx2GraphicsSystemSession::OpenNvsrvSession, "OpenNvsrvSession (guessed)"},
+ {1510, &INx2GraphicsSystemSession::CalculateRequirements, "CalculateRequirements"},
+ {991, &INx2GraphicsSystemSession::CalculateDecoderImagePoolBuffer,
+ "CalculateDecoderImagePoolBuffer"},
+ {901, &INx2GraphicsSystemSession::Stub, "VideoInitialize (guessed)"},
+ {902, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSession (guessed)"},
+ {1001, &INx2GraphicsSystemSession::Stub, "VideoInitialize2 (guessed)"},
+ {1002, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSession2 (guessed)"},
+ // Same request as 429: the image backing a decoder output (nn::codec image pools).
+ {911, &INx2GraphicsSystemSession::CreateNativeImage, "CreateDecoderImage (guessed)"},
+ });
+ }
+
+ ~INx2GraphicsSystemSession() override {
+ if (m_syncpt_poll) {
+ system.CoreTiming().UnscheduleEvent(m_syncpt_poll);
+ }
+ for (auto* event : m_syncpt_events) {
+ if (event != nullptr) {
+ m_context.CloseEvent(event);
+ }
+ }
+ }
+
+private:
+ // Index of the first raw data word of a TIPC request: header (2 words), then an optional
+ // special header with pid and copy/move handle words.
+ static size_t TipcDataOffset(const u32* cmd_buf) {
+ if ((cmd_buf[1] & 0x80000000) == 0) {
+ return 2;
+ }
+ const u32 desc = cmd_buf[2];
+ return 3 + ((desc & 1) ? 2 : 0) + ((desc >> 1) & 0xF) + ((desc >> 5) & 0xF);
+ }
+
+ void Stub(HLERequestContext& ctx) {
+ const auto* cmd_buf = ctx.CommandBuffer();
+ LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC {} [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x}",
+ ctx.GetCommand(), cmd_buf[2], cmd_buf[3], cmd_buf[4], cmd_buf[5]);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(ResultSuccess);
+ }
+
+ // nn::codec::video::detail::ipc::Initialize sends 901, 902, 1001, 1002. 902 and 1002 return
+ // a session handle at word 3 and the result at word 4; left to the auto stub, the handle was 0
+ // and InitializeVp9Decoder failed with 2001-0114 (title 1 aborts when a movie starts).
+ void OpenVideoSession(HLERequestContext& ctx) {
+ const bool second = ctx.GetCommand() == 1002;
+ LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC {}, returning a video decoder session",
+ ctx.GetCommand());
+ ReturnRawSession(system, ctx,
+ std::make_shared<INx2VideoSession>(system, m_objects,
+ second ? "nx2:vdec2" : "nx2:vdec"));
+ }
+
+ void OpenSyncptSession(HLERequestContext& ctx) {
+ // Used by nn::syncpt::sdk::InitializeEx; handle at word 3, result at word 4.
+ LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 202, returning a nx2:syncpt session");
+ ReturnRawSession(system, ctx, std::make_shared<INx2SyncptSession>(system));
+ }
+
+ // nn::syncpt::sdk::InitializeEx sends two transfer memories (0x1000 bytes RW, then 0x2000
+ // bytes the client maps read-only) and expects the server to have written a header into the
+ // second one: u32 version (must be 1) at +0, u32 base id at +4, u32 count at +8 (the client
+ // requires ((count + 0x40) >> 6) - 1 < 0x40), then offsets of: the u64 syncpoint values (RO,
+ // +0xc), the RW "armed" summary word (+0x10), the RO summary word (+0x14), the RW armed bits
+ // (+0x18) and the RO acknowledge bits (+0x1c).
+ // To wait, the client makes its RW bit differ from the RO bit, signals the first returned
+ // event (it attaches that handle as writable), and waits on the second until the server makes
+ // the bits equal again; it then rereads the value and rearms if the threshold isn't reached.
+ // Reply: the two handles (words 3 and 4), then the result.
+ static constexpr u32 SyncptValuesOffset = 0x40; // RO, u64 per syncpoint
+ static constexpr u32 SyncptRoSummaryOffset = 0x1040;
+ static constexpr u32 SyncptRoBitsOffset = 0x1100; // RO, up to 0x40 u64 words
+ static constexpr u32 SyncptRwSummaryOffset = 0x0;
+ static constexpr u32 SyncptRwBitsOffset = 0x100; // RW, up to 0x40 u64 words
+ static constexpr u32 HostSyncpoints = 192; // Tegra::Host1x::SyncpointManager
+
+ void AttachSyncptMemory(HLERequestContext& ctx) {
+ // RequestParser doesn't skip TIPC handle words, so read the raw data words.
+ const auto* cmd_buf = ctx.CommandBuffer();
+ const size_t data = TipcDataOffset(cmd_buf);
+ const u32 version_requested = cmd_buf[data];
+ const u32 max_syncpoints = std::min<u32>(cmd_buf[data + 1], (0x1000 - 0x40) / 8);
+ LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 204, version={} max_syncpoints={:#x}",
+ version_requested, max_syncpoints);
+
+ auto rw = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(ctx.GetCopyHandle(0));
+ auto ro = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(ctx.GetCopyHandle(1));
+ if (rw.IsNotNull() && ro.IsNotNull() && ro->GetSize() >= 0x2000) {
+ m_syncpt_memory = &ro->GetOwner()->GetMemory();
+ m_syncpt_rw = GetInteger(rw->GetSourceAddress());
+ m_syncpt_ro = GetInteger(ro->GetSourceAddress());
+ m_syncpt_count = max_syncpoints;
+ auto& memory = *m_syncpt_memory;
+ memory.Write32(m_syncpt_ro + 0x00, 1); // version
+ memory.Write32(m_syncpt_ro + 0x04, 0); // base id (guess)
+ memory.Write32(m_syncpt_ro + 0x08, max_syncpoints);
+ memory.Write32(m_syncpt_ro + 0x0c, SyncptValuesOffset);
+ memory.Write32(m_syncpt_ro + 0x10, SyncptRwSummaryOffset);
+ memory.Write32(m_syncpt_ro + 0x14, SyncptRoSummaryOffset);
+ memory.Write32(m_syncpt_ro + 0x18, SyncptRwBitsOffset);
+ memory.Write32(m_syncpt_ro + 0x1c, SyncptRoBitsOffset);
+ UpdateSyncptValues();
+ if (!m_syncpt_poll) {
+ m_syncpt_poll = Core::Timing::CreateEvent(
+ "nx2:gsv:SyncptPoll",
+ [this](s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> {
+ PollSyncpts();
+ return std::nullopt;
+ });
+ system.CoreTiming().ScheduleLoopingEvent(SyncptPollInterval, SyncptPollInterval,
+ m_syncpt_poll);
+ }
+ } else {
+ LOG_ERROR(Service, "nx2:gsv TIPC 204: missing syncpt transfer memory");
+ }
+
+ if (m_syncpt_events[0] == nullptr) {
+ m_syncpt_events[0] = m_context.CreateEvent("nx2:gsv:SyncptRequest");
+ m_syncpt_events[1] = m_context.CreateEvent("nx2:gsv:SyncptNotify");
+ }
+ IPC::ResponseBuilder rb{ctx, 2, 2, 0};
+ rb.Push(ResultSuccess);
+ rb.PushCopyObjects(ctx, *m_syncpt_events[0]); // signalled by the client
+ rb.PushCopyObjects(ctx, m_syncpt_events[1]->GetReadableEvent());
+ }
+
+ // Mirrors Eden's guest syncpoint values into the shared RO page. Returns true if any changed.
+ bool UpdateSyncptValues() {
+ auto& manager = system.Host1x().GetSyncpointManager();
+ bool changed = false;
+ for (u32 id = 0; id < std::min(m_syncpt_count, HostSyncpoints); ++id) {
+ const u64 value = manager.GetGuestSyncpointValue(id);
+ const u64 address = m_syncpt_ro + SyncptValuesOffset + id * 8;
+ if (m_syncpt_memory->Read64(address) != value) {
+ m_syncpt_memory->Write64(address, value);
+ changed = true;
+ }
+ }
+ return changed;
+ }
+
+ // Answers every armed wait: the value page is current, so the client rechecks its threshold
+ // and rearms if it still has to wait. Acknowledging without a value change costs the client a
+ // wakeup per poll, which keeps waits whose syncpoint we never see from hanging.
+ void PollSyncpts() {
+ if (m_syncpt_memory == nullptr) {
+ return;
+ }
+ UpdateSyncptValues();
+ bool acked = false;
+ const u32 words = (m_syncpt_count + 63) / 64;
+ for (u32 w = 0; w < words; ++w) {
+ const u64 rw_bits = m_syncpt_memory->Read64(m_syncpt_rw + SyncptRwBitsOffset + w * 8);
+ const u64 ro_address = m_syncpt_ro + SyncptRoBitsOffset + w * 8;
+ if (m_syncpt_memory->Read64(ro_address) != rw_bits) {
+ m_syncpt_memory->Write64(ro_address, rw_bits);
+ acked = true;
+ }
+ }
+ m_syncpt_memory->Write64(m_syncpt_ro + SyncptRoSummaryOffset,
+ m_syncpt_memory->Read64(m_syncpt_rw + SyncptRwSummaryOffset));
+ if (acked) {
+ m_syncpt_events[1]->Signal(system.Kernel());
+ }
+ }
+
+ // nn::gsv::shim::CalculateRequirements(u64* size, u64* alignment, preference, u32): the 0x40-byte
+ // InterModuleUserImagePreference (words 2..17) and a u32 (word 18). A block-linear preference
+ // (MakeInterModuleBlockUserImagePreference) holds kind 2 at word 3, (width - 1) | (height - 1)
+ // << 16 at word 4, two u32s at words 6-7 and the pixel format at word 8. Reply: result, u64
+ // size, u64 alignment; the client raises the alignment to at least 4 KiB and rounds the size.
+ void CalculateRequirements(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ const auto layout = Nx2CalculateImageLayout(w + 2);
+ LOG_DEBUG(Service, "nx2:gsv 1510 kind={} {}x{} block={} pitch={:#x} format={} -> size={:#x} align={:#x}",
+ w[3], layout.width, layout.height, layout.block_height_log2, layout.pitch, w[8],
+ layout.size, layout.alignment);
+ TipcReplyWords(ctx, {static_cast<u32>(layout.size), static_cast<u32>(layout.size >> 32),
+ static_cast<u32>(layout.alignment),
+ static_cast<u32>(layout.alignment >> 32)});
+ }
+
+ // nn::codec::CalculateDecoderImagePoolBuffer(size*, alignment*, VideoDecoderImagePreference,
+ // u32 count) sends 991(preference[16], u32 count) and reads result, u64 size, u64 alignment;
+ // the client raises the alignment to at least 4 KiB and rounds the size up to it. The
+ // preference (MakeDecodeImagePreference) holds an output layout code at word 1 (1/4 for
+ // format 3, which the movie player uses, read here as NV12) and (width - 1) | (height - 1) << 16
+ // at word 2. Left to the auto stub, the reply words keep the request's, so the size became
+ // 0x167027f'00000001 and title 1's movie player aborted (main+0x19e5b58) when its first movie
+ // started. Sizes are block-linear NV12 (luma, then half-height chroma, 128-row blocks) or 4
+ // bytes per pixel for the other codes; nothing is decoded into them yet.
+ void CalculateDecoderImagePoolBuffer(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ const u32 code = w[3];
+ const u64 width = (w[4] & 0xFFFF) + 1;
+ const u64 height = (w[4] >> 16) + 1;
+ const u32 count = std::max<u32>(w[18], 1);
+ constexpr u64 BlockRows = 128;
+ const bool nv12 = code == 1 || code == 4;
+ const u64 pitch = Common::AlignUp(width * (nv12 ? 1 : 4), 64);
+ u64 size = pitch * Common::AlignUp(height, BlockRows);
+ if (nv12) {
+ size += pitch * Common::AlignUp((height + 1) / 2, BlockRows);
+ }
+ size *= count;
+ const u64 alignment = 64 * BlockRows;
+ LOG_INFO(Service, "nx2:gsv 991 decoder image pool code={} {}x{} count={} -> size={:#x}",
+ code, width, height, count, size);
+ TipcReplyWords(ctx, {static_cast<u32>(size), static_cast<u32>(size >> 32),
+ static_cast<u32>(alignment), static_cast<u32>(alignment >> 32)});
+ }
+
+ // 435(preference[16], u32 count): result, u64 size of a GPU native image.
+ void CalculateNativeImageSize(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ const u64 size = Nx2CalculateImageLayout(w + 2).size * std::max<u32>(w[18], 1);
+ LOG_DEBUG(Service, "nx2:gsv 435 count={} -> size={:#x}", w[18], size);
+ TipcReplyWords(ctx, {static_cast<u32>(size), static_cast<u32>(size >> 32)});
+ }
+
+ static u64 Pair(const u32* w, size_t i) {
+ return u64{w[i]} | (u64{w[i + 1]} << 32);
+ }
+
+ // 111(u8 mode at word 2, u64 size): result, u32 native memory id.
+ void CreateNativeMemory(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ const u64 size = Pair(w, 3);
+ std::scoped_lock lk{m_objects->mutex};
+ const u32 id = m_objects->next_memory++;
+ m_objects->memories[id] = {size, 0};
+ LOG_INFO(Service, "nx2:gsv 111 CreateNativeMemory mode={} size={:#x} -> id {}", w[2] & 0xFF,
+ size, id);
+ TipcReplyWords(ctx, {id});
+ }
+
+ void DestroyNativeMemory(HLERequestContext& ctx) {
+ const u32 id = ctx.CommandBuffer()[2];
+ std::scoped_lock lk{m_objects->mutex};
+ m_objects->memories.erase(id);
+ LOG_DEBUG(Service, "nx2:gsv 112 DestroyNativeMemory id {}", id);
+ TipcReplyWords(ctx, {});
+ }
+
+ // 114(u32 id, u64 address, u64 size, u64 flags, u64 offset): back native memory with process
+ // memory.
+ void BindNativeMemory(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ BindMemory(w[2], Pair(w, 3), Pair(w, 5), "114");
+ TipcReplyWords(ctx, {});
+ }
+
+ // 423(u32 id, u64 address, u64 size, u32 parent GpuMemoryHandle, u64 offset, ...): back
+ // native memory with part of GPU-accessible memory.
+ void BindNativeGpuMemory(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ LOG_DEBUG(Service, "nx2:gsv 423 parent={} offset={:#x}", w[7], Pair(w, 8));
+ BindMemory(w[2], Pair(w, 3), Pair(w, 5), "423");
+ TipcReplyWords(ctx, {});
+ }
+
+ void BindMemory(u32 id, u64 address, u64 size, const char* command) {
+ std::scoped_lock lk{m_objects->mutex};
+ auto& memory = m_objects->memories[id];
+ memory.address = address;
+ if (memory.size == 0) {
+ memory.size = size;
+ }
+ LOG_INFO(Service, "nx2:gsv {} bind native memory {} at {:#x}+{:#x}", command, id, address,
+ size);
+ }
+
+ // 429(preference[16], u32 count, u32 memory id): result, u32 native image id.
+ void CreateNativeImage(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ std::scoped_lock lk{m_objects->mutex};
+ const u32 id = m_objects->next_image++;
+ auto& image = m_objects->images[id];
+ image.memory = w[19];
+ std::copy_n(w + 2, image.preference.size(), image.preference.begin());
+ image.size = Nx2CalculateImageLayout(w + 2).size;
+ LOG_INFO(Service, "nx2:gsv 429 CreateNativeImage memory={} count={} size={:#x} -> id {}",
+ w[19], w[18], image.size, id);
+ TipcReplyWords(ctx, {id});
+ }
+
+ void DestroyNativeImage(HLERequestContext& ctx) {
+ const u32 id = ctx.CommandBuffer()[2];
+ std::scoped_lock lk{m_objects->mutex};
+ m_objects->images.erase(id);
+ TipcReplyWords(ctx, {});
+ }
+
+ // 315(u32 image, u32 count) / 314(u32 image): add/remove a window alias image pool.
+ void ConfigureWindowImagePool(HLERequestContext& ctx) {
+ const u32 id = ctx.CommandBuffer()[2];
+ const bool create = ctx.GetCommand() == 315;
+ std::scoped_lock lk{m_objects->mutex};
+ if (auto it = m_objects->images.find(id); it != m_objects->images.end()) {
+ it->second.window_pool = create;
+ }
+ LOG_INFO(Service, "nx2:gsv {} window image pool image {}", ctx.GetCommand(), id);
+ TipcReplyWords(ctx, {});
+ }
+
+ void OpenViSession(HLERequestContext& ctx) {
+ // Used by nn::vi2::sdk::Initialize; takes one u64, returns a session (handle at word 3,
+ // result at word 4).
+ const auto* cmd_buf = ctx.CommandBuffer();
+ const size_t data = TipcDataOffset(cmd_buf);
+ LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 305 arg={:#x}, returning a nx2:vi2 session",
+ u64{cmd_buf[data]} | (u64{cmd_buf[data + 1]} << 32));
+ ReturnRawSession(system, ctx, std::make_shared<INx2Vi2Session>(system, m_objects));
+ }
+
+ void OpenGpuSession(HLERequestContext& ctx) {
+ // Used by nn::gpu::sdk::BootWithRemoteMemory; takes one u64, returns a session (handle at
+ // word 3, result at word 4) that the GPU SDK keeps.
+ const auto* cmd_buf = ctx.CommandBuffer();
+ const size_t data = TipcDataOffset(cmd_buf);
+ LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 497 arg={:#x}, returning a nx2:gpu session",
+ u64{cmd_buf[data]} | (u64{cmd_buf[data + 1]} << 32));
+ ReturnRawSession(system, ctx, std::make_shared<INx2GpuSession>(system));
+ }
+
+ void OpenNvsrvSession(HLERequestContext& ctx) {
+ // The SDK reads the handle at word 3 and the result at word 4.
+ // nvsrv speaks the nvdrv INvDrvServices protocol (Open/Ioctl/Close/...,
+ // 13 = SetGraphicsFirmwareMemoryMarginEnabled), so serve it with Eden's nvdrv on the shared
+ // module. The nx2: name keeps unknown commands auto-stubbed and logged.
+ LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 1402, returning an nvdrv-backed nx2:nvsrv session");
+ auto module = system.ServiceManager().GetService<Nvidia::NVDRV>("nvdrv", true)->GetModule();
+ auto nvsrv = std::make_shared<Nvidia::NVDRV>(system, module, "nx2:nvsrv");
+ nvsrv->InitializeForProcess(ctx.GetThread().GetOwnerProcess());
+ ReturnRawSession(system, ctx, std::move(nvsrv));
+ }
+
+ static constexpr std::chrono::nanoseconds SyncptPollInterval{std::chrono::microseconds{500}};
+
+ std::shared_ptr<Nx2NativeObjects> m_objects{std::make_shared<Nx2NativeObjects>()};
+ KernelHelpers::ServiceContext m_context;
+ std::array<Kernel::KEvent*, 2> m_syncpt_events{};
+ Core::Memory::Memory* m_syncpt_memory{};
+ u64 m_syncpt_rw{};
+ u64 m_syncpt_ro{};
+ u32 m_syncpt_count{};
+ std::shared_ptr<Core::Timing::EventType> m_syncpt_poll;
+};
+
+} // namespace Service
diff --git a/src/core/hle/service/nx2_stub_interface.h b/src/core/hle/service/nx2_stub_interface.h
new file mode 100644
index 0000000000..6890298d1b
--- /dev/null
+++ b/src/core/hle/service/nx2_stub_interface.h
@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <deque>
+#include <mutex>
+#include <string>
+#include <string_view>
+
+#include <fmt/format.h>
+
+#include "core/hle/service/ipc_helpers.h"
+#include "core/hle/service/service.h"
+
+namespace Service {
+
+// Prefix of placeholder interfaces for Switch 2-only IPC. ReportUnimplementedFunction answers
+// every call on such an interface with an empty success, so boot can go on while the log shows
+// which commands are used.
+inline constexpr std::string_view Nx2StubPrefix = "nx2:";
+
+class INx2StubInterface final : public ServiceFramework<INx2StubInterface> {
+public:
+ explicit INx2StubInterface(Core::System& system_, const char* name)
+ : ServiceFramework{system_, name} {}
+
+ // Names must outlive the interface, so they are interned here.
+ static const char* InternName(std::string name) {
+ static std::mutex mutex;
+ static std::deque<std::string> names;
+ std::scoped_lock lk{mutex};
+ for (const auto& n : names) {
+ if (n == name) {
+ return n.c_str();
+ }
+ }
+ return names.emplace_back(std::move(name)).c_str();
+ }
+};
+
+// Handler for an unknown Switch 2 command that returns an interface: hands back a stub named
+// after the owning service and the command id.
+inline void ReturnNx2StubInterface(Core::System& system, HLERequestContext& ctx,
+ std::string_view owner) {
+ const char* name =
+ INx2StubInterface::InternName(fmt::format("{}{}::{}", Nx2StubPrefix, owner, ctx.GetCommand()));
+ LOG_WARNING(Service, "(STUBBED) Switch 2 {}::{} -> stub interface {}", owner, ctx.GetCommand(),
+ name);
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(ResultSuccess);
+ rb.PushIpcInterface<INx2StubInterface>(ctx, system, name);
+}
+
+} // namespace Service
diff --git a/src/core/hle/service/service.cpp b/src/core/hle/service/service.cpp
index b4fe2b3ed5..5a779af3ca 100644
--- a/src/core/hle/service/service.cpp
+++ b/src/core/hle/service/service.cpp
@@ -4,6 +4,14 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include "common/scope_exit.h"
+#include <vector>
+#include <tuple>
+#include <mutex>
+#include <map>
+#include <chrono>
+#include <algorithm>
+#include <cstdlib>
#include <fmt/ranges.h>
#include "common/assert.h"
#include "common/logging.h"
@@ -13,6 +21,7 @@
#include "core/hle/kernel/kernel.h"
#include "core/hle/service/ipc_helpers.h"
#include "core/hle/service/service.h"
+#include "core/hle/service/nx2_stub_interface.h"
#include "core/hle/service/sm/sm.h"
#include "core/reporter.h"
@@ -63,13 +72,18 @@ void ServiceFrameworkBase::ReportUnimplementedFunction(HLERequestContext& ctx,
fmt::memory_buffer buf;
fmt::format_to(std::back_inserter(buf), "function '{}({})': port='{}' cmd_buf={{[0]={:#x}", ctx.GetCommand(), function_name, service_name, cmd_buf[0]);
- for (int i = 1; i <= 8; ++i)
+ // The Switch 2 video decoder sessions are being reverse engineered: dump whole requests.
+ const int last_word = std::string_view{service_name}.starts_with("nx2:vdec") ? 47 : 8;
+ for (int i = 1; i <= last_word; ++i)
fmt::format_to(std::back_inserter(buf), ", [{}]={:#x}", i, cmd_buf[i]);
buf.push_back('}');
system.GetReporter().SaveUnimplementedFunctionReport(ctx, ctx.GetCommand(), function_name, service_name);
UNIMPLEMENTED_MSG("Unknown / unimplemented {}", fmt::to_string(buf));
- if (Settings::values.use_auto_stub) {
+ // EDEN_NX2_AUTO_STUB=1 auto-stubs every unknown command without touching the config.
+ static const bool env_auto_stub = std::getenv("EDEN_NX2_AUTO_STUB") != nullptr;
+ if (Settings::values.use_auto_stub || env_auto_stub ||
+ std::string_view{service_name}.starts_with(Nx2StubPrefix)) {
LOG_WARNING(Service, "Using auto stub fallback!");
IPC::ResponseBuilder rb{ctx, 2};
rb.Push(ResultSuccess);
@@ -83,7 +97,25 @@ void ServiceFrameworkBase::InvokeRequest(HLERequestContext& ctx) {
return ReportUnimplementedFunction(ctx, info);
LOG_TRACE(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer()));
+ // EDEN_NX2_IPC_TRACE=1 logs every dispatched command and the result word it returned.
+ static const bool ipc_trace = std::getenv("EDEN_NX2_IPC_TRACE") != nullptr;
+ if (ipc_trace) {
+ LOG_INFO(Service, "ipc {}::{} ({})", service_name, ctx.GetCommand(), info->name);
+ }
handler_invoker(this, info->handler_callback, ctx);
+ if (ipc_trace) {
+ // Response layout: [0..1] header, then 'SFCO', version, result.
+ const auto* cmd_buf = ctx.CommandBuffer();
+ for (int i = 0; i < 12; ++i) {
+ if (cmd_buf[i] == 0x4F434653) { // "SFCO"
+ if (cmd_buf[i + 2] != 0) {
+ LOG_INFO(Service, "ipc {}::{} -> result {:#x}", service_name, ctx.GetCommand(),
+ cmd_buf[i + 2]);
+ }
+ break;
+ }
+ }
+ }
}
void ServiceFrameworkBase::InvokeRequestTipc(HLERequestContext& ctx) {
@@ -93,12 +125,68 @@ void ServiceFrameworkBase::InvokeRequestTipc(HLERequestContext& ctx) {
return ReportUnimplementedFunction(ctx, info);
LOG_TRACE(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer()));
+ static const bool ipc_trace = std::getenv("EDEN_NX2_IPC_TRACE") != nullptr;
+ if (ipc_trace) {
+ const auto* w = ctx.CommandBuffer();
+ LOG_INFO(Service, "tipc {}::{} ({}) [2]={:#x} [3]={:#x} [4]={:#x}", service_name,
+ ctx.GetCommand(), info->name, w[2], w[3], w[4]);
+ }
handler_invoker(this, info->handler_callback, ctx);
}
+namespace {
+// EDEN_NX2_IPC_SUMMARY: every 5 s, log how often each service command ran and with which result
+// since the last summary (finds a game polling something that never becomes ready).
+void CountRequest(const std::string& service, u32 command, u32 type, u32 result) {
+ static const bool enabled = std::getenv("EDEN_NX2_IPC_SUMMARY") != nullptr;
+ if (!enabled) {
+ return;
+ }
+ static std::mutex mutex;
+ static std::map<std::tuple<std::string, u32, u32, u32>, u32> counts;
+ static auto last = std::chrono::steady_clock::now();
+ std::scoped_lock lock{mutex};
+ ++counts[{service, command, type, result}];
+ const auto now = std::chrono::steady_clock::now();
+ if (now - last < std::chrono::seconds{5}) {
+ return;
+ }
+ last = now;
+ std::vector<std::pair<u32, std::string>> lines;
+ for (const auto& [key, count] : counts) {
+ lines.emplace_back(count, fmt::format("{}::{}{} -> {:#x}", std::get<0>(key), std::get<1>(key),
+ std::get<2>(key) >= 16 ? " (tipc)" : "",
+ std::get<3>(key)));
+ }
+ std::sort(lines.begin(), lines.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
+ std::string text;
+ for (size_t i = 0; i < std::min<size_t>(lines.size(), 24); ++i) {
+ text += fmt::format(" | {}x {}", lines[i].first, lines[i].second);
+ }
+ LOG_INFO(Service, "ipc summary ({} kinds){}", lines.size(), text);
+ counts.clear();
+}
+} // Anonymous namespace
+
Result ServiceFrameworkBase::HandleSyncRequest(Kernel::KServerSession& session,
HLERequestContext& ctx) {
const auto guard = LockService();
+ SCOPE_EXIT {
+ const auto* cmd_buf = ctx.CommandBuffer();
+ u32 result_word = 0;
+ const u32 type = static_cast<u32>(ctx.GetCommandType());
+ if (type >= 16) {
+ result_word = cmd_buf[2];
+ } else {
+ for (int i = 0; i < 12; ++i) {
+ if (cmd_buf[i] == 0x4F434653) { // "SFCO"
+ result_word = cmd_buf[i + 2];
+ break;
+ }
+ }
+ }
+ CountRequest(service_name, ctx.GetCommand(), type, result_word);
+ };
Result result = ResultSuccess;
diff --git a/src/core/launch_params.cpp b/src/core/launch_params.cpp
index a0483226ec..25fdd45fdb 100644
--- a/src/core/launch_params.cpp
+++ b/src/core/launch_params.cpp
@@ -94,7 +94,7 @@ LaunchParams ParseLaunchParams(Core::System& system, int argc, char *argv[], wch
break;
case 'u': {
// Launch game with a specific user
- bool argument_ok = isdigit(optarg[0]);
+ bool argument_ok = isdigit(optarg[0]) != 0;
p.selected_user = atoi(optarg);
if (!argument_ok) {
// try to look it up by username, only finds the first username that matches.
diff --git a/src/core/loader/deconstructed_rom_directory.cpp b/src/core/loader/deconstructed_rom_directory.cpp
index 5e2ec7e0b9..458e0210a0 100644
--- a/src/core/loader/deconstructed_rom_directory.cpp
+++ b/src/core/loader/deconstructed_rom_directory.cpp
@@ -12,6 +12,7 @@
#include "core/file_sys/content_archive.h"
#include "core/file_sys/control_metadata.h"
#include "core/file_sys/patch_manager.h"
+#include "core/file_sys/romfs.h"
#include "core/file_sys/romfs_factory.h"
#include "core/hle/kernel/k_page_table.h"
#include "core/hle/kernel/k_process.h"
@@ -71,6 +72,30 @@ struct PatchCollection {
std::array<s32, 13> module_patcher_indices{};
};
+namespace {
+
+// RomFS for a loose ExeFS: the first .romfs file next to main, else a romfs/ directory next to
+// the exefs/ directory (the usual layout of an extracted title).
+FileSys::VirtualFile FindLooseRomFS(const FileSys::VirtualDir& exefs_dir) {
+ if (exefs_dir == nullptr) {
+ return nullptr;
+ }
+ for (const auto& f : exefs_dir->GetFiles()) {
+ if (f->GetExtension() == "romfs") {
+ return f;
+ }
+ }
+ if (const auto parent = exefs_dir->GetParentDirectory()) {
+ if (const auto romfs_dir = parent->GetSubdirectory("romfs")) {
+ LOG_INFO(Loader, "Using RomFS directory next to the ExeFS");
+ return FileSys::CreateRomFS(romfs_dir);
+ }
+ }
+ return nullptr;
+}
+
+} // namespace
+
AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(FileSys::VirtualFile file_, bool override_update_)
: AppLoader(std::move(file_))
, override_update(override_update_)
@@ -122,6 +147,8 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(FileSys
FileSys::NACP nacp(nacp_file);
name = nacp.GetApplicationName();
}
+
+ romfs = FindLooseRomFS(file_dir);
}
AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(
@@ -129,7 +156,9 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(
: AppLoader(directory->GetFile("main"))
, dir(std::move(directory))
, override_update(override_update_)
-{}
+{
+ romfs = FindLooseRomFS(dir);
+}
FileType AppLoader_DeconstructedRomDirectory::IdentifyType(const FileSys::VirtualFile& dir_file) {
if (FileSys::IsDirectoryExeFS(dir_file->GetContainingDirectory())) {
@@ -145,6 +174,9 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect
return {ResultStatus::ErrorAlreadyLoaded, {}};
}
+ // Loaded directly from a loose ExeFS (not wrapped by the NCA/NSP loaders, which register
+ // the process with the filesystem controller themselves).
+ const bool standalone = dir == nullptr;
if (dir == nullptr) {
if (file == nullptr) {
return {ResultStatus::ErrorNullFile, {}};
@@ -266,6 +298,14 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect
LOG_DEBUG(Loader, "loaded module {} @ {:#x}", module, load_addr);
}
+ if (standalone) {
+ // Without this, fsp-srv SetCurrentProcess fails with ResultTargetNotFound.
+ system.GetFileSystemController().RegisterProcess(
+ process.GetProcessId(), metadata.GetTitleID(),
+ std::make_shared<FileSys::RomFSFactory>(*this, system.GetContentProvider(),
+ system.GetFileSystemController()));
+ }
+
is_loaded = true;
return {ResultStatus::Success,
LoadParameters{metadata.GetMainThreadPriority(), metadata.GetMainThreadStackSize()}};
diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp
index 92e2e2f602..5871ac412f 100644
--- a/src/core/loader/loader.cpp
+++ b/src/core/loader/loader.cpp
@@ -393,6 +393,13 @@ std::unique_ptr<AppLoader> GetLoader(Core::System& system, FileSys::VirtualFile
FileType type = IdentifyFile(file);
const FileType filename_type = GuessFromFilename(file->GetName());
+ // A "main" inside an ExeFS directory is also a valid NSO, but loading it alone skips
+ // main.npdm, rtld and the sdk/subsdk modules. Load the whole directory instead.
+ if (type == FileType::NSO && filename_type == FileType::DeconstructedRomDirectory &&
+ IdentifyFileLoader<AppLoader_DeconstructedRomDirectory>(file)) {
+ type = FileType::DeconstructedRomDirectory;
+ }
+
// Special case: 00 is either a NCA or NAX.
if (type != filename_type && !(file->GetName() == "00" && type == FileType::NAX)) {
LOG_WARNING(Loader, "File {} has a different type ({}) than its extension.",
diff --git a/src/core/loader/nso.cpp b/src/core/loader/nso.cpp
index ea1d1c9ecd..7e3b9a54c2 100644
--- a/src/core/loader/nso.cpp
+++ b/src/core/loader/nso.cpp
@@ -247,6 +247,7 @@ std::optional<VAddr> AppLoader_NSO::LoadModule(Kernel::KProcess& process, Core::
}
// Load codeset for current process
+ LOG_INFO(Loader, "Module {} loaded at {:#x} (size {:#x})", nso_file.GetName(), load_base, image_size);
process.LoadModule(system.Kernel(), std::move(codeset), load_base);
return load_base + image_size;
}
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index eabc6dfccb..6e0ab0ee6d 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -5,6 +5,7 @@
// SPDX-FileCopyrightText: 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <atomic>
#include <algorithm>
#include <cstring>
#include <mutex>
@@ -36,6 +37,11 @@
namespace Core::Memory {
+u32& UnmappedReadFlag() {
+ thread_local u32 flag{};
+ return flag;
+}
+
static inline bool AddressSpaceContains(const Common::PageTable& table, const Common::ProcessAddress addr, const std::size_t size) {
const Common::ProcessAddress max_addr = 1ULL << table.GetAddressSpaceBits();
return addr + size >= addr && addr + size <= max_addr;
@@ -623,7 +629,13 @@ struct Memory::Impl {
inline T Read(Common::ProcessAddress vaddr) noexcept requires(std::is_trivially_copyable_v<T>) {
const u64 addr = GetInteger(vaddr);
if (auto const ptr = GetPointerImpl(addr, [addr]() {
- LOG_ERROR(HW_Memory, "Unmapped Read{} @ {:#016x}", sizeof(T) * 8, addr);
+ // A guest spinning on a bad pointer reads unmapped memory tens of thousands of times
+ // a second: keep the first ones and then a sample.
+ static std::atomic<u64> unmapped_reads{};
+ if (const u64 count = unmapped_reads.fetch_add(1); count < 1000 || count % 100000 == 0) {
+ LOG_ERROR(HW_Memory, "Unmapped Read{} @ {:#016x} (#{})", sizeof(T) * 8, addr, count);
+ }
+ ++UnmappedReadFlag();
}, [&]() {
HandleRasterizerDownload(addr, sizeof(T));
}); ptr) [[likely]] {
diff --git a/src/core/memory.h b/src/core/memory.h
index 7167efbb84..204b7514f6 100644
--- a/src/core/memory.h
+++ b/src/core/memory.h
@@ -38,6 +38,9 @@ class MemoryManager;
namespace Core::Memory {
+/// Counts unmapped reads on this host thread since the CPU callbacks last checked (debug aid).
+u32& UnmappedReadFlag();
+
/**
* Page size used by the ARM architecture. This is the smallest granularity with which memory can
* be mapped.
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc b/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc
index 8a3fab7ac1..ee9f0faa18 100644
--- a/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc
+++ b/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc
@@ -2,6 +2,24 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// DO NOT REORDER
+// ARMv8.1+ encodings used by Switch 2 code. Kept first so they win over broader patterns.
+INST(LDAPR_sz, "LDAPRB, LDAPRH, LDAPR", "zz11100010111111110000nnnnnttttt")
+INST(AtomicMemOp, "LDADD/LDCLR/LDEOR/LDSET/LDSMAX/LDSMIN/LDUMAX/LDUMIN/SWP", "zz111000AR1sssssoooo00nnnnnttttt")
+INST(CompareAndSwap, "CASB, CASH, CAS", "zz0010001L1sssssO11111nnnnnttttt")
+INST(CompareAndSwapPair, "CASP", "0z0010000L1sssssO11111nnnnnttttt")
+INST(PACAUT_reg, "PACIA/PACIB/PACDA/PACDB/AUTIA/AUTIB/AUTDA/AUTDB", "110110101100000100Zooonnnnnddddd")
+INST(XPAC_1, "XPACI, XPACD", "110110101100000101000D11111ddddd")
+INST(PACGA, "PACGA", "10011010110mmmmm001100nnnnnddddd")
+INST(RETA, "RETAA, RETAB", "110101100101111100001M1111111111")
+INST(BRA_BLRA, "BRAA/BRAB/BLRAA/BLRAB (+Z)", "1101011Z00L1111100001Mnnnnnmmmmm")
+INST(LDRA, "LDRAA, LDRAB", "11111000MS1iiiiiiiiiW1nnnnnttttt")
+INST(STG_family, "STG/STZG/ST2G/STZ2G/LDG", "11011001oo1iiiiiiiiixxnnnnnttttt")
+INST(STGP, "STGP", "0110100xx0iiiiiiiuuuuunnnnnttttt")
+INST(IRG, "IRG", "10011010110mmmmm000100nnnnnddddd")
+INST(GMI, "GMI", "10011010110mmmmm000101nnnnnddddd")
+INST(ADDG, "ADDG", "1001000110iiiiii00ttttnnnnnddddd")
+INST(SUBG, "SUBG", "1101000110iiiiii00ttttnnnnnddddd")
+
INST(FMOV_3, "FMOV (vector, immediate)", "0Q00111100000abc111111defghddddd")
INST(FMOV_2, "FMOV (vector, immediate)", "0Qo0111100000abc111101defghddddd")
INST(MOVI, "MOVI, MVNI, ORR, BIC (vector, immediate)", "0Qo0111100000abcmmmm01defghddddd")
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp
index 7840c63d79..70d5098a65 100644
--- a/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp
+++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp
@@ -82,6 +82,9 @@ bool TranslateSingleInstruction(IR::Block& block, LocationDescriptor descriptor,
#include "dynarmic/frontend/A64/translate/impl/impl.cpp"
#include "dynarmic/frontend/A64/translate/impl/impl.h"
#include "dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp"
+#include "dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp"
+#include "dynarmic/frontend/A64/translate/impl/memory_tagging.cpp"
+#include "dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp"
#include "dynarmic/frontend/A64/translate/impl/load_store_load_literal.cpp"
#include "dynarmic/frontend/A64/translate/impl/load_store_multiple_structures.cpp"
#include "dynarmic/frontend/A64/translate/impl/load_store_no_allocate_pair.cpp"
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h
index c8afef3283..d8e6f2b739 100644
--- a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h
+++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h
@@ -326,6 +326,17 @@ struct TranslatorVisitor final {
bool SWP(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
bool LDAPR(Reg Rn, Reg Rt);
+ // ARMv8.1+ forms decoded for Switch 2 code (load_store_atomic.cpp, pointer_authentication.cpp,
+ // memory_tagging.cpp)
+ bool AtomicMemOp(Imm<2> size, bool A, bool R, Reg Rs, Imm<4> op, Reg Rn, Reg Rt);
+ bool LDAPR_sz(Imm<2> size, Reg Rn, Reg Rt);
+ bool CompareAndSwap(Imm<2> size, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt);
+ bool CompareAndSwapPair(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt);
+ bool PACAUT_reg(bool Z, Imm<3> op, Reg Rn, Reg Rd);
+ bool BRA_BLRA(bool Z, bool L, bool M, Reg Rn, Reg Rm);
+ bool STG_family(Imm<2> opc, Imm<9> imm9, Imm<2> mode, Reg Rn, Reg Rt);
+ bool STGP(Imm<2> mode, Imm<7> imm7, Reg Rt2, Reg Rn, Reg Rt);
+
// Loads and stores - Load/Store register (register offset)
bool STRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, Reg Rt);
bool LDRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, Reg Rt);
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp
new file mode 100644
index 0000000000..eafc993988
--- /dev/null
+++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp
@@ -0,0 +1,190 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// ARMv8.1 LSE atomics (LDADD/LDCLR/LDEOR/LDSET/LD{S,U}{MAX,MIN}/SWP, CAS, CASP) and
+// ARMv8.3 RCpc LDAPR.
+//
+// Each read-modify-write is built from an exclusive load and an exclusive store. If the
+// store fails (another core touched the monitor in between) no register is changed and the
+// same instruction is executed again, so the operation stays atomic with multiple cores.
+
+#include "dynarmic/frontend/A64/translate/impl/impl.h"
+
+namespace Dynarmic::A64 {
+
+static Reg AtomicNextReg(Reg r) {
+ return static_cast<Reg>(static_cast<size_t>(r) + 1);
+}
+
+static IR::U64 AtomicAddress(TranslatorVisitor& v, Reg Rn) {
+ return Rn == Reg::SP ? IR::U64{v.SP(64)} : IR::U64{v.X(64, Rn)};
+}
+
+// All-ones when value == 0, otherwise zero.
+static IR::U64 AtomicMaskIfZero(TranslatorVisitor& v, IR::U64 value) {
+ const IR::U64 nonzero = v.ir.LogicalShiftRight(v.ir.Or(value, v.ir.Sub(v.ir.Imm64(0), value)), v.ir.Imm8(63));
+ return v.ir.Sub(nonzero, v.ir.Imm64(1));
+}
+
+static IR::U64 AtomicSelect(TranslatorVisitor& v, IR::U64 mask, IR::U64 if_set, IR::U64 if_clear) {
+ return v.ir.Or(v.ir.And(if_set, mask), v.ir.AndNot(if_clear, mask));
+}
+
+static IR::UAny AtomicTruncate(TranslatorVisitor& v, IR::U64 value, size_t bytes) {
+ switch (bytes) {
+ case 1:
+ return v.ir.LeastSignificantByte(value);
+ case 2:
+ return v.ir.LeastSignificantHalf(value);
+ case 4:
+ return v.ir.LeastSignificantWord(value);
+ default:
+ return value;
+ }
+}
+
+static IR::U64 AtomicLoadZext(TranslatorVisitor& v, IR::U64 address, size_t bytes, IR::AccType acctype) {
+ const IR::UAny value = v.ExclusiveMem(address, bytes, acctype);
+ return bytes == 8 ? IR::U64{value} : v.ir.ZeroExtendToLong(value);
+}
+
+static IR::U64 AtomicSignExtend(TranslatorVisitor& v, IR::U64 value, size_t bytes) {
+ return bytes == 8 ? value : v.ir.SignExtendToLong(AtomicTruncate(v, value, bytes));
+}
+
+static IR::U64 AtomicZeroExtendReg(TranslatorVisitor& v, Reg r, size_t bytes) {
+ const IR::U64 value = v.X(64, r);
+ return bytes == 8 ? value : v.ir.ZeroExtendToLong(AtomicTruncate(v, value, bytes));
+}
+
+// Writes `value` to `r` only when the exclusive store succeeded (ok_mask all-ones).
+static void AtomicCommitReg(TranslatorVisitor& v, Reg r, IR::U64 ok_mask, IR::U64 value) {
+ if (r == Reg::ZR) {
+ return;
+ }
+ v.X(64, r, AtomicSelect(v, ok_mask, value, v.X(64, r)));
+}
+
+// Ends the block: continue with the next instruction on success, retry this one on failure.
+static bool AtomicFinish(TranslatorVisitor& v, IR::U32 status) {
+ v.ir.SetCheckBit(v.ir.IsZero(status));
+ v.ir.SetTerm(IR::Term::CheckBit{IR::Term::LinkBlock{v.ir.current_location->AdvancePC(4)},
+ IR::Term::LinkBlock{*v.ir.current_location}});
+ return false;
+}
+
+static IR::AccType AtomicAccType(bool acquire, bool release) {
+ return acquire || release ? IR::AccType::ORDERED : IR::AccType::ATOMIC;
+}
+
+bool TranslatorVisitor::AtomicMemOp(Imm<2> size, bool A, bool R, Reg Rs, Imm<4> op, Reg Rn, Reg Rt) {
+ const size_t bytes = size_t{1} << size.ZeroExtend<size_t>();
+ const size_t opv = op.ZeroExtend<size_t>();
+ if (opv > 0b1000) {
+ return UnallocatedEncoding();
+ }
+
+ const IR::U64 address = AtomicAddress(*this, Rn);
+ const IR::AccType acctype = AtomicAccType(A, R);
+ const IR::U64 old = AtomicLoadZext(*this, address, bytes, acctype);
+ const IR::U64 s = AtomicZeroExtendReg(*this, Rs, bytes);
+
+ IR::U64 result;
+ switch (opv) {
+ case 0b0000: // LDADD
+ result = ir.Add(old, s);
+ break;
+ case 0b0001: // LDCLR
+ result = ir.AndNot(old, s);
+ break;
+ case 0b0010: // LDEOR
+ result = ir.Eor(old, s);
+ break;
+ case 0b0011: // LDSET
+ result = ir.Or(old, s);
+ break;
+ case 0b0100: // LDSMAX
+ result = ir.MaxSigned(AtomicSignExtend(*this, old, bytes), AtomicSignExtend(*this, s, bytes));
+ break;
+ case 0b0101: // LDSMIN
+ result = ir.MinSigned(AtomicSignExtend(*this, old, bytes), AtomicSignExtend(*this, s, bytes));
+ break;
+ case 0b0110: // LDUMAX
+ result = ir.MaxUnsigned(old, s);
+ break;
+ case 0b0111: // LDUMIN
+ result = ir.MinUnsigned(old, s);
+ break;
+ default: // SWP
+ result = s;
+ break;
+ }
+
+ const IR::U32 status = ExclusiveMem(address, bytes, acctype, AtomicTruncate(*this, result, bytes));
+ const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status));
+ AtomicCommitReg(*this, Rt, ok, old);
+ return AtomicFinish(*this, status);
+}
+
+bool TranslatorVisitor::LDAPR_sz(Imm<2> size, Reg Rn, Reg Rt) {
+ const size_t bytes = size_t{1} << size.ZeroExtend<size_t>();
+ const size_t regsize = bytes == 8 ? 64 : 32;
+ const IR::UAny data = Mem(AtomicAddress(*this, Rn), bytes, IR::AccType::ORDERED);
+ X(regsize, Rt, ZeroExtend(data, regsize));
+ return true;
+}
+
+bool TranslatorVisitor::CompareAndSwap(Imm<2> size, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) {
+ const size_t bytes = size_t{1} << size.ZeroExtend<size_t>();
+ const IR::U64 address = AtomicAddress(*this, Rn);
+ const IR::AccType acctype = AtomicAccType(L, o0);
+
+ const IR::U64 old = AtomicLoadZext(*this, address, bytes, acctype);
+ const IR::U64 expected = AtomicZeroExtendReg(*this, Rs, bytes);
+ const IR::U64 desired = AtomicZeroExtendReg(*this, Rt, bytes);
+ const IR::U64 equal = AtomicMaskIfZero(*this, ir.Eor(old, expected));
+ const IR::U64 result = AtomicSelect(*this, equal, desired, old);
+
+ const IR::U32 status = ExclusiveMem(address, bytes, acctype, AtomicTruncate(*this, result, bytes));
+ const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status));
+ AtomicCommitReg(*this, Rs, ok, old);
+ return AtomicFinish(*this, status);
+}
+
+bool TranslatorVisitor::CompareAndSwapPair(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) {
+ if ((static_cast<size_t>(Rs) & 1) != 0 || (static_cast<size_t>(Rt) & 1) != 0) {
+ return UnallocatedEncoding();
+ }
+ const Reg Rs2 = AtomicNextReg(Rs);
+ const Reg Rt2 = AtomicNextReg(Rt);
+ const IR::U64 address = AtomicAddress(*this, Rn);
+ const IR::AccType acctype = AtomicAccType(L, o0);
+
+ if (!sz) {
+ // Two 32-bit words, accessed as one 64-bit exclusive.
+ const IR::U64 old = ExclusiveMem(address, 8, acctype);
+ const IR::U64 expected = ir.Pack2x32To1x64(X(32, Rs), X(32, Rs2));
+ const IR::U64 desired = ir.Pack2x32To1x64(X(32, Rt), X(32, Rt2));
+ const IR::U64 equal = AtomicMaskIfZero(*this, ir.Eor(old, expected));
+ const IR::U32 status = ExclusiveMem(address, 8, acctype, AtomicSelect(*this, equal, desired, old));
+ const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status));
+ AtomicCommitReg(*this, Rs, ok, ir.ZeroExtendToLong(ir.LeastSignificantWord(old)));
+ AtomicCommitReg(*this, Rs2, ok, ir.LogicalShiftRight(old, ir.Imm8(32)));
+ return AtomicFinish(*this, status);
+ }
+
+ const IR::U128 old = ExclusiveMem(address, 16, acctype);
+ const IR::U64 old_lo = ir.VectorGetElement(64, old, 0);
+ const IR::U64 old_hi = ir.VectorGetElement(64, old, 1);
+ const IR::U64 diff = ir.Or(ir.Eor(old_lo, X(64, Rs)), ir.Eor(old_hi, X(64, Rs2)));
+ const IR::U64 equal = AtomicMaskIfZero(*this, diff);
+ const IR::U64 new_lo = AtomicSelect(*this, equal, X(64, Rt), old_lo);
+ const IR::U64 new_hi = AtomicSelect(*this, equal, X(64, Rt2), old_hi);
+ const IR::U32 status = ExclusiveMem(address, 16, acctype, ir.Pack2x64To1x128(new_lo, new_hi));
+ const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status));
+ AtomicCommitReg(*this, Rs, ok, old_lo);
+ AtomicCommitReg(*this, Rs2, ok, old_hi);
+ return AtomicFinish(*this, status);
+}
+
+} // namespace Dynarmic::A64
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/memory_tagging.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/memory_tagging.cpp
new file mode 100644
index 0000000000..9c6bcb7872
--- /dev/null
+++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/memory_tagging.cpp
@@ -0,0 +1,98 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// ARMv8.5 memory tagging, with tags ignored: every allocation tag is 0, tag stores only do
+// their address writeback, and the STZG forms still zero the granules they cover.
+
+#include "dynarmic/frontend/A64/translate/impl/impl.h"
+
+namespace Dynarmic::A64 {
+
+static IR::U64 TagBase(TranslatorVisitor& v, Reg Rn) {
+ return Rn == Reg::SP ? IR::U64{v.SP(64)} : IR::U64{v.X(64, Rn)};
+}
+
+static void TagSetBase(TranslatorVisitor& v, Reg Rn, IR::U64 value) {
+ if (Rn == Reg::SP) {
+ v.SP(64, value);
+ } else {
+ v.X(64, Rn, value);
+ }
+}
+
+bool TranslatorVisitor::STG_family(Imm<2> opc, Imm<9> imm9, Imm<2> mode, Reg Rn, Reg Rt) {
+ const size_t opcv = opc.ZeroExtend<size_t>();
+ const size_t modev = mode.ZeroExtend<size_t>();
+ if (modev == 0b00) {
+ // LDG (opc 01) leaves Xt as-is since tags are 0; STZGM/STGM/LDGM are EL1-only.
+ if (opcv == 0b01) {
+ return true;
+ }
+ return UnallocatedEncoding();
+ }
+
+ const s64 offset = imm9.SignExtend<s64>() << 4;
+ const IR::U64 base = TagBase(*this, Rn);
+ const IR::U64 offset_address = ir.Add(base, ir.Imm64(static_cast<u64>(offset)));
+ const IR::U64 address = modev == 0b01 ? base : offset_address; // 01 = post-index
+
+ const bool zero = (opcv & 1) != 0; // STZG, STZ2G
+ const size_t granules = (opcv & 2) ? 2 : 1; // ST2G, STZ2G
+ if (zero) {
+ const IR::U64 zero64 = ir.Imm64(0);
+ for (size_t i = 0; i < granules * 2; ++i) {
+ Mem(ir.Add(address, ir.Imm64(i * 8)), 8, IR::AccType::NORMAL, zero64);
+ }
+ }
+ (void)Rt;
+ if (modev != 0b10) {
+ TagSetBase(*this, Rn, offset_address);
+ }
+ return true;
+}
+
+bool TranslatorVisitor::STGP(Imm<2> mode, Imm<7> imm7, Reg Rt2, Reg Rn, Reg Rt) {
+ const size_t modev = mode.ZeroExtend<size_t>();
+ if (modev == 0b00) {
+ return UnallocatedEncoding();
+ }
+ const s64 offset = imm7.SignExtend<s64>() << 4;
+ const IR::U64 base = TagBase(*this, Rn);
+ const IR::U64 offset_address = ir.Add(base, ir.Imm64(static_cast<u64>(offset)));
+ const IR::U64 address = modev == 0b01 ? base : offset_address;
+ Mem(address, 8, IR::AccType::NORMAL, X(64, Rt));
+ Mem(ir.Add(address, ir.Imm64(8)), 8, IR::AccType::NORMAL, X(64, Rt2));
+ if (modev != 0b10) {
+ TagSetBase(*this, Rn, offset_address);
+ }
+ return true;
+}
+
+bool TranslatorVisitor::IRG(Reg Rm, Reg Rn, Reg Rd) {
+ (void)Rm;
+ TagSetBase(*this, Rd, TagBase(*this, Rn));
+ return true;
+}
+
+bool TranslatorVisitor::GMI(Reg Rm, Reg Rn, Reg Rd) {
+ (void)Rn;
+ // Tag of Xn is always 0, so the exclusion mask gains bit 0.
+ X(64, Rd, ir.Or(X(64, Rm), ir.Imm64(1)));
+ return true;
+}
+
+bool TranslatorVisitor::ADDG(Imm<6> offset_imm, Imm<4> tag_offset, Reg Rn, Reg Rd) {
+ (void)tag_offset;
+ const u64 offset = offset_imm.ZeroExtend<u64>() << 4;
+ TagSetBase(*this, Rd, ir.Add(TagBase(*this, Rn), ir.Imm64(offset)));
+ return true;
+}
+
+bool TranslatorVisitor::SUBG(Imm<6> offset_imm, Imm<4> tag_offset, Reg Rn, Reg Rd) {
+ (void)tag_offset;
+ const u64 offset = offset_imm.ZeroExtend<u64>() << 4;
+ TagSetBase(*this, Rd, ir.Sub(TagBase(*this, Rn), ir.Imm64(offset)));
+ return true;
+}
+
+} // namespace Dynarmic::A64
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp
new file mode 100644
index 0000000000..e36942c942
--- /dev/null
+++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp
@@ -0,0 +1,73 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// ARMv8.3 pointer authentication, implemented as the identity: PAC* never adds a code and
+// AUT*/XPAC* have nothing to strip, so signed pointers are just plain pointers. The HINT-space
+// forms (PACIASP, AUTIBSP, ...) already decode as HINT/NOP.
+
+#include "dynarmic/frontend/A64/translate/impl/impl.h"
+
+namespace Dynarmic::A64 {
+
+bool TranslatorVisitor::PACAUT_reg(bool Z, Imm<3> op, Reg Rn, Reg Rd) {
+ (void)op;
+ (void)Rd;
+ if (Z && Rn != Reg::R31) {
+ return UnallocatedEncoding();
+ }
+ return true;
+}
+
+bool TranslatorVisitor::XPAC_1(bool D, Reg Rd) {
+ (void)D;
+ (void)Rd;
+ return true;
+}
+
+bool TranslatorVisitor::PACGA(Reg Rm, Reg Rn, Reg Rd) {
+ // Any deterministic value works, since the code is only ever compared to a recomputation.
+ const IR::U64 m = Rm == Reg::SP ? IR::U64{SP(64)} : IR::U64{X(64, Rm)};
+ X(64, Rd, ir.And(ir.Eor(X(64, Rn), m), ir.Imm64(0xFFFFFFFF00000000ULL)));
+ return true;
+}
+
+bool TranslatorVisitor::RETA(bool M) {
+ (void)M;
+ ir.SetPC(X(64, Reg::R30));
+ ir.SetTerm(IR::Term::PopRSBHint{});
+ return false;
+}
+
+bool TranslatorVisitor::BRA_BLRA(bool Z, bool L, bool M, Reg Rn, Reg Rm) {
+ (void)M;
+ if (!Z && Rm != Reg::R31) {
+ return UnallocatedEncoding();
+ }
+ const auto target = X(64, Rn);
+ if (L) {
+ X(64, Reg::R30, ir.Imm64(ir.PC() + 4));
+ ir.PushRSB(ir.current_location->AdvancePC(4));
+ }
+ ir.SetPC(target);
+ ir.SetTerm(IR::Term::FastDispatchHint{});
+ return false;
+}
+
+bool TranslatorVisitor::LDRA(bool M, bool S, Imm<9> imm9, bool W, Reg Rn, Reg Rt) {
+ (void)M;
+ const s64 offset = concatenate(Imm<1>{S ? 1u : 0u}, imm9).SignExtend<s64>() << 3;
+ const IR::U64 base = Rn == Reg::SP ? IR::U64{SP(64)} : IR::U64{X(64, Rn)};
+ const IR::U64 address = ir.Add(base, ir.Imm64(static_cast<u64>(offset)));
+ const IR::U64 data = Mem(address, 8, IR::AccType::NORMAL);
+ if (W) {
+ if (Rn == Reg::SP) {
+ SP(64, address);
+ } else {
+ X(64, Rn, address);
+ }
+ }
+ X(64, Rt, data);
+ return true;
+}
+
+} // namespace Dynarmic::A64
diff --git a/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h b/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h
index 05b067bdd8..46fce86bda 100644
--- a/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h
+++ b/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h
@@ -82,7 +82,7 @@ private:
static constexpr VAddr RESERVATION_GRANULE_MASK = 0xFFFF'FFFF'FFFF'FFFFull;
static constexpr VAddr INVALID_EXCLUSIVE_ADDRESS = 0xDEAD'DEAD'DEAD'DEADull;
- static constexpr size_t MAX_NUM_CPU_CORES = 4; // Sync with src/core/hardware_properties
+ static constexpr size_t MAX_NUM_CPU_CORES = 6; // Sync with src/core/hardware_properties
boost::container::static_vector<VAddr, MAX_NUM_CPU_CORES> exclusive_addresses;
boost::container::static_vector<Vector, MAX_NUM_CPU_CORES> exclusive_values;
SpinLock lock;
diff --git a/src/shader_recompiler/CMakeLists.txt b/src/shader_recompiler/CMakeLists.txt
index cc9ce95d92..39484a1bd5 100644
--- a/src/shader_recompiler/CMakeLists.txt
+++ b/src/shader_recompiler/CMakeLists.txt
@@ -58,6 +58,9 @@ add_library(shader_recompiler STATIC
frontend/ir/type.h
frontend/ir/value.cpp
frontend/ir/value.h
+ frontend/ampere/lower_to_maxwell.h
+ frontend/ampere/program_override.h
+ frontend/ampere/program_address_map.h
frontend/maxwell/control_flow.cpp
frontend/maxwell/control_flow.h
frontend/maxwell/decode.cpp
diff --git a/src/shader_recompiler/environment.h b/src/shader_recompiler/environment.h
index 184b56ebf3..67486430c8 100644
--- a/src/shader_recompiler/environment.h
+++ b/src/shader_recompiler/environment.h
@@ -74,6 +74,8 @@ public:
[[nodiscard]] virtual u64 ReadInstruction(u32 address) = 0;
+ [[nodiscard]] virtual u32 TranslateProgramAddress(u32 address) const { return address; }
+
[[nodiscard]] virtual u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) = 0;
[[nodiscard]] virtual TextureType ReadTextureType(u32 raw_handle) = 0;
diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h
new file mode 100644
index 0000000000..365efa1118
--- /dev/null
+++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h
@@ -0,0 +1,2249 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+// Encoding reference: Mesa NAK sm70_encode.rs (MIT), Copyright 2022 Collabora, Ltd.
+#pragma once
+
+#include <cstdlib>
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <span>
+#include <optional>
+#include <string_view>
+#include <vector>
+#include "common/common_types.h"
+
+namespace Shader::Ampere {
+
+struct Instruction {
+ std::array<u32, 4> words;
+
+ u32 Bits(u32 offset, u32 count) const {
+ u64 bits = words[offset / 32];
+ if (offset / 32 + 1 < words.size()) {
+ bits |= u64{words[offset / 32 + 1]} << 32;
+ }
+ return static_cast<u32>((bits >> (offset % 32)) & ((u64{1} << count) - 1));
+ }
+};
+static_assert(sizeof(Instruction) == 16);
+
+struct LoweredProgram {
+ std::vector<u64> instructions;
+ std::vector<u32> source_instruction_offsets;
+ u32 source_size{};
+ u32 required_texture_buffer = ~0U;
+ u32 error_offset{};
+ std::string_view error;
+
+ explicit operator bool() const { return error.empty(); }
+};
+
+// Lower only operations whose semantics are representable by the existing
+// Maxwell frontend. These are new instructions, not reinterpretations of the
+// native bytes. Scheduling/control words are added by the environment adapter.
+inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
+ LoweredProgram result;
+ struct Relocation { u32 output_index; s64 target; };
+ std::vector<Relocation> relocations;
+ std::vector<u32> barrier_tokens;
+ std::vector<u32> call_targets;
+ std::vector<u32> call_links;
+ bool main_exited = false;
+ bool computed_branch = false; // a jump table: the program ends at its self-branch, not an EXIT
+ bool uses_uniform_bank = false;
+ for (const auto& instruction : code) {
+ const u32 op = instruction.Bits(0, 12);
+ uses_uniform_bank |= op == 0x3c4 || op == 0x2bd || op == 0xc0c || op == 0x986 || op == 0x9a8 || op == 0xc12 || op == 0xc17 || op == 0xe24 || op == 0xc24 || op == 0xd00 || op == 0xd09 || op == 0x9c3 || op == 0xc45 || op == 0x2a4 || op == 0x891 || op == 0x890 || op == 0x290 || op == 0x28c || op == 0x88c || op == 0x287 || op == 0x887 || op == 0x958 || op == 0xab9 || op == 0xabb || op == 0x899 || op == 0xc20 || op == 0xe21 || op == 0xe23 || op == 0xc02 ||
+ op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || op == 0x292 || op == 0x892 || op == 0x291 || op == 0x981 || op == 0xc13 || op == 0xc10 || op == 0xc23 || op == 0x526 || op == 0x2be || op == 0x8a4 || op == 0xd05 || (op == 0x984 && instruction.Bits(91, 1)) || op == 0x89c || op == 0x883 || op == 0x2a5 || op == 0xd06 || op == 0x299 || op == 0xc09 || (op == 0x81c && instruction.Bits(67, 1));
+ }
+ const auto address_of = [](u32 index) { return 0x50U + index * 8 + ((index + 1) / 3) * 8; };
+ const auto fail = [&](u32 index, std::string_view reason) {
+ result.instructions.clear();
+ result.source_instruction_offsets.clear();
+ result.error_offset = index * sizeof(Instruction);
+ result.error = reason;
+ return result;
+ };
+ const auto finish = [&](u32 index) {
+ for (const auto& relocation : relocations) {
+ if (relocation.target > index) return fail(index, "native branch or call leaves captured program");
+ const u32 target = result.source_instruction_offsets[static_cast<size_t>(relocation.target)];
+ const s64 relative = s64{address_of(target)} - address_of(relocation.output_index) - 8;
+ if (relative < -0x800000 || relative > 0x7fffff) return fail(index, "lowered branch distance");
+ result.instructions[relocation.output_index] |= (static_cast<u64>(relative) & 0xffffff) << 20;
+ }
+ result.source_size = (index + 1) * sizeof(Instruction);
+ return result;
+ };
+ if (code.empty() || code.size() > 65536) return fail(0, "invalid native program size");
+ for (u32 index = 0; index < code.size(); ++index) {
+ auto insn = code[index];
+ result.source_instruction_offsets.push_back(static_cast<u32>(result.instructions.size()));
+ u32 opcode = insn.Bits(0, 12);
+ if (insn.Bits(12, 4) == 15) continue; // @!PT: never executes (compiler padding)
+ std::optional<u32> active_link;
+ u32 active_entry = 0;
+ if (main_exited) {
+ for (size_t call = 0; call < call_targets.size(); ++call) {
+ if (call_targets[call] <= index && call_targets[call] >= active_entry) {
+ active_entry = call_targets[call];
+ active_link = call_links[call];
+ }
+ }
+ if (active_link && insn.Bits(16, 8) == *active_link)
+ return fail(index, "native callee overwrites link register");
+ }
+ // Scratch values and emulated uniform state occupy reserved high GPRs.
+ // Reject native accesses that could alias that state.
+ for (u32 bit : {16U, 24U, 32U, 64U}) {
+ if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66) continue;
+ const u32 reg = insn.Bits(bit, 8);
+ if (reg >= (uses_uniform_bank ? 216U : 246U) && reg < 255) return fail(index, "native program uses lowering scratch registers");
+ }
+ // UR0-UR21 live in R216-R237 and UR22-UR27 in R240-R245 (R239 holds the uniform
+ // predicates), so even/odd pairs stay adjacent. Sites that still check reg < 16 rely on
+ // UR0-UR15 being R216-R231.
+ const auto uniform_register = [&](u32 reg) -> u32 {
+ if (reg < 22) return 216 + reg;
+ if (reg < 28) return 218 + reg;
+ if (reg == 63) return 255;
+ return ~0U;
+ };
+ bool rewritten_uniform = false;
+ bool uniform_guard = false;
+ u32 uniform_output = 7; // ULOP3's UPn output: computed in P(6-n), then merged into R239
+ if (opcode == 0x2a4 || opcode == 0x8a4 || opcode == 0x891 || opcode == 0x890 || opcode == 0x290 ||
+ opcode == 0x292 || opcode == 0x892 || opcode == 0x291) {
+ if (insn.Bits(12, 4) != 7 && opcode != 0x292 && opcode != 0x892)
+ return fail(index, "native uniform arithmetic predication");
+ uniform_guard = insn.Bits(12, 4) != 7;
+ rewritten_uniform = true;
+ const u32 dst = uniform_register(insn.Bits(16, 8));
+ const u32 a = uniform_register(insn.Bits(24, 8));
+ const u32 c = uniform_register(insn.Bits(64, 8));
+ if (dst == ~0U || a == ~0U || c == ~0U) return fail(index, "native uniform arithmetic register");
+ insn.words[0] = (insn.words[0] & 0xffff) | (dst << 16) | (a << 24);
+ insn.words[2] = (insn.words[2] & ~0x080000ffU) | c;
+ if (opcode == 0x2a4 || opcode == 0x290 || opcode == 0x292 || opcode == 0x291) {
+ const u32 b = uniform_register(insn.Bits(32, 8));
+ if (b == ~0U) return fail(index, "native uniform arithmetic second register");
+ insn.words[1] = (insn.words[1] & ~0xffU) | b;
+ }
+ if ((opcode == 0x292 || opcode == 0x892) && insn.Bits(80, 4) != 0xe) {
+ if (insn.Bits(80, 1)) return fail(index, "native uniform logic predicate output");
+ uniform_output = insn.Bits(81, 3);
+ insn.words[2] = (insn.words[2] & ~(7U << 17)) | ((6 - uniform_output) << 17);
+ }
+ opcode = opcode == 0x2a4 ? 0x224 : opcode == 0x8a4 ? 0x824 : opcode == 0x891 ? 0x811 : opcode == 0x890 ? 0x810
+ : opcode == 0x292 ? 0x212 : opcode == 0x892 ? 0x812 : opcode == 0x291 ? 0x211 : 0x210;
+ }
+ const bool uniform_predication = opcode == 0xabb || opcode == 0xab9 || opcode == 0xc82 || opcode == 0x882;
+ const u32 native_predicate = insn.Bits(12, 4);
+ const bool guarded_uniform = (uniform_predication || uniform_guard) && native_predicate != 7;
+ // A uniform guard UPn lives in R239 bit 6-n and is swapped into P(6-n) for the op.
+ const u32 guard_slot = 6 - (native_predicate & 7);
+ u64 predicate = u64{guarded_uniform ? ((native_predicate & 8) | guard_slot) : native_predicate} << 16;
+ const u64 dest = insn.Bits(16, 8);
+ const auto base = [&](u64 opcode_bits) { return opcode_bits | predicate | dest; };
+ // NVN2 fullscreen vertex shaders read bank 24, which the game never binds (the console reads
+ // zeros from an unbound bank); Maxwell has 18, so read it from bank 17, also never bound.
+ const auto bank = [&]() -> u32 {
+ const u32 value = insn.Bits(54, 5);
+ return value == 24 ? 17U : value;
+ };
+ const auto cbuf = [&]() -> u64 {
+ return (u64{insn.Bits(38, 16) / 4} << 20) | (u64{bank()} << 34);
+ };
+ const auto valid_cbuf = [&] {
+ return !insn.Bits(91, 1) && bank() < 18 &&
+ (insn.Bits(38, 16) % 4) == 0;
+ };
+ std::string_view operand_error;
+ const auto emit = [&](u64 word) { result.instructions.push_back(word); };
+ // P(guard slot) while a uniform guard is swapped in; P6 for lowerings that use it as
+ // scratch (R2UR, MATCH, HFMA2.RELU), which save and restore it with the defaults below.
+ const u32 guard_mask = guard_slot < 7 ? 1U << guard_slot : 0U;
+ const auto save_predicate = [&](u32 mask = 1U << 6) {
+ emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20));
+ };
+ const auto restore_predicate = [&](u32 mask = 1U << 6) {
+ emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20));
+ };
+ if (guarded_uniform) {
+ save_predicate(guard_mask);
+ emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{guard_mask} << 20));
+ }
+ const u32 output_mask = uniform_output == 7 ? 0U : 1U << (6 - uniform_output);
+ if (output_mask) {
+ // Keep the guest P(6-n) in R254 and start from the current UPn (a false guard keeps it).
+ emit(0x38e8000000070000ULL | 254 | (u64{255} << 8) | (u64{output_mask} << 20));
+ emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{output_mask} << 20));
+ }
+ // Pd = LUT(a, b, c) over predicate operands (index | negate << 3): each input becomes
+ // 0 or ~0 (SEL), and LOP3 sets Pd when the result is nonzero.
+ const auto predicate_lut = [&](u32 lut, u32 a, u32 b, u32 c, u32 target, bool uniform_c = false) {
+ const u32 inputs[3]{a, b, c};
+ for (u32 i = 0; i < 3; ++i) {
+ if (i == 2 && uniform_c) {
+ // UPn from R239 bit 6-n as 0 or ~0 (UPT is ~0); a negation flips the table.
+ if ((c & 7) == 7) {
+ emit(0x0100000000000000ULL | predicate | 252 | (u64{15} << 12) | (u64{~0U} << 20));
+ } else {
+ emit(0x3800000000000000ULL | predicate | 252 | (u64{239} << 8) |
+ (u64{(6 - (c & 7)) | (1U << 8)} << 20));
+ emit(0x5c10000000000000ULL | predicate | 252 | (u64{255} << 8) |
+ (u64{252} << 20) | (u64{1} << 48));
+ }
+ if (c & 8) lut = ((lut & 0x55) << 1) | ((lut & 0xaa) >> 1);
+ continue;
+ }
+ emit(0x38a0000000000000ULL | predicate | (250 + i) | (u64{255} << 8) |
+ (u64{0x7ffff} << 20) | (u64{1} << 56) | (u64{inputs[i] ^ 8U} << 39));
+ }
+ emit(0x5be0000000000000ULL | predicate | 255 | (u64{250} << 8) | (u64{251} << 20) |
+ (u64{lut} << 28) | (u64{3} << 36) | (u64{252} << 39) | (u64{target} << 48));
+ };
+ const auto mov_imm = [&](u32 reg, u32 value) {
+ emit(0x0100000000000000ULL | predicate | reg | (u64{15} << 12) | (u64{value} << 20));
+ };
+ const auto materialize = [&](u32 kind, u32 reg, u32 scratch, bool absolute, bool negate) -> u32 {
+ if (kind == 3) {
+ const u32 mapped = uniform_register(reg);
+ if (mapped == ~0U) { operand_error = "native uniform source register"; return 255; }
+ reg = mapped;
+ } else if (kind == 1) {
+ mov_imm(scratch, insn.words[1]);
+ reg = scratch;
+ } else if (kind == 2) {
+ if (!valid_cbuf()) { operand_error = "native constant buffer address"; return 255; }
+ emit(0x4c98000000000000ULL | predicate | scratch | (u64{15} << 39) | cbuf());
+ reg = scratch;
+ }
+ if (absolute || negate) {
+ if (reg != scratch) emit(0x5c98000000000000ULL | predicate | scratch | (u64{reg} << 20) | (u64{15} << 39));
+ if (absolute) emit(0x0400000000000000ULL | predicate | scratch | (u64{scratch} << 8) | (u64{0x7fffffff} << 20));
+ if (negate) emit(0x0400000000000000ULL | predicate | scratch | (u64{scratch} << 8) | (u64{0x80000000} << 20) | (u64{2} << 53));
+ reg = scratch;
+ }
+ return reg;
+ };
+ const auto operand = [&](u32 which, bool floating = true) -> u32 {
+ const u32 form = insn.Bits(9, 3);
+ u32 reg{}, kind{}, abs{}, neg{};
+ if (which == 0) {
+ reg = insn.Bits(24, 8); abs = insn.Bits(73, 1); neg = insn.Bits(72, 1);
+ } else {
+ if (form != 1 && form != 2 && form != 3 && form != 4 && form != 5 && form != 6 && form != 7) {
+ operand_error = "native ALU source form"; return 255;
+ }
+ const bool wide = which == 1 ? (form == 4 || form == 5 || form == 6) : (form == 2 || form == 3 || form == 7);
+ if (wide) {
+ kind = form == 6 || form == 7 ? 3 : (form == 4 || form == 2) ? 1 : 2;
+ if (kind == 3) reg = insn.Bits(32, 8);
+ if (kind == 2 || kind == 3) { abs = insn.Bits(62, 1); neg = insn.Bits(63, 1); }
+ } else {
+ const bool upper = which == 1 ? (form == 2 || form == 3 || form == 7) : true;
+ reg = insn.Bits(upper ? 64 : 32, 8);
+ abs = insn.Bits(upper ? 74 : 62, 1); neg = insn.Bits(upper ? 75 : 63, 1);
+ }
+ }
+ return materialize(kind, reg, 250 + which, floating && abs, floating && neg);
+ };
+ struct HalfSource { u32 reg, swizzle; bool absolute, negate; };
+ const auto half_source = [&](u32 which) -> HalfSource {
+ if (which == 0) return {insn.Bits(24, 8), insn.Bits(74, 2),
+ insn.Bits(73, 1) != 0, insn.Bits(72, 1) != 0};
+ const u32 form = insn.Bits(9, 3);
+ const bool wide = which == 1 ? (form == 4 || form == 5) : (form == 2 || form == 3);
+ if (wide) {
+ const bool immediate = form == 2 || form == 4;
+ const u32 reg = materialize(immediate ? 1 : 2, 255, 250 + which, false, false);
+ return {reg, immediate ? 0U : insn.Bits(60, 2),
+ !immediate && insn.Bits(62, 1), !immediate && insn.Bits(63, 1)};
+ }
+ const bool upper = which == 2 || form == 2 || form == 3;
+ return {insn.Bits(upper ? 64 : 32, 8), insn.Bits(upper ? 81 : 60, 2),
+ insn.Bits(upper ? 83 : 62, 1) != 0, insn.Bits(upper ? 84 : 63, 1) != 0};
+ };
+ const auto extract_half = [&](HalfSource source, u32 lane, u32 target) {
+ const bool high = source.swizzle == 3 || (source.swizzle == 0 && lane == 1);
+ emit(0x5ca8000000000000ULL | predicate | target | (u64{2} << 8) |
+ (u64{source.swizzle == 1 ? 2U : 1U} << 10) | (u64{source.reg} << 20) |
+ (u64{high} << 41) | (u64{source.negate} << 45) | (u64{source.absolute} << 49));
+ };
+ const auto xmad = [&](u32 target, u32 a, u32 b, u32 c, bool a_high,
+ bool b_high, u32 mode, bool immediate) {
+ emit((immediate ? 0x3600000000000000ULL : 0x5b00000000000000ULL) |
+ predicate | target | (u64{a} << 8) |
+ (u64{immediate ? ((b >> (b_high ? 16 : 0)) & 0xffff) : b} << 20) |
+ (u64{!immediate && b_high} << 35) | (u64{c} << 39) |
+ (u64{mode} << 50) | (u64{a_high} << 53));
+ };
+ const auto multiply_low = [&](u32 a, u32 b, u32 c, bool immediate) {
+ xmad(250, a, b, c, false, false, 0, immediate);
+ xmad(251, a, b, 255, true, false, 0, immediate);
+ xmad(251, a, b, 251, false, true, 0, immediate);
+ emit(base(0x5c18000000000000ULL) | (u64{251} << 8) |
+ (u64{250} << 20) | (u64{16} << 39));
+ };
+ switch (opcode) {
+ case 0x304: { // F2F.F16.F32, scalar round-to-nearest conversion.
+ if (insn.words[2] != 0x00200800 || insn.Bits(24, 8) ||
+ (insn.words[1] & ~0xc00000ffU))
+ return fail(index, "native scalar float conversion mode");
+ emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(62, 1)} << 49) |
+ (u64{insn.Bits(63, 1)} << 45));
+ break;
+ }
+ case 0x23e: case 0xa3e: { // F2FP.PACK_AB / MERGE_C, round-to-nearest
+ if (insn.Bits(62, 2) || insn.Bits(73, 5) || insn.Bits(79, 2) ||
+ insn.Bits(81, 10) || insn.Bits(91, 1)) {
+ return fail(index, "native float packing modifiers");
+ }
+ const bool merge = insn.Bits(78, 1);
+ if (insn.Bits(72, 1)) return fail(index, "native float packing selector");
+ const u32 a = merge ? operand(2, false) : operand(0, false);
+ const u32 b = operand(1, false);
+ // Canonical F2F writes the converted half in its low 16 bits.
+ if (merge) {
+ emit(0x5c98000000000000ULL | predicate | 250 | (u64{a} << 20) | (u64{15} << 39));
+ } else {
+ emit(0x5ca8000000000000ULL | predicate | 250 | (u64{1} << 8) |
+ (u64{2} << 10) | (u64{a} << 20));
+ }
+ emit(0x5ca8000000000000ULL | predicate | 251 | (u64{1} << 8) |
+ (u64{2} << 10) | (u64{b} << 20));
+ emit(base(0x5c18000000000000ULL) | (u64{250} << 8) | (u64{251} << 20) | (u64{16} << 39));
+ break;
+ }
+ case 0x232: case 0x832: case 0xa32: { // HMUL2
+ if (insn.Bits(78, 2) || insn.Bits(81, 10) || insn.Bits(91, 1))
+ return fail(index, "native half multiply mode");
+ const auto a = half_source(0), b = half_source(1);
+ const u32 precision = insn.Bits(76, 1) ? 2 : insn.Bits(80, 1);
+ emit(base(0x5d08000000000000ULL) | (u64{a.reg} << 8) | (u64{b.reg} << 20) |
+ (u64{b.swizzle} << 28) | (u64{b.absolute} << 30) | (u64{a.negate != b.negate} << 31) |
+ (u64{insn.Bits(77, 1)} << 32) | (u64{precision} << 39) |
+ (u64{a.absolute} << 44) | (u64{a.swizzle} << 47));
+ break;
+ }
+ case 0x230: case 0x430: case 0x630: { // HADD2
+ if (insn.Bits(76, 1) || insn.Bits(79, 1) || insn.Bits(81, 10) || insn.Bits(91, 1))
+ return fail(index, "native half add mode");
+ const auto a = half_source(0);
+ const auto b = half_source(opcode == 0x230 ? 1 : 2);
+ if (insn.Bits(78, 1)) {
+ if (a.reg != 255 || !a.negate || a.absolute || a.swizzle != 0 ||
+ b.swizzle < 2 || insn.Bits(77, 1) || insn.Bits(80, 1))
+ return fail(index, "native float-output half add mode");
+ extract_half(b, 0, static_cast<u32>(dest));
+ } else {
+ emit(base(0x5d10000000000000ULL) | (u64{a.reg} << 8) | (u64{b.reg} << 20) |
+ (u64{b.swizzle} << 28) | (u64{b.absolute} << 30) | (u64{b.negate} << 31) |
+ (u64{insn.Bits(77, 1)} << 32) | (u64{insn.Bits(80, 1)} << 39) |
+ (u64{a.negate} << 43) | (u64{a.absolute} << 44) | (u64{a.swizzle} << 47));
+ }
+ break;
+ }
+ case 0x231: case 0x431: case 0x631: case 0x831: case 0xa31: { // HFMA2
+ const bool relu = insn.Bits(79, 1) != 0;
+ if (insn.Bits(78, 1) || insn.Bits(85, 1) || insn.Bits(91, 1) ||
+ (relu ? (insn.Bits(87, 4) != 7 ||
+ insn.Bits(77, 1)) : insn.Bits(87, 4) != 0))
+ return fail(index, "native half fused multiply-add mode");
+ if (relu && native_predicate != 7) {
+ // Guard the whole expansion before its temporary P6 updates.
+ relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1});
+ emit(0xe24000000000000fULL | (u64{native_predicate ^ 8U} << 16));
+ predicate = u64{7} << 16;
+ }
+ auto a = half_source(0);
+ auto b = half_source(1);
+ const auto c = half_source(2);
+ if (insn.Bits(86, 1)) { // H0_NH1: negate only the upper packed component.
+ if (b.swizzle != 0 || b.absolute || b.reg >= 250)
+ return fail(index, "native half mixed-sign source mode");
+ emit(0x0400000000000000ULL | predicate | 251 | (u64{b.reg} << 8) |
+ (u64{0x80000000} << 20) | (u64{2} << 53));
+ b.reg = 251;
+ }
+ if (b.absolute || c.absolute || (a.absolute && a.reg >= 248))
+ return fail(index, "native half fused multiply-add absolute source");
+ if (a.absolute) {
+ emit(0x0400000000000000ULL | predicate | 250 | (u64{a.reg} << 8) |
+ (u64{a.swizzle == 1 ? 0x7fffffffU : 0x7fff7fffU} << 20));
+ a.reg = 250;
+ a.absolute = false;
+ }
+ const u32 precision = insn.Bits(76, 1) ? 2 : insn.Bits(80, 1);
+ emit(0x5d00000000000000ULL | predicate | (relu ? 254U : dest) | (u64{a.reg} << 8) | (u64{b.reg} << 20) |
+ (u64{b.swizzle} << 28) | (u64{c.negate} << 30) | (u64{a.negate != b.negate} << 31) |
+ (u64{insn.Bits(77, 1)} << 32) | (u64{c.swizzle} << 35) |
+ (u64{precision} << 37) | (u64{c.reg} << 39) | (u64{a.swizzle} << 47));
+ if (relu) {
+ // Clamp the rounded binary16 lanes, preserving positive infinity
+ // and canonicalizing either sign of NaN without saturating to one.
+ save_predicate();
+ for (u32 lane = 0; lane < 2; ++lane) {
+ emit(0x3828000000070000ULL | 251 | (u64{254} << 8) | (u64{lane * 16} << 20));
+ emit(0x0400000000070000ULL | 251 | (u64{251} << 8) | (u64{0xffff} << 20));
+ emit(0x0400000000070000ULL | 252 | (u64{251} << 8) | (u64{0x7fff} << 20));
+ mov_imm(250, 0x7c00);
+ emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{250} << 8) |
+ (u64{252} << 20) | (u64{7} << 39) | (u64{1} << 49));
+ emit(0x3828000000070000ULL | 252 | (u64{251} << 8) | (u64{15} << 20));
+ mov_imm(250, 1);
+ emit(0x5c10000000070000ULL | 252 | (u64{252} << 8) | (u64{250} << 20) | (u64{1} << 48));
+ emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48));
+ mov_imm(250, 0x7fff);
+ emit(0x5ca0000000070000ULL | 251 | (u64{250} << 8) | (u64{251} << 20) | (u64{6} << 39));
+ if (lane == 0) {
+ emit(base(0x5c98000000000000ULL) | (u64{251} << 20) | (u64{15} << 39));
+ } else {
+ emit(0x3848000000070000ULL | 251 | (u64{251} << 8) | (u64{16} << 20));
+ emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{251} << 20) |
+ (u64{1} << 41) | (u64{7} << 48));
+ }
+ }
+ restore_predicate();
+ }
+ break;
+ }
+ case 0x240: case 0x840: { // HMNMX2, exact half promotion for min/max
+ if (insn.Bits(76, 11) || insn.Bits(91, 1))
+ return fail(index, "native half min/max mode");
+ const auto a = half_source(0);
+ const auto b = opcode == 0x840 ? HalfSource{255, 0, false, false} : half_source(1);
+ if (a.swizzle == 1 || b.swizzle == 1 || b.reg >= 250 && b.reg != 255)
+ return fail(index, "native half min/max source mode");
+ for (u32 lane = 0; lane < 2; ++lane) {
+ extract_half(a, lane, 250);
+ if (opcode == 0x840) {
+ const u32 half = (insn.words[1] >> (lane * 16)) & 0xffff;
+ mov_imm(251, half);
+ extract_half(HalfSource{251, 2, false, false}, 0, 251);
+ } else {
+ extract_half(b, lane, 251);
+ }
+ const u32 target = lane == 0 ? 252 : 250;
+ emit(0x5c60000000000000ULL | predicate | target | (u64{250} << 8) | (u64{251} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ emit(0x5ca8000000000000ULL | predicate | target | (u64{1} << 8) |
+ (u64{2} << 10) | (u64{target} << 20));
+ }
+ emit(base(0x5c18000000000000ULL) | (u64{250} << 8) | (u64{252} << 20) | (u64{16} << 39));
+ break;
+ }
+ case 0xc24: case 0xe24: { // IMAD low word with a uniform multiplier or addend.
+ // 0xe24 keeps the uniform addend in bits 32-37; bit 63 negates it ("RZ, RZ, -UR4").
+ const bool negate_c = opcode == 0xe24 && insn.Bits(63, 1) != 0;
+ if ((insn.words[2] & ~0x080002ffU) != 0x078e0000 ||
+ (insn.words[1] & ~(0xffU | (negate_c ? 0x80000000U : 0U))))
+ return fail(index, "native uniform-addend multiply mode");
+ const u32 a = operand(0, false), b = operand(1, false);
+ u32 c = operand(2, false);
+ if (negate_c) {
+ emit(0x5c10000000000000ULL | predicate | 252 | (u64{255} << 8) | (u64{c} << 20) |
+ (u64{1} << 48));
+ c = 252;
+ }
+ multiply_low(a, b, c, false);
+ break;
+ }
+ case 0x224: { // IMAD register / zero-product move
+ if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 && insn.Bits(75, 1) &&
+ (insn.words[2] & ~0xaffU) == 0x078e0000) {
+ emit(base(0x5c10000000000000ULL) | (u64{255} << 8) |
+ (u64{insn.Bits(64, 8)} << 20) | (u64{1} << 48));
+ break;
+ }
+ if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 &&
+ (insn.words[2] & ~0xffU) == 0x078e0000) {
+ emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.Bits(64, 8)} << 20));
+ break;
+ }
+ if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (insn.words[1] & ~0xffU))
+ return fail(index, "native integer register multiply mode");
+ multiply_low(insn.Bits(24, 8), insn.Bits(32, 8), insn.Bits(64, 8), false);
+ break;
+ }
+ case 0x2a5: { // UIMAD.WIDE.U32 UR(d):UR(d+1) = URa * URb + URc:UR(c+1)
+ const u32 a = uniform_register(insn.Bits(24, 8)), b = uniform_register(insn.Bits(32, 8));
+ const u32 c = insn.Bits(64, 8);
+ const u32 c_low = uniform_register(c), c_high = c == 63 ? 255U : uniform_register(c + 1);
+ const u32 d_low = uniform_register(insn.Bits(16, 8)), d_high = uniform_register(insn.Bits(16, 8) + 1);
+ if (native_predicate != 7 || (insn.words[1] & ~0xffU) || (insn.words[2] & ~0xffU) != 0x0f8e0000 ||
+ (insn.Bits(16, 8) & 1) || (c != 63 && (c & 1)) || a == ~0U || b == ~0U || c_low == ~0U ||
+ c_high == ~0U || d_low == ~0U || d_high == ~0U || d_low == 255)
+ return fail(index, "native uniform wide multiply mode");
+ // High word in R253 (base-2^16 long multiplication, as IMAD.HI), low word in R250.
+ xmad(250, a, b, 255, false, false, 0, false);
+ xmad(251, a, b, 250, true, false, 2, false);
+ xmad(250, a, b, 251, false, true, 1, false);
+ xmad(252, a, b, 251, true, true, 2, false);
+ emit(0x3828000000070000ULL | 250 | (u64{250} << 8) | (u64{16} << 20));
+ emit(0x5c10000000070000ULL | 253 | (u64{250} << 8) | (u64{252} << 20));
+ xmad(250, a, b, 255, false, false, 0, false);
+ xmad(251, a, b, 255, true, false, 0, false);
+ xmad(251, a, b, 251, false, true, 0, false);
+ emit(0x5c18000000070000ULL | 250 | (u64{251} << 8) | (u64{250} << 20) | (u64{16} << 39));
+ emit(0x5c10000000070000ULL | 250 | (u64{250} << 8) | (u64{c_low} << 20) | (u64{1} << 47));
+ emit(0x5c10000000070000ULL | d_high | (u64{253} << 8) | (u64{c_high} << 20) | (u64{1} << 43));
+ emit(0x5c10000000070000ULL | d_low | (u64{250} << 8) | (u64{255} << 20));
+ break;
+ }
+ case 0x827: { // IMAD.HI.U32, captured zero addend
+ if (insn.words[2] != 0x078e00ff) return fail(index, "native high multiply mode");
+ const u32 a = insn.Bits(24, 8), b = insn.words[1];
+ // Base-2^16 long multiplication. Each intermediate sum fits U32.
+ xmad(250, a, b, 255, false, false, 0, true);
+ xmad(251, a, b, 250, true, false, 2, true);
+ xmad(250, a, b, 251, false, true, 1, true);
+ xmad(252, a, b, 251, true, true, 2, true);
+ emit(0x3828000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{16} << 20));
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20));
+ break;
+ }
+ case 0xa24: { // IMAD, constant-buffer multiplier
+ if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || !valid_cbuf() || insn.Bits(62, 2))
+ return fail(index, "native integer constant-buffer multiply mode");
+ emit(0x4c98000000000000ULL | predicate | 252 | (u64{15} << 39) | cbuf());
+ multiply_low(insn.Bits(24, 8), 252, insn.Bits(64, 8), false);
+ break;
+ }
+ case 0x424: case 0x624: { // IMAD.MOV.U32: zero product plus immediate/cbuf
+ if (opcode == 0x624 && insn.Bits(24, 8) == 255 &&
+ (insn.words[2] & ~0x07800000U) == 0x000e06ffU) {
+ const u32 source = operand(2, false);
+ mov_imm(250, 1);
+ emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ emit(base(0x5c10000000000000ULL) | (u64{source} << 8) | (u64{250} << 20));
+ break;
+ }
+ if (insn.Bits(24, 8) != 255 || insn.Bits(64, 8) != 255 || insn.words[2] != 0x078e00ff) {
+ if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (opcode == 0x624 && insn.Bits(62, 2)))
+ return fail(index, "native multiply wide addend mode");
+ const u32 a = operand(0, false), b = operand(1, false), c = operand(2, false);
+ multiply_low(a, b, c, false);
+ break;
+ }
+ if (insn.Bits(24, 8) != 255 || insn.Bits(64, 8) != 255 ||
+ insn.words[2] != 0x078e00ff)
+ return fail(index, "native integer move mode");
+ if (opcode == 0x624) {
+ if (!valid_cbuf() || insn.Bits(62, 2))
+ return fail(index, "native integer move constant buffer");
+ emit(base(0x4c98000000000000ULL) | (u64{15} << 39) | cbuf());
+ } else {
+ emit(base(0x0100000000000000ULL) | (u64{15} << 12) |
+ (u64{insn.words[1]} << 20));
+ }
+ break;
+ }
+ case 0x824: { // Captured low-word IMAD forms; bit 75 negates the addend.
+ const bool negate_c = insn.Bits(75, 1) != 0;
+ if (insn.words[1] == 1 && (insn.words[2] & 0xfffff700U) == 0x078e0200) {
+ emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(64, 8)} << 20) | (u64{negate_c} << 48));
+ break;
+ }
+ const u32 multiplier = insn.words[1];
+ if (insn.Bits(64, 8) == 255 && !negate_c && multiplier > 1 && !(multiplier & (multiplier - 1)) &&
+ (insn.words[2] & ~0x2ffU) == 0x078e0000) {
+ // IMAD.SHL by a power of two: one shift (also the form Eden's jump-table tracking
+ // expects between the clamp and the table load).
+ u32 shift = 0;
+ while ((1U << shift) != multiplier) ++shift;
+ emit(base(0x3848000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{shift} << 20));
+ break;
+ }
+ if (insn.words[1] != 48 || insn.words[2] != 0x078e02ff) {
+ if ((insn.words[2] & ~0xaffU) != 0x078e0000)
+ return fail(index, "native integer multiply mode");
+ u32 c = insn.Bits(64, 8);
+ if (negate_c && c != 255) {
+ emit(0x5c10000000000000ULL | predicate | 252 | (u64{255} << 8) | (u64{c} << 20) |
+ (u64{1} << 48));
+ c = 252;
+ }
+ multiply_low(insn.Bits(24, 8), insn.words[1], c, true);
+ break;
+ }
+ const u64 source = insn.Bits(24, 8);
+ emit(0x5c18000000000000ULL | predicate | 250 | (source << 8) | (source << 20) | (u64{1} << 39));
+ emit(base(0x3848000000000000ULL) | (u64{250} << 8) | (u64{4} << 20));
+ break;
+ }
+ case 0x806: { // VOTE: register ballot, ordinary predicate input and output.
+ if (insn.words[1] || insn.Bits(72, 2) > 2 || (insn.words[2] & ~0x078e0300U))
+ return fail(index, "native vote mode");
+ emit(base(0x50d8000000000000ULL) | (u64{insn.Bits(87, 4)} << 39) |
+ (u64{insn.Bits(81, 3)} << 45) | (u64{insn.Bits(72, 2)} << 48));
+ break;
+ }
+ case 0x886: { // VOTEU ballot and independent uniform predicate output.
+ const u32 output = insn.Bits(81, 3), target = insn.Bits(16, 8);
+ if ((target >= 16 && target != 63) || insn.words[1] ||
+ insn.Bits(72, 2) > 2 || native_predicate != 7)
+ return fail(index, "native uniform vote mode");
+ const u32 temporary = output == 7 ? 7 : 6 - output;
+ const u32 mask = 1U << temporary;
+ if (output != 7) emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20));
+ emit(0x50d8000000070000ULL | uniform_register(target) |
+ (u64{insn.Bits(87, 4)} << 39) | (u64{temporary} << 45) | (u64{insn.Bits(72, 2)} << 48));
+ if (output != 7) {
+ emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20));
+ emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20));
+ }
+ break;
+ }
+ case 0x3c2: { // R2UR: broadcast the first lane satisfying the execution predicate.
+ const u32 target = uniform_register(static_cast<u32>(dest));
+ if (target == ~0U || target == 255 || insn.words[1] || insn.words[2] != 0x000e0000)
+ return fail(index, "native register-to-uniform mode");
+ save_predicate();
+ emit(0x50d8000000070000ULL | 250 | (u64{native_predicate} << 39) |
+ (u64{6} << 45) | (u64{1} << 48));
+ emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49));
+ emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48));
+ emit(0x5c30000000070000ULL | 250 | (u64{250} << 20));
+ emit(0xef10000000070000ULL | 251 | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
+ emit(0x5ca0000000070000ULL | target | (u64{251} << 8) |
+ (u64{target} << 20) | (u64{6} << 39));
+ restore_predicate();
+ break;
+ }
+ case 0x81c: { // PLOP3.LUT; bit 67 makes the third source a uniform predicate.
+ const u32 target = insn.Bits(81, 3);
+ const u32 lut = insn.Bits(64, 3) | (insn.Bits(72, 5) << 3);
+ if (target == 7 || insn.Bits(84, 3) != 7 ||
+ insn.Bits(16, 8) || insn.words[1])
+ return fail(index, "native predicate LUT mode");
+ if (!insn.Bits(67, 1) && (lut == 0xfe || lut == 0xa8)) {
+ emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 |
+ (u64{insn.Bits(87, 4)} << 12) | (u64{1} << 24) |
+ (u64{insn.Bits(77, 4)} << 29) | (u64{insn.Bits(68, 4)} << 39) |
+ (u64{lut == 0xfe} << 45));
+ break;
+ }
+ if (!insn.Bits(67, 1) && (lut == 0x80 ||
+ (lut == 8 && insn.Bits(68, 4) == 7 && insn.Bits(77, 4) == 7))) {
+ emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 |
+ (u64{insn.Bits(87, 4) ^ (lut == 8 ? 8U : 0U)} << 12) | (u64{insn.Bits(77, 4)} << 29) |
+ (u64{insn.Bits(68, 4)} << 39));
+ break;
+ }
+ predicate_lut(lut, insn.Bits(87, 4), insn.Bits(77, 4), insn.Bits(68, 4), target,
+ insn.Bits(67, 1) != 0);
+ break;
+ }
+ case 0x89c: { // UPLOP3.LUT: load the uniform predicates into P0-P6, combine, merge back.
+ const u32 target = insn.Bits(81, 3);
+ const u32 lut = insn.Bits(64, 3) | (insn.Bits(72, 5) << 3);
+ if (insn.Bits(84, 3) != 7 || insn.Bits(67, 1) || insn.Bits(16, 8) || insn.words[1])
+ return fail(index, "native uniform predicate LUT mode");
+ if (target == 7) break;
+ const auto slot = [](u32 operand) { return (operand & 7) == 7 ? operand : (operand & 8) | (6 - (operand & 7)); };
+ predicate = u64{slot(native_predicate)} << 16; // a UPn guard is loaded as well
+ emit(0x38e8000000070000ULL | 254 | (u64{255} << 8) | (u64{0x7f} << 20));
+ emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{0x7f} << 20));
+ predicate_lut(lut, slot(insn.Bits(87, 4)), slot(insn.Bits(77, 4)), slot(insn.Bits(68, 4)), 6 - target);
+ emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{1U << (6 - target)} << 20));
+ emit(0x38f0000000070000ULL | (u64{254} << 8) | (u64{0x7f} << 20));
+ break;
+ }
+ case 0x883: { // UP2UR UR, UPR, URZ, mask: bit k of the result is UPk (R239 bit 6-k).
+ const u32 target = uniform_register(insn.Bits(16, 8)), mask = insn.words[1];
+ if (target == ~0U || insn.Bits(24, 8) != 63 || native_predicate != 7 ||
+ insn.words[2] != 0x08000000 || mask > 0x7f)
+ return fail(index, "native uniform predicate move mode");
+ if (target == 255) break;
+ mov_imm(target, 0);
+ for (u32 k = 0; k < 7; ++k) {
+ if (!((mask >> k) & 1)) continue;
+ emit(0x3800000000070000ULL | 250 | (u64{239} << 8) | (u64{(6 - k) | (1U << 8)} << 20));
+ emit(0x36f0000000070000ULL | target | (u64{250} << 8) | (u64{k | (1U << 8)} << 20) |
+ (u64{target} << 39));
+ }
+ break;
+ }
+ case 0x882: { // UMOV immediate.
+ if (dest >= 16 || insn.words[2]) return fail(index, "native uniform immediate move mode");
+ mov_imm(static_cast<u32>(dest) + 216, insn.words[1]);
+ break;
+ }
+ case 0xc82: { // UMOV UR, UR (or URZ), including uniform execution predicates.
+ const u32 source = insn.words[1] < 64 ? uniform_register(insn.words[1]) : ~0U;
+ if (dest >= 16 || source == ~0U || insn.words[2] != 0x08000000)
+ return fail(index, "native uniform move mode");
+ emit(0x5c98000000000000ULL | predicate | (dest + 216) |
+ (u64{source} << 20) | (u64{15} << 39));
+ break;
+ }
+ case 0xab9: case 0xabb: { // ULDC U32/U64, direct or indexed.
+ const u32 uniform_dest = insn.Bits(16, 8), index_reg = insn.Bits(24, 8);
+ const u32 width = insn.Bits(73, 3);
+ const u32 target = uniform_register(uniform_dest);
+ const u32 index_target = opcode == 0xabb ? uniform_register(index_reg) : 255U;
+ if (target == ~0U || target == 255 || (width != 4 && width != 5) ||
+ (width == 5 && ((uniform_dest & 1) || uniform_register(uniform_dest + 1) != target + 1)) ||
+ bank() >= 18 || insn.Bits(38, 16) % 4 || insn.Bits(62, 2) ||
+ (insn.words[2] & ~0xe00U) != (opcode == 0xab9 ? 0U : 0x08000000U) ||
+ index_target == ~0U)
+ return fail(index, "native uniform constant load mode");
+ if (opcode == 0xab9 && width == 4) {
+ emit(0x4c98000000000000ULL | predicate | target | (u64{15} << 39) | cbuf());
+ } else {
+ emit(0xef90000000000000ULL | predicate | target |
+ (u64{index_target} << 8) | (u64{insn.Bits(38, 16)} << 20) |
+ (u64{bank()} << 36) | (u64{width} << 48));
+ }
+ break;
+ }
+ case 0x899: { // USHF.L.U32; accept a zero word outside the selected half.
+ const u32 dst = insn.Bits(16, 8), low = insn.Bits(24, 8), high = insn.Bits(64, 8);
+ if (insn.words[2] == (0x08011400U | high) && low == 63 && high < 16 &&
+ dst < 16 && native_predicate == 7 && insn.words[1] <= 32) {
+ emit(0x3828000000000000ULL | predicate | (dst + 216) |
+ (u64{high + 216} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48));
+ break;
+ }
+ // Logical right shifts of one register (bit 76 right, bit 80 high word, bits 73-74
+ // U32): USHF.R.U32 URd, URa, n, URZ and USHF.R.U32.HI URd, URZ, n, URb.
+ const bool right_low = insn.words[2] == 0x0800163f && low < 16;
+ const bool right_high = insn.words[2] == (0x08011600U | high) && low == 63 && high < 16;
+ if ((right_low || right_high) && dst < 16 && native_predicate == 7 && insn.words[1] <= 32) {
+ emit(0x3828000000000000ULL | predicate | (dst + 216) |
+ (u64{(right_high ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20));
+ break;
+ }
+ const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high < 16;
+ const bool low_result = insn.words[2] == 0x0800063f && low < 16;
+ if (dst >= 16 || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result))
+ return fail(index, "native uniform shift mode");
+ emit(0x3848000000000000ULL | predicate | (dst + 216) |
+ (u64{(high_result ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20));
+ break;
+ }
+ case 0xb82: { // LDC.64 indexed byte offset
+ const u32 width = insn.Bits(73, 3);
+ if ((width != 4 && width != 5) || insn.Bits(78, 2) || insn.Bits(91, 1) ||
+ bank() >= 18 ||
+ (width == 5 && ((dest & 1) || dest >= 254))) return fail(index, "native indexed constant load mode");
+ emit(base(0xef90000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(38, 16)} << 20) | (u64{bank()} << 36) | (u64{width} << 48));
+ break;
+ }
+ case 0xf89: { // SHFL with immediate lane and segmentation/clamp mask, any mode.
+ const u32 lane = insn.Bits(53, 5), mask = insn.Bits(40, 13), mode = insn.Bits(58, 2);
+ if (insn.words[2] != 0x000e0000 || insn.Bits(32, 8) || insn.Bits(60, 4))
+ return fail(index, "native immediate shuffle mode");
+ emit(base(0xef10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{lane} << 20) | (u64{1} << 28) | (u64{1} << 29) |
+ (u64{mode} << 30) | (u64{mask} << 34) | (u64{7} << 48));
+ break;
+ }
+ case 0x822: { // FSWZADD: per lane 0 = a+b, 1 = a-b, 2 = b-a, 3 = b (nvdisasm PP/PN/NP/ZP).
+ const u32 swizzle = insn.words[1];
+ if (swizzle > 0xff || insn.words[2] > 255)
+ return fail(index, "native swizzled add mode");
+ // Maxwell's lane codes 1 and 2 are the other way round (1 = b-a, 2 = a-b).
+ const u32 maxwell = ((swizzle & 0x55) << 1) | ((swizzle & 0xaa) >> 1);
+ emit(base(0x50f8000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(64, 8)} << 20) | (u64{maxwell} << 28));
+ break;
+ }
+ case 0xb6c: { // TXD ARRAY_2D, four channels and consecutive split destinations.
+ const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14);
+ const u32 coordinates = insn.Bits(24, 8), derivatives = insn.Bits(32, 8);
+ if ((insn.words[2] & ~0xffU) != 0x001e0f00 || insn.Bits(59, 2) ||
+ insn.Bits(61, 3) != 5 || binding >= 18 || offset > 0x1fff ||
+ (dest & 1) || dest >= 248 || insn.Bits(64, 8) != dest + 2 ||
+ coordinates >= 248 || derivatives >= 247)
+ return fail(index, "native array gradient texture mode");
+ if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
+ return fail(index, "native texture descriptor buffer conflict");
+ result.required_texture_buffer = binding;
+ emit(base(0xde00000000000000ULL) | (u64{coordinates} << 8) |
+ (u64{derivatives} << 20) | (u64{3} << 28) | (u64{15} << 31) |
+ (u64{offset} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51));
+ break;
+ }
+ case 0x989: { // SHFL, immediate lane and register clamp/segmentation mask, any mode.
+ if ((insn.words[1] & 0xf01fffffU) || (insn.words[2] & ~0x000e00ffU))
+ return fail(index, "native register-mask shuffle mode");
+ emit(base(0xef10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(53, 5)} << 20) | (u64{1} << 28) | (u64{insn.Bits(58, 2)} << 30) |
+ (u64{insn.Bits(64, 8)} << 39) | (u64{insn.Bits(81, 3)} << 48));
+ break;
+ }
+ case 0x589: { // SHFL.IDX, register lane and immediate segmentation mask.
+ if ((insn.words[2] & ~0x000e0000U) || insn.Bits(53, 11))
+ return fail(index, "native shuffle mode");
+ emit(0xef10000000000000ULL | predicate | dest |
+ (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) |
+ (u64{1} << 29) | (u64{insn.Bits(40, 13)} << 34) | (u64{insn.Bits(81, 3)} << 48));
+ break;
+ }
+ case 0x98e: { // RED.E.{ADD..XOR}.STRONG.GPU U32 (op in bits 87-89), register address.
+ if ((insn.words[2] & ~0x0380003fU) == 0x0c10e100 && insn.Bits(24, 8) == 255) {
+ // [URa + offset]: a uniform 64-bit address with a signed byte offset.
+ const u32 address = uniform_register(insn.Bits(64, 6));
+ const s32 offset = static_cast<s32>(insn.Bits(40, 24) << 8) >> 8;
+ if (address == ~0U || (address & 1) || offset < -0x80000 || offset > 0x7ffff)
+ return fail(index, "native global reduction uniform address");
+ emit(0xebf8000000000000ULL | predicate | insn.Bits(32, 8) | (u64{address} << 8) |
+ (u64{insn.Bits(87, 3)} << 23) | (u64{static_cast<u32>(offset) & 0xfffff} << 28) |
+ (u64{1} << 48));
+ break;
+ }
+ if ((insn.words[2] & ~0x03800000U) != 0x0010e100 || insn.Bits(40, 24) || (insn.Bits(24, 8) & 1))
+ return fail(index, "native global reduction mode");
+ emit(0xebf8000000000000ULL | predicate | insn.Bits(32, 8) |
+ (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(87, 3)} << 23) | (u64{1} << 48));
+ break;
+ }
+ case 0x2bd: // UFLO.U32, the highest set-bit index (.SH, bit 74: its shift amount).
+ if (dest >= 16 || insn.words[1] >= 16 || (insn.words[2] & ~0x400U) != 0x080e0000 || native_predicate != 7)
+ return fail(index, "native uniform leading bit mode");
+ emit(0x5c30000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 20) |
+ (u64{insn.Bits(74, 1)} << 41));
+ break;
+ case 0x2be: // UBREV, as a bit-reversing zero shift like BREV.
+ if (dest >= 16 || insn.words[1] >= 16 || insn.words[2] != 0x08000000 || native_predicate != 7)
+ return fail(index, "native uniform bit reverse mode");
+ emit(0x3828000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 8) |
+ (u64{1} << 39) | (u64{1} << 40));
+ break;
+ case 0x3a1: { // MATCH.ALL.U64 with only the all-equal predicate retained.
+ const u32 source = insn.Bits(24, 8), target = insn.Bits(81, 3);
+ if (dest != 255 || source >= 254 || (source & 1) || target >= 6 ||
+ native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0000U) != 0x200)
+ return fail(index, "native warp match mode");
+ save_predicate();
+ emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48));
+ emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49));
+ emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48));
+ emit(0x5c30000000070000ULL | 250 | (u64{250} << 20));
+ for (u32 word = 0; word < 2; ++word)
+ emit(0xef10000000070000ULL | (251 + word) | (u64{source + word} << 8) |
+ (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
+ emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source} << 8) |
+ (u64{251} << 20) | (u64{7} << 39) | (u64{2} << 49));
+ emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source + 1} << 8) |
+ (u64{252} << 20) | (u64{6} << 39) | (u64{2} << 49));
+ emit(0x50d8000000070000ULL | 255 | (u64{6} << 39) | (u64{target} << 45));
+ restore_predicate();
+ break;
+ }
+ case 0x3c4: { // REDUX.SUM (modulo 2^32) or .MAX (unsigned) over the actual active lanes.
+ const u32 source = insn.Bits(24, 8), target = uniform_register(static_cast<u32>(dest));
+ const bool maximum = insn.words[2] == 0x14000;
+ if (target == ~0U || target == 255 || insn.words[1] ||
+ (insn.words[2] != 0xc000 && !maximum) || native_predicate != 7)
+ return fail(index, "native warp sum mode");
+ emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48));
+ mov_imm(target, 0);
+ for (u32 lane = 0; lane < 32; ++lane) {
+ emit(0xef10000000070000ULL | 251 | (u64{source} << 8) | (u64{lane} << 20) |
+ (u64{1} << 28) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
+ emit(0x3828000000070000ULL | 252 | (u64{250} << 8) | (u64{lane} << 20));
+ emit(0x0400000000070000ULL | 252 | (u64{252} << 8) | (u64{1} << 20));
+ emit(0x5c10000000070000ULL | 252 | (u64{255} << 8) | (u64{252} << 20) | (u64{1} << 48));
+ emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48));
+ if (maximum) {
+ emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | (u64{15} << 39));
+ } else {
+ emit(0x5c10000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20));
+ }
+ }
+ break;
+ }
+ case 0xf8c: { // ATOMS.POPC.INC.32 with no returned value: one increment per active lane.
+ const u32 stride = insn.Bits(78, 2);
+ if (dest != 255 || insn.words[1] || (insn.words[2] & ~0xc000U) != 0x0d00003f)
+ return fail(index, "native shared increment mode");
+ u32 address = insn.Bits(24, 8);
+ if (stride && address != 255) {
+ emit(0x3848000000000000ULL | predicate | 251 | (u64{address} << 8) |
+ (u64{stride + 1} << 20));
+ address = 251;
+ }
+ mov_imm(250, 1);
+ emit(0xec00000000000000ULL | predicate | 255 | (u64{address} << 8) | (u64{250} << 20));
+ break;
+ }
+ case 0x38c: { // ATOMS.ADD.U32, naturally aligned shared byte offset.
+ const u32 offset = insn.Bits(40, 24);
+ if (insn.words[2] || (offset & 3)) return fail(index, "native shared atomic mode");
+ emit(0xec00000000000000ULL | predicate | dest |
+ (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) |
+ (u64{offset >> 2} << 30));
+ break;
+ }
+ case 0x3a8: { // ATOMG.E.ADD.STRONG.GPU U32 at a 64-bit register address plus an offset.
+ const u32 address = insn.Bits(24, 8);
+ const s32 offset = static_cast<s32>(insn.Bits(40, 24) << 8) >> 8;
+ if (insn.words[2] != 0x001ee100 || (address & 1) || address >= 246 ||
+ offset < -0x80000 || offset > 0x7ffff)
+ return fail(index, "native global atomic register mode");
+ emit(0xed00000000000000ULL | predicate | dest | (u64{address} << 8) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{static_cast<u32>(offset) & 0xfffff} << 28) |
+ (u64{1} << 48));
+ break;
+ }
+ case 0x9a8: { // ATOMG.E.ADD.STRONG.GPU U32 through a uniform pointer plus an offset.
+ const u32 address = uniform_register(insn.Bits(64, 6));
+ const s32 offset = static_cast<s32>(insn.Bits(40, 24) << 8) >> 8;
+ if (insn.Bits(24, 8) != 255 || offset < -0x80000 || offset > 0x7ffff ||
+ (insn.words[2] & ~0x3fU) != 0x081ee140 || address == ~0U || (address & 1))
+ return fail(index, "native global atomic mode");
+ emit(0xed00000000000000ULL | predicate | dest | (u64{address} << 8) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{static_cast<u32>(offset) & 0xfffff} << 28) |
+ (u64{1} << 48));
+ break;
+ }
+ case 0xb97: { // SULD.P.2D/2D_ARRAY.IGN, direct descriptor, channel mask at 72-75.
+ const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14);
+ const u32 coordinate = insn.Bits(24, 8);
+ const u32 channels = insn.Bits(72, 4);
+ const u32 dimension = insn.Bits(61, 3); // Maxwell's codes: 3 = 2D, 4 = 2D_ARRAY
+ if ((insn.words[2] & ~0xf00U) != 0x001e0000 || channels == 0 || (dimension != 3 && dimension != 4) ||
+ binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || dest == 255 ||
+ coordinate > (dimension == 4 ? 252U : 253U))
+ return fail(index, "native formatted surface load mode");
+ if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
+ return fail(index, "native surface descriptor buffer conflict");
+ result.required_texture_buffer = binding;
+ emit(base(0xeb00000000000000ULL) | (u64{coordinate} << 8) |
+ (u64{channels} << 20) | (u64{dimension} << 33) | (u64{offset} << 36) | (u64{1} << 51));
+ break;
+ }
+ case 0xb9b: { // SUST.P.2D/3D.IGN, direct descriptor and all four channels.
+ const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14);
+ const u32 data = insn.Bits(32, 8), coordinate = insn.Bits(24, 8);
+ const u32 dimension = insn.Bits(61, 3);
+ if (insn.words[2] != 0x00100f00 || dimension < 3 || dimension > 5 ||
+ binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || data > 252 || coordinate > (dimension >= 4 ? 252U : 253U))
+ return fail(index, "native formatted surface store mode");
+ if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
+ return fail(index, "native surface descriptor buffer conflict");
+ result.required_texture_buffer = binding;
+ emit(0xeb20000000000000ULL | predicate | data | (u64{coordinate} << 8) |
+ (u64{15} << 20) | (u64{dimension} << 33) | (u64{offset} << 36) | (u64{1} << 51));
+ break;
+ }
+ case 0x301: // BREV register.
+ if (insn.words[2] || insn.words[1] > 255) return fail(index, "native bit reverse mode");
+ emit(base(0x3828000000000000ULL) | (u64{insn.words[1]} << 8) |
+ (u64{1} << 39) | (u64{1} << 40));
+ break;
+ case 0x219: case 0xc19: { // U32 register (or uniform, 0xc19) shifts with a zero word outside the selected result.
+ const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8);
+ const u32 shift = opcode == 0xc19 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8);
+ const bool right = insn.Bits(76, 1) != 0, high_result = insn.Bits(80, 1) != 0;
+ // SHF.R.U32 of the low word with a zero high word: a plain logical right shift.
+ if (right && !high_result && high == 255 && insn.Bits(73, 2) == 3 && shift != ~0U &&
+ insn.words[1] <= 255 &&
+ (insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) == 0x00000600U) {
+ emit(base(0x5c28000000000000ULL) | (u64{low} << 8) | (u64{shift} << 20));
+ break;
+ }
+ if (insn.words[1] > 255 || insn.Bits(73, 2) != 3 || shift == ~0U ||
+ (insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) != 0x00000600U ||
+ (right ? (!high_result || low != 255) : (high_result || high != 255)))
+ return fail(index, "native register shift mode");
+ emit(base(right ? 0x5cf8000000000000ULL : 0x5bf8000000000000ULL) |
+ (u64{right ? high : 255U} << 8) | (u64{shift} << 20) |
+ (u64{right ? 255U : low} << 39) | (u64{insn.Bits(75, 1)} << 50));
+ break;
+ }
+ case 0x300: case 0xd00: { // FLO, including a uniform source.
+ if ((insn.words[2] & ~(0x600U | (opcode == 0xd00 ? 0x08000000U : 0U))) != 0x000e0000U)
+ return fail(index, "native leading bit mode");
+ const u32 source = operand(1, false);
+ emit(base(0x5c30000000000000ULL) | (u64{source} << 20) |
+ (u64{insn.Bits(63, 1)} << 40) | (u64{insn.Bits(74, 1)} << 41) |
+ (u64{insn.Bits(73, 1)} << 48));
+ break;
+ }
+ case 0x309: case 0xd09: { // POPC, including a uniform source.
+ if (insn.words[2] != (opcode == 0xd09 ? 0x08000000U : 0U))
+ return fail(index, "native population count mode");
+ const u32 source = operand(1, false);
+ emit(base(0x5c08000000000000ULL) | (u64{source} << 20) | (u64{insn.Bits(63, 1)} << 40));
+ break;
+ }
+ case 0x918:
+ if (insn.words[1] || insn.words[2]) return fail(index, "native NOP mode");
+ emit(0x50b0000000000f00ULL | predicate);
+ break;
+ case 0x992: { // MEMBAR.SC: preserve workgroup/device visibility.
+ const u32 scope = insn.Bits(76, 3);
+ if (insn.words[1] || insn.words[2] != (scope << 12) ||
+ (scope != 0 && scope != 2 && scope != 3 && scope != 5))
+ return fail(index, "native memory barrier scope");
+ const u32 legacy_scope = scope == 0 ? 0 : scope == 2 ? 1 : scope == 3 ? 2 : 3;
+ emit(0xef98000000000000ULL | predicate | (u64{legacy_scope} << 8));
+ break;
+ }
+ case 0x9ab: // ERRBAR: wait for prior memory operations on the host backend.
+ if (insn.words[1] || insn.words[2]) return fail(index, "native error barrier mode");
+ emit(0xef98000000000000ULL | predicate | (u64{1} << 8));
+ break;
+ case 0xb1d: // BAR.SYNC 0, full workgroup.
+ if (insn.words[1] || insn.words[2] || native_predicate != 7)
+ return fail(index, "native workgroup barrier mode");
+ emit(0xf0a8000000000000ULL | predicate | (u64{1} << 43) |
+ (u64{1} << 44) | (u64{1} << 39) | (u64{7} << 39));
+ break;
+ case 0x388: case 0x984: { // Shared memory load/store with scaled register offsets.
+ const u32 width = insn.Bits(73, 3), stride = insn.Bits(78, 2);
+ // LDS [Ra.Xn + URb + offset]: bit 91 adds a uniform register (bits 32-39).
+ const bool uniform_base = opcode == 0x984 && insn.Bits(91, 1);
+ const u32 base_register = uniform_base ? uniform_register(insn.Bits(32, 8)) : 255U;
+ if (width > 6 || (insn.words[2] & ~(uniform_base ? 0x08000000U : 0U)) != ((width << 9) | (stride << 14)) ||
+ (opcode == 0x984 && !uniform_base && insn.Bits(32, 8)) || base_register == ~0U)
+ return fail(index, "native shared memory addressing mode");
+ u32 address = insn.Bits(24, 8);
+ if (stride && address != 255) {
+ emit(0x3848000000000000ULL | predicate | 250 |
+ (u64{address} << 8) | (u64{stride + 1} << 20));
+ address = 250;
+ }
+ if (base_register != 255) {
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{address} << 8) | (u64{base_register} << 20));
+ address = 250;
+ }
+ const u32 data = opcode == 0x388 ? insn.Bits(32, 8) : static_cast<u32>(dest);
+ emit((opcode == 0x388 ? 0xef58000000000000ULL : 0xef48000000000000ULL) |
+ predicate | data | (u64{address} << 8) | (u64{insn.Bits(40, 24)} << 20) |
+ (u64{width} << 48));
+ break;
+ }
+ case 0x23a: { // MOVM.16.MT88: transpose an 8x8 16-bit matrix held one row pair per lane.
+ // Lane t holds row t/4, columns 2(t%4)..+1; the result is the transposed matrix in the
+ // same layout, so each half comes from lane 8(t%4) + t/8 (+4) at half (t/4)%2.
+ const u32 source = insn.Bits(24, 8);
+ if (insn.words[1] || insn.words[2] || source >= 246 || dest >= 246)
+ return fail(index, "native matrix move mode");
+ emit(0xf0c8000000000000ULL | predicate | 246); // laneid
+ emit(0x0400000000000000ULL | predicate | 249 | (u64{246} << 8) | (u64{3} << 20));
+ emit(0x3848000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{3} << 20)); // 8(t%4)
+ emit(0x3828000000000000ULL | predicate | 247 | (u64{246} << 8) | (u64{3} << 20)); // t/8
+ emit(0x5c10000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{247} << 20));
+ emit(0xef10000000000000ULL | predicate | 247 | (u64{source} << 8) | (u64{249} << 20) |
+ (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
+ emit(0x1c00000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{4} << 20));
+ emit(0xef10000000000000ULL | predicate | 248 | (u64{source} << 8) | (u64{249} << 20) |
+ (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
+ emit(0x3828000000000000ULL | predicate | 249 | (u64{246} << 8) | (u64{2} << 20));
+ emit(0x0400000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{1} << 20));
+ emit(0x3848000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{4} << 20)); // 0 or 16
+ emit(0x5c28000000000000ULL | predicate | 247 | (u64{247} << 8) | (u64{249} << 20));
+ emit(0x0400000000000000ULL | predicate | 247 | (u64{247} << 8) | (u64{0xffff} << 20));
+ emit(0x5c28000000000000ULL | predicate | 248 | (u64{248} << 8) | (u64{249} << 20));
+ emit(0x5c18000000000000ULL | predicate | dest | (u64{248} << 8) | (u64{247} << 20) | (u64{16} << 39));
+ break;
+ }
+ case 0x23c: { // HMMA.16816.F16: D(16x8) = A(16x16) * B(16x8) + C, f16 fragments.
+ // Lane t (group g = t/4, i = t%4): A regs hold rows g / g+8 at k 2i..2i+1 (a0, a1) and
+ // 8+2i.. (a2, a3); B regs hold k 2i.. (b0) / 8+2i.. (b1) of column g; D holds rows
+ // g / g+8 at columns 2i, 2i+1. Rebuilt from shuffles with HFMA2, accumulating in R252/R253.
+ // The expansion is correct but large: title 4's 372-HMMA network takes the host driver
+ // minutes to compile, so it stays opt-in (the QMD still completes when rejected).
+ // EDEN_NX2_TENSOR=stub keeps the rest of the program but makes every HMMA D = C.
+ static const char* const mode = std::getenv("EDEN_NX2_TENSOR");
+ if (!mode) return fail(index, "native matrix multiply (set EDEN_NX2_TENSOR to emulate)");
+ const u32 a = insn.Bits(24, 8), b = insn.Bits(32, 8), c = insn.Bits(64, 8);
+ if ((insn.words[1] & ~0xffU) || (insn.words[2] & ~0xffU) != 0x800 || dest + 2 > 246 ||
+ a + 4 > 246 || b + 2 > 246 || (c != 255 && c + 2 > 246))
+ return fail(index, "native matrix multiply mode");
+ if (std::string_view{mode} == "stub") {
+ for (u32 row = 0; row < 2; ++row) {
+ if (c == 255) mov_imm(static_cast<u32>(dest) + row, 0);
+ else if (c != dest) emit(0x5c98000000000000ULL | predicate | (dest + row) | (u64{15} << 39) | (u64{c + row} << 20));
+ }
+ break;
+ }
+ const auto shuffle = [&](u32 target, u32 source_reg) {
+ emit(0xef10000000000000ULL | predicate | target | (u64{source_reg} << 8) | (u64{246} << 20) |
+ (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
+ };
+ const auto lane_a = [&](u32 i) { // 4g + i
+ emit(0xf0c8000000000000ULL | predicate | 246);
+ emit(0x0400000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{0x1c} << 20));
+ if (i) emit(0x1c00000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{i} << 20));
+ };
+ const auto lane_b = [&](u32 add) { // 8(t%4) + add: column 2(t%4) (+1 when add >= 4)
+ emit(0xf0c8000000000000ULL | predicate | 246);
+ emit(0x0400000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{3} << 20));
+ emit(0x3848000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{3} << 20));
+ if (add) emit(0x1c00000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{add} << 20));
+ };
+ const auto hfma2 = [&](u32 accumulator, u32 swizzle_a, u32 pair) {
+ emit(0x5d00000000000000ULL | predicate | accumulator | (u64{251} << 8) | (u64{pair} << 20) |
+ (u64{accumulator} << 39) | (u64{swizzle_a} << 47));
+ };
+ for (u32 row = 0; row < 2; ++row) {
+ if (c == 255) mov_imm(252 + row, 0);
+ else emit(0x5c98000000000000ULL | predicate | (252 + row) | (u64{15} << 39) | (u64{c + row} << 20));
+ }
+ for (u32 i = 0; i < 4; ++i) {
+ for (u32 k_block = 0; k_block < 2; ++k_block) {
+ lane_b(i);
+ shuffle(247, b + k_block);
+ lane_b(4 + i);
+ shuffle(248, b + k_block);
+ // P0 = (B[2i][2t], B[2i][2t+1]), P1 = (B[2i+1][2t], B[2i+1][2t+1])
+ emit(0x0400000000000000ULL | predicate | 249 | (u64{247} << 8) | (u64{0xffff} << 20));
+ emit(0x5c18000000000000ULL | predicate | 249 | (u64{248} << 8) | (u64{249} << 20) | (u64{16} << 39));
+ emit(0x3828000000000000ULL | predicate | 250 | (u64{247} << 8) | (u64{16} << 20));
+ emit(0x0400000000000000ULL | predicate | 248 | (u64{248} << 8) | (u64{0xffff0000} << 20));
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{248} << 20));
+ for (u32 row = 0; row < 2; ++row) {
+ lane_a(i);
+ shuffle(251, a + k_block * 2 + row);
+ hfma2(252 + row, 2, 249); // A.H0 broadcast * P0
+ hfma2(252 + row, 3, 250); // A.H1 broadcast * P1
+ }
+ }
+ }
+ for (u32 row = 0; row < 2; ++row)
+ emit(0x5c98000000000000ULL | predicate | (dest + row) | (u64{15} << 39) | (u64{252 + row} << 20));
+ break;
+ }
+ case 0x83b: { // LDSM.16.M88.{1,2,4} (ldmatrix): 8x8 16-bit matrices from shared memory.
+ // Lanes 8m..8m+7 give the row addresses of matrix m; lane t receives the 32 bits at
+ // row t/4, byte (t%4)*4 of each matrix in Rd+m. Emulated with SHFL.IDX + LDS.32.
+ const u32 count = 1U << insn.Bits(72, 2);
+ const bool transpose = insn.Bits(78, 1) != 0;
+ if (insn.Bits(72, 2) == 3 || (insn.words[2] & ~0x4300U) || insn.Bits(32, 8) ||
+ dest + count > 246)
+ return fail(index, "native matrix load mode");
+ u32 address = insn.Bits(24, 8);
+ const u64 offset = insn.Bits(40, 24);
+ if (address != 255 && address >= dest && address < dest + count) {
+ emit(0x5c10000000000000ULL | predicate | 253 | (u64{address} << 8) | (u64{255} << 20));
+ address = 253;
+ }
+ if (transpose) {
+ // .MT88: lane t receives stored rows 2(t%4) and 2(t%4)+1 at column t/4 (16 bits each).
+ emit(0xf0c8000000000000ULL | predicate | 246); // laneid
+ emit(0x0400000000000000ULL | predicate | 247 | (u64{246} << 8) | (u64{3} << 20));
+ emit(0x3848000000000000ULL | predicate | 247 | (u64{247} << 8) | (u64{1} << 20)); // row 2(t%4)
+ emit(0x3828000000000000ULL | predicate | 248 | (u64{246} << 8) | (u64{2} << 20));
+ emit(0x3848000000000000ULL | predicate | 248 | (u64{248} << 8) | (u64{1} << 20)); // byte 2(t/4)
+ for (u32 matrix = 0; matrix < count; ++matrix) {
+ for (u32 half = 0; half < 2; ++half) {
+ emit(0x1c00000000000000ULL | predicate | 249 | (u64{247} << 8) |
+ (u64{matrix * 8 + half} << 20));
+ emit(0xef10000000000000ULL | predicate | 250 | (u64{address} << 8) | (u64{249} << 20) |
+ (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{248} << 20));
+ emit(0xef48000000000000ULL | predicate | (251 + half) | (u64{250} << 8) |
+ (offset << 20) | (u64{2} << 48)); // LDS.U16
+ }
+ emit(0x5c18000000000000ULL | predicate | (dest + matrix) | (u64{252} << 8) |
+ (u64{251} << 20) | (u64{16} << 39));
+ }
+ break;
+ }
+ emit(0xf0c8000000000000ULL | predicate | 250); // S2R laneid
+ emit(0x3828000000000000ULL | predicate | 251 | (u64{250} << 8) | (u64{2} << 20)); // row
+ emit(0x0400000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{3} << 20));
+ emit(0x3848000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{2} << 20)); // column bytes
+ for (u32 matrix = 0; matrix < count; ++matrix) {
+ u32 lane = 251;
+ if (matrix) {
+ emit(0x1c00000000000000ULL | predicate | 252 | (u64{251} << 8) | (u64{matrix * 8} << 20));
+ lane = 252;
+ }
+ emit(0xef10000000000000ULL | predicate | 253 | (u64{address} << 8) | (u64{lane} << 20) |
+ (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
+ emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{250} << 20));
+ emit(0xef48000000000000ULL | predicate | (dest + matrix) | (u64{253} << 8) |
+ (offset << 20) | (u64{4} << 48));
+ }
+ break;
+ }
+ case 0x919: case 0x9c3: { // S2R / S2UR thread and workgroup identities.
+ const u32 special = insn.Bits(72, 8);
+ if (insn.words[1] || insn.words[2] != (special << 8) ||
+ (special != 0 && special != 33 && special != 34 && special != 35 &&
+ special != 37 && special != 38 && special != 39 &&
+ special != 56 && special != 57 && special != 58 && special != 59 && special != 60))
+ return fail(index, "native special register");
+ const u32 target = opcode == 0x9c3 ? uniform_register(static_cast<u32>(dest)) : static_cast<u32>(dest);
+ if (target == ~0U) return fail(index, "native uniform special destination");
+ emit(0xf0c8000000000000ULL | predicate | target | (u64{special} << 20));
+ break;
+ }
+ case 0x28c: case 0x88c: { // UISETP, signed (bit 73) or unsigned, with UPT accumulation.
+ const u32 output = insn.Bits(81, 3);
+ const u32 a = uniform_register(insn.Bits(24, 8)), b = opcode == 0x88c ? 252U : uniform_register(insn.Bits(32, 8));
+ if (output > 6 || insn.Bits(84, 3) != 7 || insn.Bits(87, 4) != 7 ||
+ (insn.Bits(72, 4) & ~2U) || insn.Bits(12, 4) != 7 || a == ~0U || b == ~0U)
+ return fail(index, "native uniform comparison mode");
+ const u32 temporary = 6 - output, mask = 1U << temporary;
+ if (opcode == 0x88c) mov_imm(252, insn.words[1]);
+ emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20));
+ emit(0x5b60000000070000ULL | 7 | (u64{temporary} << 3) |
+ (u64{a} << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{insn.Bits(73, 1)} << 48) |
+ (u64{insn.Bits(76, 3)} << 49));
+ emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20));
+ emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20));
+ break;
+ }
+ case 0x287: case 0x887: { // USEL, UP0/UP1; preserve ordinary predicates.
+ const u32 condition = insn.Bits(87, 3);
+ const u32 target = uniform_register(static_cast<u32>(dest));
+ const u32 a = uniform_register(insn.Bits(24, 8)), b = opcode == 0x887 ? 252U : uniform_register(insn.Bits(32, 8));
+ if (condition > 6 || target == ~0U || a == ~0U || b == ~0U ||
+ insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x07800000U) != 0x08000000U)
+ return fail(index, "native uniform select mode");
+ if (opcode == 0x887 && index > 0 && condition == 0 && insn.words[1] <= 0x7ffff &&
+ code[index - 1].Bits(0, 12) == 0x88c &&
+ code[index - 1].Bits(24, 8) == insn.Bits(24, 8) &&
+ code[index - 1].words[1] == insn.words[1] && code[index - 1].words[2] == 0x0bf01070 &&
+ insn.words[2] == 0x08000000) {
+ // Preserve the unsigned clamp idiom for indirect-table analysis.
+ emit(0x3820000000070000ULL | target | (u64{a} << 8) |
+ (u64{insn.words[1]} << 20) | (u64{7} << 39));
+ break;
+ }
+ if (opcode == 0x887) mov_imm(252, insn.words[1]);
+ const u32 temporary = 6 - condition, mask = 1U << temporary;
+ emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20));
+ emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{mask} << 20));
+ emit(0x5ca0000000070000ULL | target | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{temporary} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20));
+ break;
+ }
+ case 0x381: case 0x981: { // LDG.E of 8 to 128 bits, register or uniform 64-bit address.
+ // Sizes as Maxwell's: U8, S8, U16, S16, 32, 64, 128.
+ const u32 size = insn.Bits(73, 3);
+ u32 address = insn.Bits(24, 8);
+ if (size > 6 || (opcode == 0x381 && insn.Bits(32, 8)) ||
+ (size == 5 && (dest & 1)) || (size == 6 && (dest & 3)))
+ return fail(index, "native global load mode");
+ if (opcode == 0x381) {
+ // Bit 77 (strong/scoped ordering) needs nothing extra from a Maxwell LDG.
+ if ((insn.words[2] & ~0x2e00U) != 0x001e8100 || (address & 1) || address >= 246)
+ return fail(index, "native global load mode");
+ } else {
+ // The uniform base sits where a register source would (bits 32-37).
+ const u32 uniform_address =
+ insn.Bits(38, 2) ? ~0U : uniform_register(insn.Bits(32, 6));
+ if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U | 0x2000U)) != 0x081e8100 ||
+ uniform_address == ~0U || (uniform_address & 1))
+ return fail(index, "native global load uniform addressing mode");
+ if (address == 255) {
+ address = uniform_address;
+ } else {
+ if (insn.Bits(90, 1) && ((address & 1) || address >= 246))
+ return fail(index, "native global load register address pair");
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{uniform_address} << 8) |
+ (u64{address} << 20) | (u64{1} << 47));
+ emit(0x5c10000000000000ULL | predicate | 251 | (u64{uniform_address + 1} << 8) |
+ (u64{insn.Bits(90, 1) ? address + 1 : 255} << 20) | (u64{1} << 43));
+ address = 250;
+ }
+ }
+ emit(base(0xeed0000000000000ULL) | (u64{address} << 8) |
+ (u64{insn.Bits(40, 24)} << 20) | (u64{1} << 45) | (u64{size} << 48));
+ break;
+ }
+ case 0x386: case 0x986: { // Global store of 32/64/128 bits, 64-bit address.
+ u32 data = insn.Bits(32, 8);
+ const u32 size = insn.Bits(73, 3);
+ if (size > 6 ||
+ (data != 255 && ((size == 5 && (data & 1)) || (size == 6 && (data & 3)) ||
+ data + (size < 4 ? 1U : 1U << (size - 4)) > 246)))
+ return fail(index, "native global store size");
+ if (data == 255 && size > 4) {
+ // Zeros wider than one register: store from zeroed scratch registers (aligned to the
+ // access: R248-R251 for 128 bits, R250-R251 for 64).
+ data = size == 6 ? 248 : 250;
+ for (u32 i = 0; i < (1U << (size - 4)); ++i) mov_imm(data + i, 0);
+ }
+ u32 address = insn.Bits(24, 8);
+ bool extended = insn.Bits(72, 1) != 0;
+ if (opcode == 0x986) {
+ const u32 uniform_address = uniform_register(insn.Bits(64, 6));
+ if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U)) != 0x0810a100 ||
+ uniform_address == ~0U || (uniform_address & 1) || !extended)
+ return fail(index, "native global store uniform addressing mode");
+ if (address == 255) address = uniform_address;
+ else {
+ if (insn.Bits(90, 1) && ((address & 1) || address >= 254))
+ return fail(index, "native global store register address pair");
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{uniform_address} << 8) |
+ (u64{address} << 20) | (u64{1} << 47));
+ emit(0x5c10000000000000ULL | predicate | 251 | (u64{uniform_address + 1} << 8) |
+ (u64{insn.Bits(90, 1) ? address + 1 : 255} << 20) | (u64{1} << 43));
+ address = 250;
+ }
+ } else if ((insn.words[2] & ~0xe00U) != 0x0010a100) {
+ return fail(index, "native global store register addressing mode");
+ }
+ emit(0xeed8000000000000ULL | predicate | data |
+ (u64{address} << 8) | (u64{insn.Bits(40, 24)} << 20) |
+ (u64{extended} << 45) | (u64{size} << 48));
+ break;
+ }
+ case 0x95b: // Fragment discard; execution predicate is preserved.
+ if (insn.words[1] || insn.words[2] != 0x03800000)
+ return fail(index, "native fragment discard mode");
+ emit(0xe33000000000000fULL | predicate);
+ break;
+ case 0x305: case 0xb05: case 0xd05: { // F2I NTZ, register/constant-buffer/uniform source
+ const u32 mode = insn.words[2] & ~(opcode == 0xd05 ? 0x08000000U : 0U);
+ const bool half = opcode == 0x305 && mode == 0x0010f000;
+ const bool small = mode == 0x00212800;
+ const bool floor = (mode & ~0x10100U) == 0x00207000;
+ const bool nearest = (mode & ~0x10100U) == 0x00203000;
+ if ((!half && !small && !floor && !nearest && (mode & ~0x100U) != 0x0021f000) ||
+ (opcode != 0xb05 && (insn.words[1] & 0xffffff00U)) ||
+ (opcode == 0xd05 && uniform_register(insn.Bits(32, 8)) == ~0U)) {
+ return fail(index, "native float-to-integer conversion mode");
+ }
+ const bool is_signed = insn.Bits(72, 1);
+ u32 source = opcode == 0xb05 ? materialize(2, 0, 252, false, false)
+ : opcode == 0xd05 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8);
+ if (half) {
+ extract_half({source, 2, false, false}, 0, 252);
+ source = 252;
+ }
+ if (!is_signed) {
+ // Clear NaNs without consuming a guest predicate. Clamp the
+ // conversion below 2^32, retaining an overflow mask for saturation.
+ emit(0x5800000000000000ULL | predicate | 250 | (u64{source} << 8) |
+ (u64{source} << 20) | (u64{7} << 39) | (u64{7} << 48));
+ emit(0x5c40000000000000ULL | predicate | 250 | (u64{source} << 8) |
+ (u64{250} << 20) | (u64{7} << 48));
+ if (!small) {
+ mov_imm(252, 0x4f800000);
+ emit(0x5800000000000000ULL | predicate | 251 | (u64{250} << 8) |
+ (u64{252} << 20) | (u64{7} << 39) | (u64{6} << 48));
+ mov_imm(252, 0x4f7fffff);
+ emit(0x5c60000000000000ULL | predicate | 250 | (u64{250} << 8) |
+ (u64{252} << 20) | (u64{7} << 39));
+ }
+ source = 250;
+ }
+ emit(base(0x5cb0000000000000ULL) | (u64{small ? 1U : 2U} << 8) | (u64{2} << 10) |
+ (u64{is_signed} << 12) | (u64{source} << 20) |
+ (u64{small || nearest ? 0U : floor ? 1U : 3U} << 39) | (u64{insn.Bits(80, 1)} << 44));
+ if (!is_signed && !small) {
+ emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{251} << 20) |
+ (u64{1} << 41) | (u64{7} << 48));
+ }
+ break;
+ }
+ case 0xb6f: { // TXQ texture dimensions, two packed scalar outputs.
+ if (insn.Bits(59, 5) || insn.Bits(64, 8) != 255 || insn.Bits(72, 4) == 0 || insn.Bits(72, 4) > 3 ||
+ insn.Bits(76, 14) || insn.Bits(91, 1) || insn.Bits(54, 5) != 2 ||
+ insn.Bits(40, 14) > 0x1fff || (insn.Bits(72, 4) == 3 && (dest & 1)) || dest >= 254)
+ return fail(index, "native texture dimension query mode");
+ result.required_texture_buffer = 2;
+ emit(0xdf48000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{1} << 22) | (u64{insn.Bits(72, 4)} << 31) | (u64{insn.Bits(40, 14)} << 36) |
+ (u64{insn.Bits(90, 1)} << 49));
+ break;
+ }
+ case 0xb66: { // TLD.SCR.LZ, direct texel fetch
+ const u32 mask = insn.Bits(72, 4);
+ const u32 dimension = insn.Bits(61, 3);
+ const bool array = dimension == 5;
+ const bool volume = dimension == 2;
+ u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8);
+ const u32 second_dest = insn.Bits(64, 8);
+ const bool split = second_dest != 255;
+ constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12};
+ constexpr std::array<u32, 5> rgba_masks{7, 11, 13, 14, 15};
+ u32 swizzle = ~0U;
+ if (split) {
+ for (u32 i = 0; i < rgba_masks.size(); ++i) if (rgba_masks[i] == mask) swizzle = i;
+ } else {
+ for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i;
+ }
+ if (!insn.Bits(60, 1) || (dimension != 0 && dimension != 1 && !array && !volume) ||
+ swizzle == ~0U || insn.Bits(76, 5) ||
+ insn.Bits(81, 3) != 7 || insn.Bits(87, 3) != 1 ||
+ insn.Bits(59, 1) ||
+ insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 ||
+ dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) ||
+ (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) ||
+ (array && ((source_a & 1) || source_a >= 249)) ||
+ (volume && ((source_a & 1) || source_a >= 254))) {
+ return fail(index, "native texel fetch mode or descriptor binding");
+ }
+ const u32 binding = insn.Bits(54, 5);
+ if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
+ return fail(index, "native texture descriptor buffer conflict");
+ result.required_texture_buffer = binding;
+ if (array) {
+ emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a + 1} << 20));
+ emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{source_b} << 20));
+ source_b = 250;
+ }
+ emit(0xda00000000000000ULL | predicate | dest | (u64{source_a} << 8) |
+ (u64{source_b} << 20) | (u64{insn.Bits(64, 8)} << 28) |
+ (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) |
+ (u64{swizzle} << 50) | (u64{array ? 8U : volume ? 7U : dimension == 0 ? 0U : 2U} << 53));
+ break;
+ }
+ case 0x91a: // DEPBAR.LE SB5, 1: scoreboard wait, represented by IR data dependencies.
+ if (insn.words[0] != 0x791a || (insn.words[1] & ~0xfc0U) != 0xd000 || insn.words[2]) {
+ return fail(index, "native dependency barrier mode");
+ }
+ break;
+ case 0xb69: { // TMML (LOD query), 2D. Results are 8.8 fixed point, as on Maxwell.
+ const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8);
+ const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8);
+ const u32 binding = insn.Bits(54, 5);
+ if (insn.Bits(61, 3) != 1 || insn.Bits(76, 4) || insn.Bits(80, 10) || insn.Bits(59, 1) ||
+ insn.Bits(40, 14) > 0x1fff || binding >= 18 || mask == 0 || source_a >= 246 ||
+ (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding))
+ return fail(index, "native texture LOD query mode");
+ result.required_texture_buffer = binding;
+ // Coordinates: (Ra, Ra+1) when Rb is RZ, else (Ra, Rb).
+ emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a} << 20));
+ emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) |
+ (u64{source_b == 255 ? source_a + 1 : source_b} << 20));
+ emit(0xdf58000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) |
+ (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) |
+ (u64{insn.Bits(90, 1)} << 49));
+ const u32 components = static_cast<u32>(std::popcount(mask));
+ const bool split = second_dest != 255;
+ for (u32 i = 0; i < components; ++i) {
+ u32 target = 255;
+ if (!split || i < 2) target = dest == 255 ? 255U : static_cast<u32>(dest) + i;
+ else target = second_dest + (i - 2);
+ if (target != 255)
+ emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{250 + i} << 20));
+ }
+ break;
+ }
+ case 0xb63: { // TLD4 (gather), 2D without offsets or depth compare; component at bits 87-88.
+ const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8);
+ const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8);
+ const u32 binding = insn.Bits(54, 5);
+ if (insn.Bits(61, 3) != 1 || insn.Bits(76, 4) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) ||
+ insn.Bits(40, 14) > 0x1fff || binding >= 18 || insn.Bits(89, 1) || mask == 0 ||
+ (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding))
+ return fail(index, "native texture gather mode");
+ result.required_texture_buffer = binding;
+ // Two coordinates: x in Ra, y in Rb (the coordinate list splits evenly across them).
+ emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a} << 20));
+ emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) |
+ (u64{source_b == 255 ? 255U : source_b} << 20));
+ emit(0xc800000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) |
+ (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) |
+ (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | (u64{insn.Bits(87, 2)} << 56));
+ const u32 components = static_cast<u32>(std::popcount(mask));
+ const bool split = second_dest != 255;
+ for (u32 i = 0; i < components; ++i) {
+ u32 target = 255;
+ if (!split || i < 2) target = dest == 255 ? 255U : static_cast<u32>(dest) + i;
+ else target = second_dest + (i - 2);
+ if (target != 255)
+ emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{250 + i} << 20));
+ }
+ break;
+ }
+ case 0xb60: { // TEX, scalar 2D implicit-LOD sample
+ const u32 mask = insn.Bits(72, 4);
+ const u32 second_dest = insn.Bits(64, 8);
+ constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12};
+ constexpr std::array<u32, 5> rgba_masks{7, 11, 13, 14, 15};
+ u32 swizzle = ~0U;
+ const bool split = second_dest != 255;
+ const bool depth_compare = insn.Bits(78, 1);
+ const u32 dimension = insn.Bits(61, 3), lod = insn.Bits(87, 3);
+ const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8);
+ if (dimension == 5 && depth_compare && lod == 0 && !split && mask == 1) {
+ if (!insn.Bits(60, 1) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 ||
+ insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 ||
+ (source_a & 1) || (source_b & 1) || source_a >= 248 || source_b >= 248 || dest >= 248)
+ return fail(index, "native array depth comparison sample mode");
+ result.required_texture_buffer = 2;
+ for (u32 i = 0; i < 3; ++i) {
+ const u32 source = i < 2 ? source_a + i : source_b;
+ emit(0x5c98000000000000ULL | predicate | (250 + i) |
+ (u64{15} << 39) | (u64{source} << 20));
+ }
+ emit(0xc000000000000000ULL | predicate | dest | (u64{250} << 8) |
+ (u64{source_b + 1} << 20) | (u64{3} << 28) | (u64{1} << 31) |
+ (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) |
+ (u64{1} << 50) | (u64{7} << 51));
+ break;
+ }
+ // General form: gather the native operand list (array index first, coordinates,
+ // then LOD/bias and the depth reference) into Maxwell TEX's coordinate and meta
+ // registers, sample into scratch and copy the packed components out. Native packs
+ // up to four values as halves of Ra and Rb (1+1, 2+1, 2+2), five or more as 4+n.
+ const auto general_tex = [&]() -> bool {
+ constexpr std::array<u32, 8> coordinate_count{1, 2, 3, 3, 2, 3, 0, 4};
+ constexpr std::array<u32, 8> maxwell_type{0, 2, 4, 6, 1, 3, 0, 7};
+ const bool offsets = insn.Bits(76, 1) != 0;
+ if (dimension == 6 || insn.Bits(77, 1) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) ||
+ insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 || (lod > 3 && lod != 6 && lod != 7) ||
+ (depth_compare && mask != 1) || mask == 0)
+ return false;
+ const u32 coordinates = coordinate_count[dimension];
+ const bool has_lod = lod == 2 || lod == 3 || lod == 6 || lod == 7;
+ const u32 count = coordinates + (has_lod ? 1 : 0) + (offsets ? 1 : 0) + (depth_compare ? 1 : 0);
+ const u32 in_a = count <= 4 ? (count + 1) / 2 : 4;
+ const auto value = [&](u32 i) -> u32 {
+ if (i < in_a) return source_a == 255 ? 255U : source_a + i;
+ return source_b == 255 ? 255U : source_b + (i - in_a);
+ };
+ if ((source_a != 255 && source_a + in_a > 246) ||
+ (count > in_a && source_b != 255 && source_b + (count - in_a) > 246))
+ return false;
+ const u32 binding = insn.Bits(54, 5);
+ if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
+ return false;
+ result.required_texture_buffer = binding;
+ const auto copy = [&](u32 target, u32 source) {
+ emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20));
+ };
+ // Coordinates to R250.., then LOD/bias, offsets and reference to R246...
+ for (u32 i = 0; i < coordinates; ++i) copy(250 + i, value(i));
+ u32 meta = 246;
+ u32 next = coordinates;
+ if (has_lod) copy(meta++, value(next++));
+ if (offsets) copy(meta++, value(next++));
+ if (depth_compare) copy(meta++, value(next++));
+ emit(0xc000000000000000ULL | predicate | 250 | (u64{250} << 8) |
+ (u64{meta == 246 ? 255U : 246U} << 20) | (u64{maxwell_type[dimension]} << 28) |
+ (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) |
+ (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) | (u64{7} << 51) |
+ (u64{offsets} << 54) | (u64{lod} << 55));
+ const u32 components = static_cast<u32>(std::popcount(mask));
+ if (insn.Bits(79, 1)) {
+ // FP16 results: (c0, c1) packed into the first destination, (c2, c3) into
+ // the second (F2F writes the half in the low 16 bits; ISCADD merges).
+ const auto pack = [&](u32 target, u32 low, u32 high) {
+ if (target == 255) return;
+ emit(0x5ca8000000000000ULL | predicate | 246 | (u64{1} << 8) | (u64{2} << 10) | (u64{low} << 20));
+ emit(0x5ca8000000000000ULL | predicate | 247 | (u64{1} << 8) | (u64{2} << 10) | (u64{high} << 20));
+ emit(0x5c18000000000000ULL | predicate | target | (u64{246} << 8) | (u64{247} << 20) | (u64{16} << 39));
+ };
+ pack(static_cast<u32>(dest), 250, components > 1 ? 251U : 255U);
+ if (components > 2) {
+ pack(split ? second_dest : (dest == 255 ? 255U : static_cast<u32>(dest) + 1),
+ 252, components > 3 ? 253U : 255U);
+ }
+ return true;
+ }
+ for (u32 i = 0; i < components; ++i) {
+ u32 target = 255;
+ if (!split) target = dest == 255 ? 255U : static_cast<u32>(dest) + i;
+ else if (i < 2) target = dest == 255 ? 255U : static_cast<u32>(dest) + i;
+ else target = second_dest + (i - 2);
+ if (target != 255) copy(target, 250 + i);
+ }
+ return true;
+ };
+ if (split) {
+ for (u32 i = 0; i < rgba_masks.size(); ++i) if (rgba_masks[i] == mask) swizzle = i;
+ } else {
+ for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i;
+ }
+ // SM70 scalar coordinates and split RG/BA destinations match TEXS.
+ // Descriptor offsets are words in both encodings, but only native
+ // explicitly encodes the constant-buffer binding.
+ if (!insn.Bits(60, 1) || (dimension != 1 && dimension != 5) ||
+ swizzle == ~0U || insn.Bits(76, 2) ||
+ insn.Bits(81, 3) != 7 || (lod > 1 && (lod != 3 || dimension != 1 || depth_compare)) ||
+ (lod == 3 && ((source_a & 1) || source_a >= 249)) ||
+ insn.Bits(59, 1) ||
+ insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 ||
+ dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) ||
+ (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) ||
+ (depth_compare && (dimension != 1 || split || mask != 1 || (source_a & 1) || source_a >= 249)) ||
+ (dimension == 5 && ((source_a & 1) || source_a >= 249))) {
+ if (!general_tex()) return fail(index, "native texture mode or descriptor binding");
+ break;
+ }
+ result.required_texture_buffer = 2;
+ emit((0xd800000000000000ULL & ~(u64{insn.Bits(79, 1)} << 59)) | predicate | dest | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(64, 8)} << 28) |
+ (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) |
+ (u64{swizzle} << 50) | (u64{dimension == 5 ? (lod ? 8U : 7U) : depth_compare ? (lod ? 6U : 4U) : (lod == 3 ? 3U : lod ? 2U : 1U)} << 53));
+ break;
+ }
+ case 0x816: { // Captured PRMT half-word concatenation
+ if (insn.words[1] > 0xffff || (insn.words[2] & ~0xffU)) return fail(index, "native permutation mode");
+ if (insn.words[1] != 0x5410 && insn.words[1] != 0x5432 && insn.words[1] != 0x7610 && insn.words[1] != 0x7632) {
+ // Byte i of the result is byte (s & 7) of c:a, or its sign bit repeated when s & 8.
+ const u32 a = insn.Bits(24, 8), c = insn.Bits(64, 8);
+ if (a >= 250 && a != 255) return fail(index, "native permutation source");
+ mov_imm(250, 0);
+ for (u32 i = 0; i < 4; ++i) {
+ const u32 select = (insn.words[1] >> (i * 4)) & 0xf;
+ const u32 source = select & 4 ? c : a, position = (select & 3) * 8;
+ emit(0x3800000000000000ULL | predicate | 251 | (u64{source} << 8) |
+ (u64{select & 8 ? (position + 7) | (1U << 8) : position | (8U << 8)} << 20) |
+ (u64{(select >> 3) & 1} << 48));
+ emit(0x36f0000000000000ULL | predicate | 250 | (u64{251} << 8) |
+ (u64{(i * 8) | (8U << 8)} << 20) | (u64{250} << 39));
+ }
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{255} << 20));
+ break;
+ }
+ const u32 a = insn.Bits(24, 8), b = insn.Bits(64, 8);
+ if (insn.words[1] != 0x5432 && insn.words[1] != 0x7632) {
+ emit(0x0400000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{0xffff} << 20));
+ } else {
+ emit(0x3828000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{16} << 20));
+ }
+ if (insn.words[1] == 0x7610 || insn.words[1] == 0x7632) {
+ emit(0x0400000000000000ULL | predicate | 251 | (u64{b} << 8) | (u64{0xffff0000} << 20));
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{251} << 20));
+ } else {
+ emit(base(0x5c18000000000000ULL) | (u64{b} << 8) | (u64{250} << 20) | (u64{16} << 39));
+ }
+ break;
+ }
+ case 0x307: { // FRND.F32, round to an integral floating-point value
+ if (insn.Bits(75, 2) != 2 || insn.Bits(84, 2) != 2 || insn.Bits(81, 3) ||
+ insn.Bits(86, 6)) return fail(index, "native floating round format");
+ emit(base(0x5ca8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{8 + insn.Bits(78, 2)} << 39) |
+ (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(63, 1)} << 45) |
+ (u64{insn.Bits(62, 1)} << 49));
+ break;
+ }
+ case 0x321: // ALD
+ case 0x322: { // AST
+ if (insn.Bits(76, 2) || insn.Bits(79, 1)) return fail(index, "native indexed/patch attribute access");
+ const u32 address = insn.Bits(40, 10);
+ const u32 components = insn.Bits(74, 2) + 1;
+ if (address % 4 || address + components * 4 > 0x400 || insn.Bits(24, 8) != 255) {
+ return fail(index, "native attribute address or indexing");
+ }
+ // a[0x5c] has no Maxwell meaning (Ampere uses it for a per-primitive hint such as the
+ // shading rate). Nothing reads it back; dropping the store keeps full-rate shading.
+ if (opcode == 0x322 && address == 0x5c && components == 1) break;
+ u32 value_reg = opcode == 0x321 ? static_cast<u32>(dest) : insn.Bits(32, 8);
+ if (value_reg == 255) {
+ if (opcode == 0x321) break; // load into RZ: nothing to do
+ // AST of RZ writes zeros: store them from zeroed scratch registers.
+ for (u32 i = 0; i < components; ++i) mov_imm(250 + i, 0);
+ value_reg = 250;
+ }
+ if (value_reg + components > 255) return fail(index, "native attribute register range");
+ const u32 vertex = opcode == 0x321 ? insn.Bits(32, 8) : insn.Bits(64, 8);
+ const u64 translated = (opcode == 0x321 ? 0xefd8000000000000ULL : 0xeff0000000000000ULL) |
+ predicate | value_reg | (u64{255} << 8) | (u64{address} << 20) |
+ (u64{vertex} << 39) | (u64{components - 1} << 47);
+ result.instructions.push_back(translated);
+ break;
+ }
+ case 0x387: case 0x983: { // Captured scalar local spill at an immediate byte offset.
+ const bool store = opcode == 0x387;
+ const u32 data = insn.Bits(32, 8), offset = insn.Bits(40, 24);
+ if (insn.Bits(24, 8) != 255 || (store && (insn.Bits(16, 8) || data > 247)) ||
+ (!store && (data || dest >= 248)) || (offset & 3) || offset > 0x7fffff ||
+ insn.words[2] != (store ? 0x00100800U : 0x00300800U))
+ return fail(index, "native local scalar spill mode");
+ emit((store ? 0xef50000000000000ULL : 0xef40000000000000ULL) | predicate |
+ (store ? data : dest) | (u64{255} << 8) | (u64{offset} << 20) | (u64{4} << 48));
+ break;
+ }
+ case 0x803: { // P2R PR, register merge source, immediate mask, byte select (76-77)
+ if (insn.words[1] > 0x7f || (insn.words[2] & ~0x3000U))
+ return fail(index, "native predicate-to-register mode");
+ emit(base(0x38e8000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.words[1]} << 20) | (u64{insn.Bits(76, 2)} << 41));
+ break;
+ }
+ case 0x804: { // R2P PR, register, immediate mask, byte select (76-77)
+ if (insn.Bits(16, 8) || insn.words[1] > 0x7f || (insn.words[2] & ~0x3000U))
+ return fail(index, "native register-to-predicate mode");
+ emit(0x38f0000000000000ULL | predicate | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.words[1]} << 20) | (u64{insn.Bits(76, 2)} << 41));
+ break;
+ }
+ case 0xc02: // MOV uniform register
+ case 0x202: // MOV register
+ case 0x802: // MOV immediate
+ case 0xa02: { // MOV constant buffer
+ if (insn.Bits(72, 4) != 15 || (insn.Bits(76, 16) & ~(opcode == 0xc02 ? 0x8000U : 0U))) return fail(index, "native MOV modifiers");
+ if (opcode == 0x802) {
+ result.instructions.push_back(base(0x0100000000000000ULL) | (u64{15} << 12) |
+ (u64{insn.words[1]} << 20));
+ } else if (opcode == 0xc02) {
+ if (insn.words[1] >= 16 || insn.words[2] != 0x08000f00) return fail(index, "native uniform move source");
+ emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.words[1] + 216} << 20));
+ } else if (opcode == 0x202) {
+ if (insn.Bits(40, 24)) return fail(index, "native MOV register modifiers");
+ result.instructions.push_back(base(0x5c98000000000000ULL) | (u64{15} << 39) |
+ (u64{insn.Bits(32, 8)} << 20));
+ } else {
+ if (!valid_cbuf() || insn.Bits(62, 2)) return fail(index, "native MOV constant buffer");
+ result.instructions.push_back(base(0x4c98000000000000ULL) | (u64{15} << 39) | cbuf());
+ }
+ break;
+ }
+ case 0x223: case 0x423: case 0x623: case 0x823: case 0xa23: case 0xc23: case 0xe23: { // FFMA
+ if (insn.Bits(76, 1) || (insn.Bits(81, 11) & ~(opcode == 0xe23 || opcode == 0xc23 ? 0x400U : 0U))) return fail(index, "native FFMA denormal/source mode");
+ const u32 a = operand(0), b = operand(1), c = operand(2);
+ emit(base(0x5980000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | (u64{c} << 39) |
+ (u64{insn.Bits(77, 1)} << 50) | (u64{insn.Bits(78, 2)} << 51) | (u64{insn.Bits(80, 1)} << 53));
+ break;
+ }
+ case 0x220: case 0x820: case 0xa20: case 0xc20: { // FMUL
+ const u32 scale = insn.Bits(84, 3);
+ if (insn.Bits(76, 1) || scale == 0 || insn.Bits(81, 3) ||
+ insn.Bits(87, 4) || insn.Bits(91, 1) != (opcode == 0xc20) ||
+ (scale != 4 && (!insn.Bits(80, 1) || insn.Bits(78, 2))))
+ return fail(index, "native FMUL scale/mode");
+ const u32 a = operand(0), b = operand(1);
+ emit(base(0x5c68000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(78, 2)} << 39) | (u64{std::array<u32, 8>{0, 3, 2, 1, 0, 6, 5, 4}[scale]} << 41) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(77, 1)} << 50));
+ break;
+ }
+ case 0x221: case 0x421: case 0x621: case 0x821: case 0xa21: case 0xe21: { // FADD
+ if (insn.Bits(81, 10) || insn.Bits(91, 1) != (opcode == 0xe21)) return fail(index, "native FADD mode");
+ const u32 a = operand(0), b = operand(insn.Bits(9, 3) == 2 || insn.Bits(9, 3) == 3 || insn.Bits(9, 3) == 7 ? 2 : 1);
+ emit(base(0x5c58000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(78, 2)} << 39) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(77, 1)} << 50));
+ break;
+ }
+ case 0x209: case 0x809: case 0xa09: case 0xc09: { // FMNMX, including a uniform source
+ if (insn.Bits(74, 6) || insn.Bits(81, 6) || (opcode != 0xc09 && insn.Bits(91, 1)))
+ return fail(index, "native floating min/max mode");
+ const u32 a = operand(0), b = operand(1);
+ emit(base(0x5c60000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) |
+ (u64{insn.Bits(80, 1)} << 44));
+ break;
+ }
+ case 0x308: case 0xb08: { // MUFU
+ const u32 operation = insn.Bits(74, 6);
+ if (operation > 8 || operation == 6 || operation == 7 || insn.Bits(80, 12)) return fail(index, "native MUFU mode");
+ // SM70+ MUFU.COS/SIN take the angle in turns (compilers multiply by 1/2pi first);
+ // Eden's Maxwell MUFU takes radians (its RRO is a plain move), so scale by 2pi.
+ const bool turns = operation == 0 || operation == 1;
+ const auto to_radians = [&](u32 source) {
+ emit(0x1e00000000000000ULL | predicate | 250 | (u64{source} << 8) |
+ (u64{0x40c90fdbU} << 20));
+ return 250U;
+ };
+ if (insn.Bits(73, 1)) {
+ if (opcode != 0x308 || insn.Bits(60, 2) > 1)
+ return fail(index, "native half special-function source");
+ emit(0x5ca8000000000000ULL | predicate | 250 | (u64{2} << 8) | (u64{1} << 10) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(60, 1)} << 41) |
+ (u64{insn.Bits(63, 1)} << 45) | (u64{insn.Bits(62, 1)} << 49));
+ if (turns) to_radians(250);
+ emit(0x5080000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{operation} << 20));
+ emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) | (u64{250} << 20));
+ } else {
+ u32 src = operand(1);
+ if (turns) src = to_radians(src);
+ emit(base(0x5080000000000000ULL) | (u64{src} << 8) | (u64{operation} << 20));
+ }
+ break;
+ }
+ case 0x526: { // IPA.PASS / CONSTANT with the attribute indexed by a uniform register
+ const u32 index_register = uniform_register(insn.Bits(24, 8));
+ if (index_register == ~0U || insn.Bits(32, 8) != 255 ||
+ (insn.Bits(76, 4) != 0 && insn.Bits(76, 4) != 4) || insn.Bits(81, 3) != 7)
+ return fail(index, "native indexed IPA mode");
+ if (index_register == 255) {
+ emit(base(0xe000000000000000ULL) | (u64{255} << 8) | (u64{255} << 20) |
+ (u64{insn.Bits(64, 8)} << 30) | (u64{insn.Bits(78, 1)} << 55));
+ break;
+ }
+ // Maxwell IPA.IDX reads the whole byte address from the index register.
+ emit(0x1c00000000000000ULL | predicate | 250 | (u64{index_register} << 8) |
+ (u64{insn.Bits(64, 8) * 4} << 20));
+ emit(base(0xe000000000000000ULL) | (u64{250} << 8) | (u64{255} << 20) |
+ (u64{insn.Bits(64, 8)} << 30) | (u64{1} << 38) | (u64{insn.Bits(78, 1)} << 55));
+ break;
+ }
+ case 0x326: { // IPA.PASS / CONSTANT, default or centroid sample location
+ // Bits 76-77 are the sample mode (title 1 interpolates most varyings at the centroid),
+ // bit 78 the constant mode; Maxwell keeps them at bits 52-53 and 55.
+ if (insn.Bits(32, 8) != 255 || insn.Bits(76, 2) > 1 || insn.Bits(79, 1) ||
+ insn.Bits(81, 3) != 7)
+ return fail(index, "native IPA interpolation mode");
+ emit(base(0xe000000000000000ULL) | (u64{255} << 8) | (u64{255} << 20) | (u64{insn.Bits(64, 8)} << 30) |
+ (u64{insn.Bits(76, 2)} << 52) | (u64{insn.Bits(78, 1)} << 55));
+ break;
+ }
+ case 0x81a: { // SGXT.U32 Rd, Ra, n: keep the low n bits (BFE.U32 at position 0)
+ if (insn.words[2] || insn.words[1] > 32 || dest >= 255)
+ return fail(index, "native sign extend mode");
+ emit(base(0x3800000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.words[1] << 8} << 20));
+ break;
+ }
+ case 0x299: { // USHF by a uniform amount, one register shifted (SHL/SHR, bit 75 wraps).
+ const u32 dst = uniform_register(insn.Bits(16, 8)), low = uniform_register(insn.Bits(24, 8));
+ const u32 shift = uniform_register(insn.Bits(32, 8)), high = uniform_register(insn.Bits(64, 8));
+ const bool right = insn.Bits(76, 1) != 0, high_result = insn.Bits(80, 1) != 0;
+ if (dst == ~0U || dst == 255 || low == ~0U || shift == ~0U || high == ~0U ||
+ native_predicate != 7 || insn.Bits(73, 2) != 3 ||
+ (insn.words[2] & ~0x000118ffU) != 0x08000600U)
+ return fail(index, "native uniform register shift mode");
+ u32 source;
+ if (high_result && low == 255) source = high; // (high:0) >> or << s, high word
+ else if (!high_result && high == 255) source = low; // (0:low) >> or << s, low word
+ else return fail(index, "native uniform funnel shift");
+ emit((right ? 0x5c28000000000000ULL : 0x5c48000000000000ULL) | predicate | dst |
+ (u64{source} << 8) | (u64{shift} << 20) | (u64{insn.Bits(75, 1)} << 39));
+ break;
+ }
+ case 0x419: { // SHF.L.W.U32.HI Rd, RZ, Rb, imm (high word of imm:0 << Rb) = imm << (Rb & 31)
+ if (insn.Bits(24, 8) != 255 || (insn.words[2] & ~0xffU) != 0x00010e00)
+ return fail(index, "native funnel shift immediate mode");
+ mov_imm(250, insn.words[1]);
+ emit(base(0x5c48000000000000ULL) | (u64{250} << 8) | (u64{insn.Bits(64, 8)} << 20) |
+ (u64{1} << 39));
+ break;
+ }
+ case 0x805: { // CS2R SRZ
+ if (insn.Bits(72, 8) != 255 || !insn.Bits(80, 1) || dest >= 249) return fail(index, "native special-register read");
+ mov_imm(static_cast<u32>(dest), 0);
+ mov_imm(static_cast<u32>(dest) + 1, 0);
+ break;
+ }
+ case 0x306: case 0xd06: { // I2F.F32.U8, low byte, round-to-nearest (0xd06: uniform source)
+ const u32 source = opcode == 0xd06 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8);
+ if (opcode == 0x306 && insn.words[2] == 0x00101000 && !(insn.words[1] & 0xcfffff00U)) {
+ // I2F.U16 Rd, Rs(.H1): Maxwell's 16-bit source format, byte selector 0 or 2.
+ emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{1} << 10) | (u64{source} << 20) |
+ (u64{insn.Bits(60, 2) * 2} << 41));
+ break;
+ }
+ if (insn.words[2] != (opcode == 0xd06 ? 0x08001000U : 0x1000U) ||
+ (insn.words[1] & 0xffffff00U) || source == ~0U)
+ return fail(index, "native byte-to-float conversion mode");
+ emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{source} << 20));
+ break;
+ }
+ case 0x245: case 0xa45: case 0xc45: { // I2FP.F32.U32
+ // Bits 78-79: rounding (RN/RM/RP/RZ), as Maxwell I2F's bits 39-40.
+ if ((insn.words[2] & ~(0x400U | 0xc000U | (opcode == 0xc45 ? 0x08000000U : 0U))) != 0x00201000) return fail(index, "native packed integer-to-float format");
+ const u32 src = operand(1, false);
+ emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | (u64{insn.Bits(74, 1)} << 13) | (u64{src} << 20) |
+ (u64{insn.Bits(78, 2)} << 39));
+ break;
+ }
+ case 0x207: case 0x807: case 0xa07:
+ case 0x208: case 0x808: case 0xa08: { // Bitwise SEL / FSEL
+ const u32 a = operand(0, false), b = operand(1, false);
+ emit(base(0x5ca0000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ break;
+ }
+ case 0x20a: case 0x80a: case 0xa0a: { // FSET.BF, combined with Pc (bits 87-90) by bits 74-75
+ if (insn.Bits(64, 8) || insn.Bits(74, 2) == 3 || insn.Bits(81, 6) || insn.Bits(91, 1))
+ return fail(index, "native floating comparison output mode");
+ const u32 a = operand(0), b = operand(1);
+ emit(base(0x5800000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(87, 4)} << 39) | (u64{insn.Bits(74, 2)} << 45) |
+ (u64{insn.Bits(76, 4)} << 48) | (u64{1} << 52) | (u64{insn.Bits(80, 1)} << 55));
+ break;
+ }
+ case 0x434: { // HSETP2 with an immediate half pair: A swizzle 74-75 (neg 72, abs 73),
+ // combine 69-70, H_AND 71, compare 76-79, FTZ 80, P/Q 81-86, input 87-90.
+ mov_imm(250, insn.words[1]);
+ emit(0x5d20000000000000ULL | predicate | insn.Bits(84, 3) | (u64{insn.Bits(81, 3)} << 3) |
+ (u64{insn.Bits(80, 1)} << 6) | (u64{insn.Bits(24, 8)} << 8) | (u64{250} << 20) |
+ (u64{insn.Bits(76, 4)} << 35) | (u64{insn.Bits(87, 3)} << 39) |
+ (u64{insn.Bits(90, 1)} << 42) | (u64{insn.Bits(72, 1)} << 43) |
+ (u64{insn.Bits(73, 1)} << 44) | (u64{insn.Bits(69, 2)} << 45) |
+ (u64{insn.Bits(74, 2)} << 47) | (u64{insn.Bits(71, 1)} << 49));
+ break;
+ }
+ case 0x20b: case 0x80b: case 0xa0b: // FSETP
+ case 0x20c: case 0x80c: case 0xa0c: case 0xc0c: { // ISETP
+ const bool floating = (opcode & 0x1ff) == 0xb;
+ if (!floating && insn.Bits(72, 1)) return fail(index, "native extended integer comparison");
+ const u32 a = operand(0, floating), b = operand(1, floating);
+ const u32 combine = insn.Bits(74, 2);
+ if (combine > 2) return fail(index, "native predicate combination");
+ u64 out = (floating ? 0x5bb0000000000000ULL : 0x5b60000000000000ULL) | predicate |
+ insn.Bits(84, 3) | (u64{insn.Bits(81, 3)} << 3) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | (u64{combine} << 45);
+ if (floating) out |= (u64{insn.Bits(80, 1)} << 47) | (u64{insn.Bits(76, 4)} << 48);
+ else out |= (u64{insn.Bits(73, 1)} << 48) | (u64{insn.Bits(76, 3)} << 49);
+ emit(out);
+ break;
+ }
+ case 0x212: case 0x812: case 0xa12: case 0xc12: { // LOP3.LUT, any table, optional P = (result != 0)
+ const u32 lut = insn.Bits(72, 8);
+ const u32 pred_target = insn.Bits(81, 3);
+ if (insn.Bits(80, 1) || insn.Bits(87, 4) != 15 ||
+ (opcode == 0x212 && insn.Bits(40, 24)) || (opcode == 0xa12 && insn.Bits(62, 2)))
+ return fail(index, "native LUT/predicate mode");
+ u32 b = 255;
+ if (opcode == 0x212) {
+ b = insn.Bits(32, 8);
+ } else if (opcode == 0x812) {
+ mov_imm(251, insn.words[1]);
+ b = 251;
+ } else if (opcode == 0xc12) {
+ b = uniform_register(insn.Bits(32, 8));
+ if (b == ~0U) return fail(index, "native uniform logic source");
+ } else {
+ b = operand(1, false);
+ }
+ const u32 a = insn.Bits(24, 8), c = insn.Bits(64, 8);
+ emit(0x5be0000000000000ULL | predicate | dest | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{lut} << 28) | (u64{pred_target == 7 ? 0U : 3U} << 36) | (u64{c} << 39) |
+ (u64{pred_target} << 48));
+ break;
+ }
+ case 0x213: case 0xa13: case 0xc13: { // IABS: max(x, 0 - x), signed.
+ if (insn.words[2] & ~(opcode == 0xc13 ? 0x08000000U : 0U))
+ return fail(index, "native integer absolute mode");
+ const u32 source = operand(1, false);
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{255} << 8) | (u64{source} << 20) |
+ (u64{1} << 48));
+ emit(base(0x5c20000000000000ULL) | (u64{source} << 8) | (u64{250} << 20) |
+ (u64{15} << 39) | (u64{1} << 48));
+ break;
+ }
+ case 0x217: case 0x817: case 0xa17: case 0xc17: { // IMNMX register, immediate, cbuf or uniform operand.
+ if ((insn.words[2] & ~(0x04000200U | (opcode == 0xc17 ? 0x08000000U : 0U))) != 0x03800000 ||
+ (opcode != 0xa17 && insn.words[1] > (opcode == 0x817 ? 0x7ffffU : 255U)) ||
+ (opcode == 0xa17 && (!valid_cbuf() || insn.Bits(62, 2))))
+ return fail(index, "native integer min/max mode");
+ const u32 source = opcode == 0xc17 ? uniform_register(insn.words[1])
+ : opcode == 0xa17 ? operand(1, false)
+ : insn.words[1];
+ if (source == ~0U) return fail(index, "native uniform min/max operand");
+ emit(base(opcode == 0x817 ? 0x3820000000000000ULL : 0x5c20000000000000ULL) |
+ (u64{insn.Bits(24, 8)} << 8) | (u64{source} << 20) |
+ (u64{insn.Bits(87, 4)} << 39) | (u64{insn.Bits(73, 1)} << 48));
+ break;
+ }
+ case 0x819: { // SHF.HI, immediate shift
+ const bool is_signed = insn.Bits(73, 2) == 2;
+ if (!insn.Bits(80, 1) && !insn.Bits(76, 1) && insn.Bits(64, 8) == 255 &&
+ insn.Bits(73, 2) == 3 && !insn.Bits(75, 1) && insn.words[1] <= 32) {
+ emit(base(0x3848000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20));
+ break;
+ }
+ if (!insn.Bits(80, 1) && insn.Bits(76, 1) && insn.Bits(64, 8) == 255 &&
+ (insn.Bits(73, 2) == 3 || is_signed) && !insn.Bits(75, 1) && insn.words[1] <= 32) {
+ // SHF.R low word with a zero high word: a logical right shift.
+ emit(base(0x3828000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20));
+ break;
+ }
+ if ((insn.Bits(73, 2) != 3 && !is_signed) || insn.Bits(75, 1) || !insn.Bits(80, 1) || insn.words[1] > 32) return fail(index, "native funnel-shift mode");
+ const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8), shift = insn.words[1];
+ if (insn.Bits(76, 1)) {
+ if (low != 255) return fail(index, "native nonzero low right-shift word");
+ emit(base(0x3828000000000000ULL) | (u64{high} << 8) | (u64{shift} << 20) |
+ (u64{is_signed} << 48));
+ } else {
+ if (is_signed) return fail(index, "native signed left funnel shift");
+ emit(base(0x36f8000000000000ULL) | (u64{low} << 8) | (u64{high} << 39) | (u64{shift} << 20));
+ }
+ break;
+ }
+ case 0xa11: { // LEA with constant base and carry predicate.
+ if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) ||
+ insn.Bits(81, 3) == 7 || insn.Bits(62, 2))
+ return fail(index, "native constant LEA carry mode");
+ const u32 b = operand(1, false);
+ emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39));
+ emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) |
+ (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49));
+ break;
+ }
+ case 0x211: case 0x811: { // LEA low word, no carry result
+ if (insn.Bits(80, 1)) {
+ // LEA.HI Rd, Ra, Rb (or an immediate, 0x811), RZ, s: Rb + high word of (Ra << s);
+ // bit 73 sign-extends Ra.
+ if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(81, 3) != 7 ||
+ insn.Bits(64, 8) != 255 || (opcode == 0x211 && insn.Bits(62, 2)))
+ return fail(index, "native high effective address mode");
+ u32 b = insn.Bits(32, 8);
+ if (opcode == 0x811) {
+ b = insn.words[1] ? 251U : 255U;
+ if (b == 251) mov_imm(251, insn.words[1]);
+ }
+ const u64 shift = (u64{32 - insn.Bits(75, 5)} << 20) | (u64{insn.Bits(73, 1)} << 48);
+ if (b == 255) {
+ emit(base(0x3828000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | shift);
+ } else {
+ emit(0x3828000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | shift);
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{b} << 20));
+ }
+ break;
+ }
+ if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || insn.Bits(81, 3) != 7 ||
+ (opcode == 0x211 && insn.Bits(62, 1))) return fail(index, "native LEA carry/high mode");
+ // 0x211 bit 63 negates Rb (LEA Rd, Ra, -Rb, s), as Maxwell ISCADD's bit 48.
+ u32 b = insn.Bits(32, 8);
+ if (opcode == 0x811) { mov_imm(251, insn.words[1]); b = 251; }
+ emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39) |
+ (u64{opcode == 0x211 ? insn.Bits(63, 1) : 0U} << 48));
+ break;
+ }
+ case 0x210: case 0x810: case 0xa10: case 0xc10: { // IADD3 with zero third source and no carry outputs
+ // 64-bit adds: IADD3 with a carry-out predicate (bits 81-83), then IADD3.X (bit 74) with
+ // a carry-in predicate (bits 87-89, negate 90). Uniform forms keep UPn in R239 bit 6-n.
+ if ((opcode == 0x210 || opcode == 0x810) && insn.Bits(64, 8) == 255 && !insn.Bits(73, 1) &&
+ !insn.Bits(75, 2) && insn.Bits(84, 3) == 7 && insn.Bits(77, 4) == 15 && !insn.Bits(72, 1) &&
+ (insn.Bits(81, 3) != 7 || insn.Bits(74, 1))) {
+ const bool carry_in = insn.Bits(74, 1) != 0;
+ const u32 carry_out = insn.Bits(81, 3);
+ const u32 a = insn.Bits(24, 8);
+ if ((carry_in && carry_out != 7) || (opcode == 0x210 && (insn.words[1] & ~0xffU)) ||
+ (rewritten_uniform && carry_out != 7 && carry_out > 1) ||
+ (rewritten_uniform && carry_in && insn.Bits(87, 3) > 1 && insn.Bits(87, 3) != 7))
+ return fail(index, "native integer carry mode");
+ if (opcode == 0x810) {
+ emit(0x1c00000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{insn.words[1]} << 20));
+ } else {
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{insn.Bits(32, 8)} << 20));
+ }
+ const auto save = [&](u32 mask) { emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); };
+ const auto restore = [&](u32 mask) { emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); };
+ if (!carry_in) {
+ // carry = (a + b) < a, unsigned
+ const u32 target = rewritten_uniform ? 6 - carry_out : carry_out;
+ const u32 mask = 1U << target;
+ if (rewritten_uniform) save(mask);
+ emit(0x5b60000000000000ULL | predicate | 7 | (u64{target} << 3) | (u64{250} << 8) |
+ (u64{a} << 20) | (u64{7} << 39) | (u64{1} << 49));
+ if (rewritten_uniform) {
+ emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20));
+ restore(mask);
+ }
+ emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{250} << 20));
+ break;
+ }
+ const u32 condition = insn.Bits(87, 3);
+ const u32 source = rewritten_uniform && condition != 7 ? 6 - condition : condition;
+ const u32 mask = 1U << source;
+ mov_imm(252, 1);
+ if (rewritten_uniform && condition != 7) {
+ save(mask);
+ emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{mask} << 20));
+ }
+ emit(0x5ca0000000000000ULL | predicate | 251 | (u64{252} << 8) | (u64{255} << 20) |
+ (u64{source} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ if (rewritten_uniform && condition != 7) restore(mask);
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{251} << 20));
+ break;
+ }
+ if (opcode == 0xa10 && insn.Bits(81, 3) != 7) {
+ if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(84, 7) != 0x7f ||
+ insn.Bits(77, 4) != 15 || insn.Bits(63, 1) || insn.Bits(72, 1))
+ return fail(index, "native constant add carry mode");
+ const u32 b = operand(1, false);
+ emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20));
+ emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) |
+ (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49));
+ break;
+ }
+ if (opcode == 0xa10 && insn.Bits(74, 1)) {
+ if (insn.Bits(24, 8) != 255 || (insn.words[2] & ~0x07800000U) != 0x007fe4ffU)
+ return fail(index, "native extended constant add mode");
+ const u32 b = operand(1, false);
+ mov_imm(250, 1);
+ emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ emit(base(0x5c10000000000000ULL) | (u64{b} << 8) | (u64{250} << 20));
+ break;
+ }
+ // Third source, optionally negated (bit 75; 63 and 72 negate b and a).
+ if (insn.Bits(64, 8) != 255 && (insn.Bits(73, 4) & ~4U) == 0 && insn.Bits(77, 7) == 0x7f && insn.Bits(84, 7) == 0x7f) {
+ const u32 b = operand(1, false);
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{b} << 20) | (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49));
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{insn.Bits(64, 8)} << 20) |
+ (u64{insn.Bits(75, 1)} << 48));
+ break;
+ }
+ if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(77, 7) != 0x7f || insn.Bits(84, 7) != 0x7f) return fail(index, "native integer carry/third source");
+ if (opcode == 0xa10 || opcode == 0xc10) {
+ if (opcode == 0xc10 && (insn.words[1] & ~0x8000003fU)) return fail(index, "native uniform add operand");
+ const u32 b = operand(1, false);
+ emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49));
+ } else if (opcode == 0x210) {
+ if (insn.words[1] & ~0x800000ffU) return fail(index, "native integer add register modifiers");
+ emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49));
+ } else {
+ emit(base(0x1c00000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20) | (u64{insn.Bits(72, 1)} << 56));
+ }
+ break;
+ }
+ case 0x949: case 0x958: { // Captured BRX / BRXU with absolute native-PC table values
+ if (insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x3ffffU) != (opcode == 0x958 ? 0x0b800000U : 0x03800000U))
+ return fail(index, "native computed branch mode");
+ const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32);
+ const s64 relative = static_cast<s64>(raw << 14) >> 14;
+ if (s64{index} * 16 + 16 + relative != 0)
+ return fail(index, "native computed branch relative base");
+ const s64 offset = -s64{address_of(static_cast<u32>(result.instructions.size()))} - 8;
+ if (offset < -0x800000) return fail(index, "native computed branch range");
+ const u32 source = opcode == 0x958 ? uniform_register(insn.Bits(24, 8)) : insn.Bits(24, 8);
+ if (source == ~0U) return fail(index, "native uniform computed branch source");
+ emit(0xe25000000000000fULL | predicate | (u64{source} << 8) |
+ ((static_cast<u64>(offset) & 0xffffff) << 20));
+ computed_branch = true;
+ break;
+ }
+ case 0x944: { // Verified CALL.REL.NOINC with R55 holding the return PC
+ // A predicated NOINC forward transfer can terminate a loop at its
+ // reconvergence point without a return-link setup or matching RET.
+ if (native_predicate != 7 && insn.words[1] == 16 && insn.words[2] == 0x03c00000 &&
+ index + 2 < code.size() && code[index + 1].Bits(0, 12) == 0x947) {
+ relocations.push_back({static_cast<u32>(result.instructions.size()), index + 2});
+ emit(0xe24000000000000fULL | predicate);
+ break;
+ }
+ if (main_exited || index == 0 || insn.words[0] != 0x7944 ||
+ (insn.words[2] & ~0x3ffffU) != 0x03c00000 ||
+ (code[index - 1].words[0] & ~0x00ff0000U) != 0x00007802 ||
+ code[index - 1].Bits(16, 8) >= 250 ||
+ code[index - 1].words[1] != (index + 1) * 16 ||
+ code[index - 1].words[2] != 0xf00)
+ return fail(index, "native call linkage mode");
+ const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32);
+ const s64 relative = static_cast<s64>(raw << 14) >> 14;
+ const s64 target = s64{index} * 16 + 16 + relative;
+ if (target < 0 || target % 16) return fail(index, "native call destination alignment");
+ const u32 link = code[index - 1].Bits(16, 8);
+ for (size_t call = 0; call < call_targets.size(); ++call) {
+ if (call_targets[call] == target / 16 && call_links[call] != link)
+ return fail(index, "native inconsistent call linkage");
+ }
+ call_targets.push_back(static_cast<u32>(target / 16));
+ call_links.push_back(link);
+ relocations.push_back({static_cast<u32>(result.instructions.size()), target / 16});
+ emit(0xe26000000007000fULL);
+ break;
+ }
+ case 0x950: { // Verified RET.REL.NODEC epilogue restoring the call link
+ const u32 return_reg = insn.Bits(24, 8);
+ // The return register is the link itself, or the last write to it in this function
+ // is a copy of the link (MOV or IMAD.MOV.U32 Rd, RZ, RZ, Rlink).
+ const auto copies_link = [&](const Instruction& prior) {
+ if (prior.Bits(16, 8) != return_reg || prior.Bits(12, 4) != 7) return false;
+ if ((prior.words[0] & 0xff00ffffU) == 0x00007202U)
+ return prior.words[1] == *active_link && prior.words[2] == 0xf00;
+ if ((prior.words[0] & 0xff00ffffU) == 0xff007224U)
+ return prior.words[1] == 0xff && prior.words[2] == (0x078e0000U | *active_link);
+ return false;
+ };
+ bool linked = active_link && return_reg == *active_link;
+ for (u32 prior = index; active_link && !linked && prior-- > active_entry;) {
+ if (copies_link(code[prior])) linked = true;
+ else if (code[prior].Bits(16, 8) == return_reg) break;
+ }
+ if (!main_exited || !linked ||
+ (insn.words[0] & 0x00ffffffU) != 0x00007950 || return_reg >= 249 ||
+ (insn.words[2] & ~0x3ffffU) != 0x03c00000 || !barrier_tokens.empty())
+ return fail(index, "native return linkage mode");
+ const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32);
+ const s64 relative = static_cast<s64>(raw << 14) >> 14;
+ if (s64{index} * 16 + 16 + relative != 0)
+ return fail(index, "native return relative base");
+ emit(0xe32000000007000fULL);
+ bool complete = !computed_branch;
+ for (u32 target : call_targets) complete &= target <= index;
+ if (complete) return finish(index);
+ break;
+ }
+ case 0x945: case 0x941: case 0x942: case 0x948: // (+ WARPSYNC)
+ // BSSY / BSYNC / BREAK only manage warp reconvergence: BSYNC waits for the other
+ // threads and falls through, BREAK drops the thread from that wait. Eden runs threads
+ // independently, so all three are no-ops (a SYNC would wrongly jump, and a branch
+ // out of a lowered SSY region would leave its token on Eden's flow stack).
+ break;
+ case 0x947: { // BRA
+ u64 branch_predicate = predicate;
+ if (opcode == 0x947 && insn.Bits(87, 4) != 7) {
+ if (insn.Bits(12, 4) != 7) {
+ relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1});
+ emit(0xe24000000000000fULL | (u64{insn.Bits(12, 4) ^ 8U} << 16));
+ }
+ branch_predicate = u64{insn.Bits(87, 4)} << 16;
+ } else if (insn.Bits(87, 4) != 7) {
+ return fail(index, "native extra branch condition");
+ }
+ const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32);
+ const s64 relative = static_cast<s64>(raw << 14) >> 14;
+ const s64 target = s64{index} * 16 + 16 + relative;
+ if (target < 0 || target % 16) return fail(index, "native branch destination alignment");
+ relocations.push_back({static_cast<u32>(result.instructions.size()), target / 16});
+ emit(0xe24000000000000fULL | branch_predicate);
+ if (computed_branch && target == s64{index} * 16 && branch_predicate == (u64{7} << 16))
+ return finish(index); // the closing self-branch
+ break;
+ }
+ case 0x94d: { // EXIT
+ if (insn.Bits(84, 2)) {
+ return fail(index, "native conditional/special EXIT");
+ }
+ if (insn.Bits(87, 4) != 7) {
+ // @Pg EXIT Pc: exits when both hold; skip over a Pc-predicated EXIT unless Pg.
+ if (native_predicate != 7) {
+ relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1});
+ emit(0xe24000000000000fULL | (u64{native_predicate ^ 8U} << 16));
+ }
+ emit(0xe30000000000000fULL | (u64{insn.Bits(87, 4)} << 16));
+ break;
+ }
+ result.instructions.push_back(0xe30000000000000fULL | predicate);
+ if (insn.Bits(12, 4) != 7) break;
+ if (!barrier_tokens.empty()) return fail(index, "native unclosed reconvergence scope");
+ if (call_targets.empty()) {
+ const bool pending_branch = std::any_of(relocations.begin(), relocations.end(),
+ [&](const Relocation& relocation) { return relocation.target > index; });
+ if (!pending_branch && !computed_branch) return finish(index);
+ break;
+ }
+ if (main_exited) {
+ // A library function may end the shader instead of returning (Eden inlines
+ // calls, so this is an ordinary exit at that point).
+ bool complete = std::none_of(relocations.begin(), relocations.end(),
+ [&](const Relocation& relocation) { return relocation.target > index; });
+ for (u32 target : call_targets) complete &= target <= index;
+ if (complete && !computed_branch) return finish(index);
+ break;
+ }
+ for (u32 target : call_targets) {
+ if (target <= index) return fail(index, "native call target overlaps main program");
+ }
+ main_exited = true;
+ break;
+ }
+ default:
+ return fail(index, "unsupported native opcode");
+ }
+ if (!operand_error.empty()) return fail(index, operand_error);
+ if (output_mask) {
+ emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{output_mask} << 20));
+ emit(0x38f0000000070000ULL | (u64{254} << 8) | (u64{output_mask} << 20));
+ }
+ if (guarded_uniform) restore_predicate(guard_mask);
+ }
+ return fail(static_cast<u32>(code.size()), main_exited ? "native program lacks captured library return" : "native program has no unconditional EXIT");
+}
+
+} // namespace Shader::Ampere
diff --git a/src/shader_recompiler/frontend/ampere/program_address_map.h b/src/shader_recompiler/frontend/ampere/program_address_map.h
new file mode 100644
index 0000000000..bee32a821c
--- /dev/null
+++ b/src/shader_recompiler/frontend/ampere/program_address_map.h
@@ -0,0 +1,29 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+#include <optional>
+#include <span>
+#include <vector>
+#include "common/common_types.h"
+
+namespace Shader::Ampere {
+constexpr u64 NativeAddressMapMagic = 0x50414d4e4544454eULL;
+inline bool HasNativeAddressMap(std::span<const u64> code) {
+ return code.size() >= 2 && code.back() == NativeAddressMapMagic;
+}
+inline void AppendNativeAddressMap(std::vector<u64>& code, std::span<const u32> offsets) {
+ if (offsets.empty()) return;
+ for (u32 index : offsets) code.push_back(0x50U + index * 8 + ((index + 1) / 3) * 8);
+ code.push_back(offsets.size());
+ code.push_back(NativeAddressMapMagic);
+}
+inline std::optional<u32> NativeProgramAddress(std::span<const u64> code, u32 address) {
+ if (!HasNativeAddressMap(code) || address % 16) return {};
+ const u64 count = code[code.size() - 2];
+ if (!count || count > 65536 || count + 2 > code.size() || address / 16 >= count) return {};
+ const size_t start = code.size() - static_cast<size_t>(count) - 2;
+ const u64 target = code[start + address / 16];
+ if (target < 0x50 || target >= start * 8 || target % 8 || target % 32 == 0) return {};
+ return static_cast<u32>(target);
+}
+} // namespace Shader::Ampere
diff --git a/src/shader_recompiler/frontend/ampere/program_capture.h b/src/shader_recompiler/frontend/ampere/program_capture.h
new file mode 100644
index 0000000000..7e308de4ab
--- /dev/null
+++ b/src/shader_recompiler/frontend/ampere/program_capture.h
@@ -0,0 +1,54 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h"
+
+namespace Shader::Ampere {
+class ProgramCaptureState {
+public:
+ enum class Result { Continue, Complete, InvalidTarget };
+ static constexpr u32 MaximumInstructions = 65536;
+
+ Result Observe(const Instruction& instruction, u32 index) {
+ const u32 opcode = instruction.Bits(0, 12);
+ if (opcode == 0x944 || opcode == 0x947 || opcode == 0x945) {
+ const u64 raw = instruction.words[1] | (u64{instruction.words[2] & 0x3ffff} << 32);
+ const s64 relative = opcode == 0x945 ? static_cast<s32>(instruction.words[1])
+ : static_cast<s64>(raw << 14) >> 14;
+ const s64 target = s64{index} * 16 + 16 + relative;
+ if (target < 0 || target % 16 || target >= MaximumInstructions * 16)
+ return Result::InvalidTarget;
+ // CALL.REL.NOINC (bit 86) pushes no return address: the compiler uses it as a jump
+ // (the lowering treats it as one), so it never pairs with a RET.
+ const bool noinc_call = opcode == 0x944 && (instruction.words[2] & 0x400000) != 0;
+ if (opcode == 0x944 && !noinc_call) {
+ last_call_target = std::max(last_call_target, static_cast<u32>(target / 16));
+ } else {
+ last_forward_target = std::max(last_forward_target, static_cast<u32>(target / 16));
+ }
+ // The compiler ends every program with an unconditional branch to itself; with a jump
+ // table (whose targets live in a constant buffer) that is the only reliable end.
+ if (opcode == 0x947 && computed_branch && instruction.Bits(12, 4) == 7 &&
+ target == s64{index} * 16)
+ return Result::Complete;
+ }
+ if (opcode == 0x949 || opcode == 0x958) computed_branch = true;
+ if (computed_branch) return Result::Continue;
+ if (opcode == 0x94d && instruction.Bits(12, 4) == 7 && index >= last_forward_target) {
+ if (!last_call_target) return Result::Complete;
+ main_exited = true;
+ }
+ if (opcode == 0x950 && instruction.Bits(12, 4) == 7 && main_exited &&
+ index >= last_call_target && index >= last_forward_target)
+ return Result::Complete;
+ return Result::Continue;
+ }
+
+private:
+ u32 last_call_target{};
+ u32 last_forward_target{};
+ bool main_exited{};
+ bool computed_branch{};
+};
+} // namespace Shader::Ampere
diff --git a/src/shader_recompiler/frontend/ampere/program_override.h b/src/shader_recompiler/frontend/ampere/program_override.h
new file mode 100644
index 0000000000..0943028b0c
--- /dev/null
+++ b/src/shader_recompiler/frontend/ampere/program_override.h
@@ -0,0 +1,85 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <cstring>
+#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h"
+
+namespace Shader::Ampere {
+
+inline u64 NativeProgramKey(std::span<const Instruction> source) {
+ u64 hash = 14695981039346656037ULL;
+ for (const auto& instruction : source) {
+ for (u32 word : instruction.words) {
+ for (u32 byte = 0; byte < 4; ++byte) hash = (hash ^ ((word >> (byte * 8)) & 255)) * 1099511628211ULL;
+ }
+ }
+ return hash;
+}
+
+// Private iteration format: version, lengths, texture bank, exact original
+// instructions, then the newly generated canonical instructions. Source bytes
+// are compared directly; the filename hash is only an index.
+inline std::vector<u8> EncodeProgramOverride(std::span<const Instruction> source,
+ const LoweredProgram& program) {
+ if (!program || program.source_size != source.size_bytes() || program.source_instruction_offsets.size() != source.size()) return {};
+ const std::array<u32, 6> header{0x4e454445, 0x3230564f,
+ static_cast<u32>(source.size_bytes()), static_cast<u32>(program.instructions.size()),
+ program.required_texture_buffer, static_cast<u32>(program.source_instruction_offsets.size())};
+ std::vector<u8> bytes(sizeof(header) + source.size_bytes() + program.instructions.size() * 8 + program.source_instruction_offsets.size() * 4);
+ std::memcpy(bytes.data(), header.data(), sizeof(header));
+ std::memcpy(bytes.data() + sizeof(header), source.data(), source.size_bytes());
+ std::memcpy(bytes.data() + sizeof(header) + source.size_bytes(), program.instructions.data(), program.instructions.size() * 8);
+ std::memcpy(bytes.data() + sizeof(header) + source.size_bytes() + program.instructions.size() * 8,
+ program.source_instruction_offsets.data(), program.source_instruction_offsets.size() * 4);
+ return bytes;
+}
+
+inline u32 ProgramOverrideSourceSize(std::span<const u8> bytes) {
+ std::array<u32, 6> header{};
+ if (bytes.size() < sizeof(header)) return 0;
+ std::memcpy(header.data(), bytes.data(), sizeof(header));
+ const bool mapped = header[1] == 0x3230564f;
+ if (header[0] != 0x4e454445 || (!mapped && header[1] != 0x3130564f) ||
+ (mapped ? header[5] != header[2] / 16 : header[5] != 0) ||
+ header[2] == 0 || header[2] % sizeof(Instruction) || header[2] > 1048576 ||
+ header[3] == 0 || header[3] > 1048576 || (header[4] != ~0U && header[4] >= 18)) return 0;
+ if (bytes.size() != sizeof(header) + size_t{header[2]} + size_t{header[3]} * 8 + size_t{header[5]} * 4) return 0;
+ return header[2];
+}
+
+inline LoweredProgram DecodeProgramOverride(std::span<const u8> bytes,
+ std::span<const Instruction> source) {
+ LoweredProgram result;
+ const auto fail = [&result](std::string_view message) {
+ result.instructions.clear();
+ result.source_instruction_offsets.clear();
+ result.error = message;
+ return result;
+ };
+ std::array<u32, 6> header{};
+ if (!ProgramOverrideSourceSize(bytes)) return fail("invalid native override header or length");
+ std::memcpy(header.data(), bytes.data(), sizeof(header));
+ if (header[2] != source.size_bytes()) return fail("native override source length mismatch");
+ if (std::memcmp(bytes.data() + sizeof(header), source.data(), source.size_bytes()))
+ return fail("native override source mismatch");
+ result.instructions.resize(header[3]);
+ std::memcpy(result.instructions.data(), bytes.data() + sizeof(header) + source.size_bytes(), header[3] * 8);
+ if (header[5]) {
+ result.source_instruction_offsets.resize(header[5]);
+ std::memcpy(result.source_instruction_offsets.data(),
+ bytes.data() + sizeof(header) + source.size_bytes() + size_t{header[3]} * 8,
+ size_t{header[5]} * 4);
+ u32 previous = 0;
+ for (u32 offset : result.source_instruction_offsets) {
+ if (offset >= result.instructions.size() || offset < previous)
+ return fail("native override address map mismatch");
+ previous = offset;
+ }
+ }
+ result.source_size = header[2];
+ result.required_texture_buffer = header[4];
+ return result;
+}
+
+} // namespace Shader::Ampere
diff --git a/src/shader_recompiler/frontend/ir/breadth_first_search.h b/src/shader_recompiler/frontend/ir/breadth_first_search.h
index afe153f180..ad19917781 100644
--- a/src/shader_recompiler/frontend/ir/breadth_first_search.h
+++ b/src/shader_recompiler/frontend/ir/breadth_first_search.h
@@ -6,10 +6,12 @@
#pragma once
+#include <algorithm>
#include <functional>
#include <optional>
#include <type_traits>
#include <queue>
+#include <unordered_set>
#include <boost/container/small_vector.hpp>
@@ -28,6 +30,23 @@ auto BreadthFirstSearch(Value value, Pred&& pred, Resolve&& resolve = {})
// Breadth-first search visiting the right most arguments first
// Small vector has been determined from shaders in Super Smash Bros. Ultimate
boost::container::small_vector<const Inst*, 2> visited;
+ // Once a search grows, a linear scan of `visited` per argument made it quadratic: a 28,000
+ // instruction Switch 2 compute shader took 3.4 s to translate, most of it here. Same visit
+ // order, so the same result.
+ std::unordered_set<const Inst*> visited_set;
+ const auto first_visit = [&](const Inst* inst) {
+ if (!visited_set.empty()) {
+ return visited_set.insert(inst).second;
+ }
+ if (std::ranges::find(visited, inst) != visited.end()) {
+ return false;
+ }
+ visited.push_back(inst);
+ if (visited.size() > 64) {
+ visited_set.insert(visited.begin(), visited.end());
+ }
+ return true;
+ };
std::queue<const Inst*> queue;
queue.push(value.InstRecursive());
@@ -47,8 +66,7 @@ auto BreadthFirstSearch(Value value, Pred&& pred, Resolve&& resolve = {})
}
// Queue instruction if it hasn't been visited
const Inst* const arg_inst{arg_value.InstRecursive()};
- if (std::ranges::find(visited, arg_inst) == visited.end()) {
- visited.push_back(arg_inst);
+ if (first_visit(arg_inst)) {
queue.push(arg_inst);
}
}
diff --git a/src/shader_recompiler/frontend/maxwell/control_flow.cpp b/src/shader_recompiler/frontend/maxwell/control_flow.cpp
index 7dabfbf16e..9fb30df664 100644
--- a/src/shader_recompiler/frontend/maxwell/control_flow.cpp
+++ b/src/shader_recompiler/frontend/maxwell/control_flow.cpp
@@ -457,7 +457,7 @@ CFG::AnalysisState CFG::AnalyzeBRX(Block* block, Location pc, Instruction inst,
block->indirect_branches.reserve(targets.size());
for (const u32 target : targets) {
- Block* const branch{AddLabel(block, block->stack, target, function_id)};
+ Block* const branch{AddLabel(block, block->stack, env.TranslateProgramAddress(target), function_id)};
block->indirect_branches.push_back({
.block = branch,
.address = target,
diff --git a/src/shader_recompiler/frontend/maxwell/translate_program.cpp b/src/shader_recompiler/frontend/maxwell/translate_program.cpp
index 0cc4b3b844..cd4678cef6 100644
--- a/src/shader_recompiler/frontend/maxwell/translate_program.cpp
+++ b/src/shader_recompiler/frontend/maxwell/translate_program.cpp
@@ -391,6 +391,7 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo
Optimization::ConstantPropagationPass(env, program);
Optimization::PositionPass(env, program);
+ Optimization::BlitConstantsPass(env, program);
Optimization::GlobalMemoryToStorageBufferPass(program, normalized_host_info);
Optimization::GeometryCompactionPass(program, normalized_host_info);
diff --git a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp
index f49d54e7ab..bdb4494a15 100644
--- a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp
+++ b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp
@@ -5,6 +5,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <optional>
+#include <unordered_map>
#include <boost/container/flat_set.hpp>
#include <boost/container/small_vector.hpp>
@@ -54,6 +55,11 @@ struct StorageInfo {
StorageBufferSet set;
StorageInstVector to_replace;
StorageWritesSet writes;
+ /// Track results by the instruction the search starts from, with and without the NVN bias:
+ /// accesses through the same address share them, and a search can walk thousands of
+ /// instructions (the IR does not change while storage buffers are collected).
+ std::unordered_map<const IR::Inst*, std::optional<StorageBufferAddr>> biased_tracks;
+ std::unordered_map<const IR::Inst*, std::optional<StorageBufferAddr>> unbiased_tracks;
};
/// Returns true when the instruction is a global memory instruction
@@ -387,10 +393,22 @@ void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info)
}
// First try to find storage buffers in the NVN address
const IR::U32 low_addr{low_addr_info->value};
- std::optional<StorageBufferAddr> storage_buffer{Track(low_addr, &nvn_bias)};
+ const auto cached_track = [&](const Bias* bias, auto& cache) {
+ if (low_addr.IsImmediate()) {
+ return Track(low_addr, bias);
+ }
+ const IR::Inst* const start{low_addr.InstRecursive()};
+ if (const auto it = cache.find(start); it != cache.end()) {
+ return it->second;
+ }
+ const std::optional<StorageBufferAddr> result{Track(low_addr, bias)};
+ cache.emplace(start, result);
+ return result;
+ };
+ std::optional<StorageBufferAddr> storage_buffer{cached_track(&nvn_bias, info.biased_tracks)};
if (!storage_buffer) {
// If it fails, track without a bias
- storage_buffer = Track(low_addr, nullptr);
+ storage_buffer = cached_track(nullptr, info.unbiased_tracks);
if (!storage_buffer) {
// If that also fails, use NVN fallbacks
LOG_WARNING(Shader, "Storage buffer failed to track, using global memory fallbacks");
diff --git a/src/shader_recompiler/ir_opt/passes.h b/src/shader_recompiler/ir_opt/passes.h
index bf325a4cd9..e3422deb9d 100644
--- a/src/shader_recompiler/ir_opt/passes.h
+++ b/src/shader_recompiler/ir_opt/passes.h
@@ -28,6 +28,7 @@ void LowerInt64ToInt32(IR::Program& program);
void RescalingPass(IR::Program& program);
void SsaRewritePass(IR::Program& program);
void PositionPass(Environment& env, IR::Program& program);
+void BlitConstantsPass(Environment& env, IR::Program& program);
void TexturePass(Environment& env, IR::Program& program, const HostTranslateInfo& host_info);
void LayerPass(IR::Program& program, const HostTranslateInfo& host_info);
void VendorWorkaroundPass(IR::Program& program);
diff --git a/src/shader_recompiler/ir_opt/position_pass.cpp b/src/shader_recompiler/ir_opt/position_pass.cpp
index 3c20b71897..4890346944 100644
--- a/src/shader_recompiler/ir_opt/position_pass.cpp
+++ b/src/shader_recompiler/ir_opt/position_pass.cpp
@@ -74,4 +74,70 @@ void PositionPass(Environment& env, IR::Program& program) {
}
}
}
+// Switch 2 fullscreen blit vertex shaders read a parameter block from constant bank 24 (lowered to
+// bank 17, which games never bind) that the console provides itself. For a blit covering the whole
+// render target it holds: texel size (0x0/0x4, again at 0x10/0x14), size (0x8/0xc), a UV scale of
+// 1 (0x18/0x1c), the UV clamp rectangle 0,0..1,1 (0x30-0x3c, read by the fragment shader), a zero
+// dither bias (0x40-0x48) and the integer source rectangle 0,0..W-1,H-1 (0x50-0x5c).
+void BlitConstantsPass(Environment& env, IR::Program& program) {
+ if (env.ShaderStage() != Stage::VertexB && env.ShaderStage() != Stage::Fragment) {
+ return;
+ }
+ bool used{};
+ for (IR::Block* const block : program.post_order_blocks) {
+ for (IR::Inst& inst : block->Instructions()) {
+ const IR::Opcode opcode{inst.GetOpcode()};
+ if (opcode != IR::Opcode::GetCbufU32 && opcode != IR::Opcode::GetCbufF32 &&
+ opcode != IR::Opcode::GetCbufU32x2) {
+ continue;
+ }
+ if (!inst.Arg(0).IsImmediate() || inst.Arg(0).U32() != 17 || !inst.Arg(1).IsImmediate()) {
+ continue;
+ }
+ IR::IREmitter ir{*block, IR::Block::InstructionList::s_iterator_to(inst)};
+ const IR::F32 width{ir.RenderAreaWidth()};
+ const IR::F32 height{ir.RenderAreaHeight()};
+ const auto word = [&](u32 offset) -> IR::U32 {
+ switch (offset) {
+ case 0x0:
+ case 0x10:
+ return ir.BitCast<IR::U32>(IR::F32{ir.FPRecip(width)});
+ case 0x4:
+ case 0x14:
+ return ir.BitCast<IR::U32>(IR::F32{ir.FPRecip(height)});
+ case 0x8:
+ return ir.BitCast<IR::U32>(width);
+ case 0xc:
+ return ir.BitCast<IR::U32>(height);
+ case 0x18:
+ case 0x1c:
+ case 0x38:
+ case 0x3c:
+ return ir.Imm32(0x3f800000U);
+ case 0x58:
+ return IR::U32{ir.ConvertFToU(32, ir.FPAdd(width, ir.Imm32(-1.0f)))};
+ case 0x5c:
+ return IR::U32{ir.ConvertFToU(32, ir.FPAdd(height, ir.Imm32(-1.0f)))};
+ default:
+ return ir.Imm32(0U);
+ }
+ };
+ const u32 offset{inst.Arg(1).U32()};
+ IR::Value value;
+ if (opcode == IR::Opcode::GetCbufU32x2) {
+ value = ir.CompositeConstruct(word(offset), word(offset + 4));
+ } else if (opcode == IR::Opcode::GetCbufF32) {
+ value = ir.BitCast<IR::F32>(word(offset));
+ } else {
+ value = word(offset);
+ }
+ inst.ReplaceUsesWith(value);
+ used = true;
+ }
+ }
+ if (used) {
+ program.info.uses_render_area = true;
+ }
+}
+
} // namespace Shader::Optimization
diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt
index 64d14ab2a6..e97b425412 100644
--- a/src/video_core/CMakeLists.txt
+++ b/src/video_core/CMakeLists.txt
@@ -90,6 +90,8 @@ add_library(video_core STATIC
host1x/vic.h
macro.cpp
macro.h
+ macro_memo.h
+ macro_turing.h
fence_manager.h
gpu.cpp
gpu.h
@@ -99,6 +101,10 @@ add_library(video_core STATIC
invalidation_accumulator.h
memory_manager.cpp
memory_manager.h
+ native_qmd.h
+ native_query_trace.h
+ nx2.h
+ nx2_profile.h
present.h
pte_kind.h
query_cache/bank_base.h
diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index f30c85911f..ad2e200438 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -9,6 +9,9 @@
#include <algorithm>
#include <memory>
#include <numeric>
+#include <stdexcept>
+#include <vector>
+#include <fmt/ranges.h>
#include "common/range_sets.inc"
#include "video_core/buffer_cache/buffer_cache_base.h"
@@ -78,6 +81,20 @@ void BufferCache<P>::TickFrame() {
return;
}
runtime.TickFrame(slot_buffers);
+ if (static u64 frames{}; ++frames % 600 == 0) {
+ u64 total{};
+ size_t count{};
+ std::vector<std::pair<u64, VAddr>> largest;
+ for (const auto [buffer_id, buffer] : slot_buffers) {
+ total += buffer->SizeBytes();
+ ++count;
+ largest.emplace_back(buffer->SizeBytes(), buffer->CpuAddr());
+ }
+ std::partial_sort(largest.begin(), largest.begin() + std::min<size_t>(6, largest.size()),
+ largest.end(), std::greater<>{});
+ LOG_INFO(HW_GPU, "Buffer cache: {} buffers {} MiB; largest {}", count, total >> 20,
+ fmt::join(largest.begin(), largest.begin() + std::min<size_t>(6, largest.size()), ", "));
+ }
// Calculate hits and shots and move hit bits to the right
const u32 hits = std::reduce(channel_state->uniform_cache_hits.begin(),
@@ -1194,8 +1211,15 @@ void BufferCache<P>::BindHostComputeUniformBuffers() {
return alignment > 1 && (offset % alignment) != 0;
}
}();
+ // Small buffers the GPU has not written are streamed, as graphics ones are: synchronizing
+ // them made every dispatch re-protect pages an inline upload had just unprotected (two
+ // VirtualProtect calls per dispatch, ~4,000 dispatches a second in title 1).
+ const bool use_fast_buffer =
+ needs_alignment_stream ||
+ (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size &&
+ !memory_tracker.IsRegionGpuModified(binding.device_addr, size));
if constexpr (!IS_OPENGL) {
- if (needs_alignment_stream) {
+ if (use_fast_buffer) {
const std::span<u8> span =
runtime.BindMappedUniformBuffer(0, binding_index, size);
device_memory.ReadBlockUnsafe(binding.device_addr, span.data(), size);
@@ -1372,7 +1396,12 @@ void BufferCache<P>::UpdateVertexBuffer(u32 index) {
const auto& array = maxwell3d->regs.vertex_streams[index];
const auto& limit = maxwell3d->regs.vertex_stream_limits[index];
const GPUVAddr gpu_addr_begin = array.Address();
- const GPUVAddr gpu_addr_end = limit.Address() + 1;
+ const bool native = maxwell3d->GetNativeMacroEngine().native_turing;
+ const u64 native_size = native ? maxwell3d->regs.NativeVertexStreamSize(index) : 0;
+ if (native && array.enable != 0 && native_size > (std::numeric_limits<u32>::max)()) {
+ throw std::runtime_error("Native vertex stream exceeds supported 32-bit buffer size");
+ }
+ const GPUVAddr gpu_addr_end = native ? gpu_addr_begin + native_size : limit.Address() + 1;
const std::optional<DAddr> device_addr = gpu_memory->GpuToCpuAddress(gpu_addr_begin);
const u32 address_size = static_cast<u32>(gpu_addr_end - gpu_addr_begin);
u32 size = address_size; // TODO: Analyze stride and number of vertices
diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h
index 26bf3dd9ed..57011752cc 100644
--- a/src/video_core/buffer_cache/buffer_cache_base.h
+++ b/src/video_core/buffer_cache/buffer_cache_base.h
@@ -21,6 +21,7 @@
#include <boost/container/small_vector.hpp>
#include "common/common_types.h"
+#include "common/spin_recursive_mutex.h"
#include "common/div_ceil.h"
#include "common/literals.h"
#include "common/lru_cache.h"
@@ -339,7 +340,7 @@ public:
} while (channel_state->has_deleted_buffers);
}
- std::recursive_mutex mutex;
+ Common::SpinRecursiveMutex mutex;
Runtime& runtime;
bool any_buffer_uploaded = false;
@@ -544,7 +545,8 @@ private:
bool immediately_free = false;
#endif
- std::array<BufferId, ((1ULL << 34) >> CACHING_PAGEBITS)> page_table;
+ // Covers the device address space (MaxwellDeviceTraits::device_virtual_bits).
+ std::array<BufferId, ((1ULL << 36) >> CACHING_PAGEBITS)> page_table;
Common::ScratchBuffer<u8> tmp_buffer;
};
diff --git a/src/video_core/buffer_cache/memory_tracker_base.h b/src/video_core/buffer_cache/memory_tracker_base.h
index b8547f3149..8936da023f 100644
--- a/src/video_core/buffer_cache/memory_tracker_base.h
+++ b/src/video_core/buffer_cache/memory_tracker_base.h
@@ -22,7 +22,7 @@ namespace VideoCommon {
template <typename DeviceTracker>
class MemoryTrackerBase {
- static constexpr size_t MAX_CPU_PAGE_BITS = 34;
+ static constexpr size_t MAX_CPU_PAGE_BITS = 36; // device addresses (MaxwellDeviceTraits)
static constexpr size_t HIGHER_PAGE_BITS = 22;
static constexpr size_t HIGHER_PAGE_SIZE = 1ULL << HIGHER_PAGE_BITS;
static constexpr size_t HIGHER_PAGE_MASK = HIGHER_PAGE_SIZE - 1ULL;
diff --git a/src/video_core/buffer_cache/usage_tracker.h b/src/video_core/buffer_cache/usage_tracker.h
index 6f84e463e3..3e2da8bc0b 100644
--- a/src/video_core/buffer_cache/usage_tracker.h
+++ b/src/video_core/buffer_cache/usage_tracker.h
@@ -6,6 +6,10 @@
#pragma once
+#include <array>
+#include <utility>
+#include <vector>
+
#include "common/alignment.h"
#include "common/common_types.h"
@@ -24,9 +28,19 @@ public:
void Reset() noexcept {
std::ranges::fill(pages, 0ULL);
+ recent.fill({~u64{0}, 0});
}
void Track(u64 offset, u64 size) noexcept {
+ // Tracking is idempotent until Reset: skip ranges tracked just before. Draws bind the
+ // same (often multi-megabyte) storage buffers again and again, and each Track wrote a
+ // word per 4 KiB of the range.
+ for (const auto& [recent_offset, recent_size] : recent) {
+ if (recent_offset == offset && recent_size == size) {
+ return;
+ }
+ }
+ recent[recent_cursor++ % recent.size()] = {offset, size};
const size_t page = offset >> BUFFER_PAGE_SHIFT;
const size_t page_end = (offset + size) >> BUFFER_PAGE_SHIFT;
if (page_end < page || page_end >= pages.size()) {
@@ -83,6 +97,9 @@ private:
private:
std::vector<u64> pages;
+ /// Ranges tracked last, (offset, size).
+ std::array<std::pair<u64, u64>, 4> recent{{{~u64{0}, 0}, {~u64{0}, 0}, {~u64{0}, 0}, {~u64{0}, 0}}};
+ size_t recent_cursor{};
};
} // namespace VideoCommon
diff --git a/src/video_core/control/channel_state.cpp b/src/video_core/control/channel_state.cpp
index 12a452dfc0..8e0b6c1b9e 100644
--- a/src/video_core/control/channel_state.cpp
+++ b/src/video_core/control/channel_state.cpp
@@ -13,7 +13,9 @@
#include "video_core/engines/maxwell_3d.h"
#include "video_core/engines/maxwell_dma.h"
#include "video_core/engines/puller.h"
+#include "video_core/macro.h"
#include "video_core/memory_manager.h"
+#include "video_core/nx2.h"
namespace Tegra::Control {
namespace {
@@ -21,15 +23,30 @@ namespace {
// Match NVK/Nouveau's initial pushbuffer subchannel layout.
constexpr u32 Nvk3DSubchannel = 0;
constexpr u32 NvkComputeSubchannel = 1;
+constexpr u32 NvkM2mfSubchannel = 2;
constexpr u32 Nvk2DSubchannel = 3;
constexpr u32 NvkCopySubchannel = 4;
+// Switch 2 channels of a process share one TSG, whose graphics context holds the macro RAM.
+std::shared_ptr<NativeMacroContext> Nx2MacroContext() {
+ static const auto context = [] {
+ auto created = std::make_shared<NativeMacroContext>();
+ created->engine.native_turing = true;
+ created->compute_engine.native_turing = true;
+ return created;
+ }();
+ return context;
+}
+
void BindNvkDefaultSubchannels(ChannelState::Payload& payload) {
auto& dma_pusher = payload.dma_pusher;
dma_pusher.BindSubchannel(&payload.maxwell_3d, Nvk3DSubchannel, Engines::EngineTypes::Maxwell3D);
dma_pusher.BindSubchannel(&payload.kepler_compute, NvkComputeSubchannel,
Engines::EngineTypes::KeplerCompute);
- // Subchannel 2 is M2MF there; Eden does not expose a 0x9039 engine yet.
+ // Subchannel 2 is M2MF/inline-to-memory there (NVN uses it without binding a class);
+ // KeplerMemory has the same LINE_LENGTH_IN/OFFSET_OUT/LAUNCH_DMA/LOAD_INLINE_DATA layout.
+ dma_pusher.BindSubchannel(&payload.kepler_memory, NvkM2mfSubchannel,
+ Engines::EngineTypes::KeplerMemory);
dma_pusher.BindSubchannel(&payload.fermi_2d, Nvk2DSubchannel, Engines::EngineTypes::Fermi2D);
dma_pusher.BindSubchannel(&payload.maxwell_dma, NvkCopySubchannel,
Engines::EngineTypes::MaxwellDMA);
@@ -56,6 +73,17 @@ void ChannelState::Init(Core::System& system, u64 program_id_) {
program_id = program_id_;
payload.emplace(system, *memory_manager, *this);
BindNvkDefaultSubchannels(*payload);
+ if (g_nx2_ampere) {
+ // Switch 2 command streams use Ampere classes with Turing+ (MME64) macros, and its
+ // block-linear surfaces the console's GOB swizzle.
+ memory_manager->EnableNativeTiling();
+ const auto macros = Nx2MacroContext();
+ payload->maxwell_3d.SetNativeMacroContext(macros);
+ payload->maxwell_3d.SetNativeTuringMacroMode(true);
+ payload->kepler_compute.SetNativeMacroMode(&macros->engine, macros->positions,
+ macros.get());
+ payload->kepler_compute.SetNativeDataRam(&payload->maxwell_3d.GetNativeDataRam());
+ }
initialized = true;
}
diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp
index 8af13e415c..24f1db1f7c 100644
--- a/src/video_core/control/scheduler.cpp
+++ b/src/video_core/control/scheduler.cpp
@@ -4,7 +4,9 @@
// SPDX-FileCopyrightText: 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
+#include <algorithm>
#include <memory>
+#include <vector>
#include "common/assert.h"
#include "video_core/control/channel_state.h"
@@ -26,6 +28,36 @@ void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) {
// Multiple channels can prepare their commands in parallel.
channel_state->payload->dma_pusher.Push(std::move(entries));
channel_state->payload->dma_pusher.DispatchCalls();
+ if (channel_state->payload->dma_pusher.HasPendingAcquire() &&
+ std::find(pending_channels.begin(), pending_channels.end(), channel_state) ==
+ pending_channels.end()) {
+ pending_channels.push_back(channel_state);
+ }
+ if (!pending_channels.empty()) {
+ ResumePending(gpu);
+ }
+}
+
+void Scheduler::ResumePending(GPU& gpu) {
+ // Running one channel can release the semaphore another waits for: retry until none moves.
+ for (bool progress = true; progress;) {
+ progress = false;
+ for (const auto& state : pending_channels) {
+ auto& dma_pusher = state->payload->dma_pusher;
+ if (!dma_pusher.HasPendingAcquire() || !dma_pusher.CanResumeAcquire()) {
+ continue;
+ }
+ {
+ std::scoped_lock lk{scheduling_guard};
+ gpu.BindChannel(state->bind_id);
+ }
+ dma_pusher.DispatchCalls();
+ progress = true;
+ }
+ }
+ std::erase_if(pending_channels, [](const auto& state) {
+ return !state->payload->dma_pusher.HasPendingAcquire();
+ });
}
void Scheduler::DeclareChannel(std::shared_ptr<ChannelState> new_channel) {
diff --git a/src/video_core/control/scheduler.h b/src/video_core/control/scheduler.h
index 342bea5025..3a3dcc58cf 100644
--- a/src/video_core/control/scheduler.h
+++ b/src/video_core/control/scheduler.h
@@ -8,6 +8,7 @@
#include <memory>
#include <mutex>
+#include <vector>
#include "common/container/unordered_map.h"
#include "video_core/dma_pusher.h"
@@ -24,11 +25,18 @@ class Scheduler {
public:
void Push(GPU& gpu, s32 channel, CommandList&& entries);
+ /// Runs channels whose pending semaphore acquire is now met.
+ void ResumePending(GPU& gpu);
+ [[nodiscard]] bool HasPendingChannels() const {
+ return !pending_channels.empty();
+ }
+
void DeclareChannel(std::shared_ptr<ChannelState> new_channel);
private:
::Common::unordered_map<s32, std::shared_ptr<ChannelState>> channels;
std::mutex scheduling_guard;
+ std::vector<std::shared_ptr<ChannelState>> pending_channels; ///< Suspended on an acquire
};
} // namespace Control
diff --git a/src/video_core/dirty_flags.cpp b/src/video_core/dirty_flags.cpp
index c2ecc12f56..e6856aaca3 100644
--- a/src/video_core/dirty_flags.cpp
+++ b/src/video_core/dirty_flags.cpp
@@ -22,6 +22,7 @@ void SetupDirtyVertexBuffers(Maxwell3D::DirtyState::Tables& tables) {
OFF(vertex_stream_limits) + i * NUM(vertex_stream_limits[0]);
FillBlock(tables, array_offset, num_array, VertexBuffer0 + i, VertexBuffers);
+ FillBlock(tables, 0x600 / sizeof(u32) + i * 2, 2, VertexBuffer0 + i, VertexBuffers);
FillBlock(tables, limit_offset, NUM(vertex_stream_limits), VertexBuffer0 + i,
VertexBuffers);
}
diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp
index 2c0ba007ef..3cc5afe967 100644
--- a/src/video_core/dma_pusher.cpp
+++ b/src/video_core/dma_pusher.cpp
@@ -4,13 +4,21 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <algorithm>
+#include <atomic>
+#include <cstdlib>
+#include <cstdio>
+
#include "common/settings.h"
#include "core/core.h"
+#include "core/memory.h"
#include "video_core/dma_pusher.h"
+#include "video_core/control/channel_state.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/gpu.h"
#include "video_core/guest_memory.h"
#include "video_core/memory_manager.h"
+#include "video_core/nx2.h"
#include "video_core/rasterizer_interface.h"
namespace Tegra {
@@ -29,7 +37,7 @@ DmaPusher::DmaPusher(Core::System& system_, MemoryManager& memory_manager_, Cont
DmaPusher::~DmaPusher() = default;
void DmaPusher::DispatchCalls() {
- dma_pushbuffer_subindex = 0;
+ // The sub-index is kept: a channel suspended on a semaphore resumes inside its command list.
dma_state.is_last_call = true;
while (system.IsPoweredOn()) {
if (!Step()) {
@@ -44,6 +52,9 @@ bool DmaPusher::Step() {
if (!ib_enable || dma_pushbuffer.empty()) {
return false;
}
+ if (!puller.ResumeAcquire(*this)) {
+ return false;
+ }
CommandList& command_list = dma_pushbuffer.front();
@@ -57,7 +68,13 @@ bool DmaPusher::Step() {
}
if (prefetch_size > 0) {
- ProcessCommands(command_list.prefetch_command_list);
+ const std::span<const CommandHeader> commands{command_list.prefetch_command_list};
+ command_resume_offset +=
+ ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset);
+ if (puller.HasPendingAcquire()) {
+ return false;
+ }
+ command_resume_offset = 0;
dma_pushbuffer.pop();
return true;
}
@@ -83,17 +100,74 @@ bool DmaPusher::Step() {
subchannels[dma_state.subchannel]->current_dirty = memory_manager.IsMemoryDirty(dma_state.dma_get, words * sizeof(u32));
}
}
+ if (command_resume_offset == 0) {
+ // A method's data continues across GP entries, whatever their level: title 4 ends a
+ // main entry after a macro's first word and supplies the remaining parameters in a
+ // 5-word subroutine entry. EDEN_NX2_PARK_SPLIT_METHODS restores the old guess (an
+ // entry elsewhere is a call that starts clean; resume where the split one ended).
+ static const bool park_split_methods = std::getenv("EDEN_NX2_PARK_SPLIT_METHODS") != nullptr;
+ if (g_nx2_ampere) {
+ if (static u32 split_logs{}; dma_state.method_count != 0 && split_logs < 64) {
+ ++split_logs;
+ LOG_DEBUG(HW_GPU, "Split method {:#x} ({} left) on ch {}: next entry {:#x} size {} raw {:#x} (level bit {}, priv bit {}), expected {:#x}, parked {}",
+ dma_state.method, dma_state.method_count, channel_state.bind_id,
+ u64{header.addr}, u64{header.size}, header.raw, u64{header.is_push_buffer},
+ u64{header.allow_flush}, split_method_end, parked_methods.size());
+ std::array<u32, 8> next_words{};
+ std::array<u32, 4> prev_words{};
+ memory_manager.ReadBlock(header.addr, next_words.data(),
+ std::min<size_t>(header.size, 8) * sizeof(u32));
+ memory_manager.ReadBlock(split_method_end - sizeof(prev_words), prev_words.data(),
+ sizeof(prev_words));
+ LOG_DEBUG(HW_GPU, " previous entry ends {:08x}; next entry starts {:08x}",
+ fmt::join(prev_words, " "), fmt::join(next_words, " "));
+ }
+ if (park_split_methods) {
+ ResolveSplitMethod(header.addr);
+ }
+ }
+ if (dma_pushbuffer_subindex < command_list.submit_hashes.size()) {
+ CheckSubmitHash(header, command_list.submit_hashes[dma_pushbuffer_subindex]);
+ }
+ }
+ // A resumed entry is read again: the words after the acquire may be new by now.
const bool use_safe = Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe();
if (use_safe) {
Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::SafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers);
- ProcessCommands(headers);
+ const std::span<const CommandHeader> commands{headers.data(), headers.size()};
+ RecordParsedEntry(header, commands);
+ command_resume_offset +=
+ ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset);
} else {
Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::UnsafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers);
- ProcessCommands(headers);
+ const std::span<const CommandHeader> commands{headers.data(), headers.size()};
+ RecordParsedEntry(header, commands);
+ command_resume_offset +=
+ ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset);
+ }
+ if (puller.HasPendingAcquire()) {
+ return false;
+ }
+ command_resume_offset = 0;
+ if (dma_state.method_count != 0) {
+ split_method_end = header.addr + header.size * sizeof(u32);
}
}
if (++dma_pushbuffer_subindex >= command_list_size) {
+ // A split method resumes within the submission that split it; ring addresses repeat
+ // every frame, so a stale one could otherwise swallow the headers of a later entry.
+ for (const auto& parked : parked_methods) {
+ static std::atomic<u32> reported{};
+ if (reported.fetch_add(1) < 16) {
+ LOG_WARNING(HW_GPU,
+ "Submission ended with parked method {:#x} (sub {}, {} words left, "
+ "resume {:#x})",
+ parked.method, parked.subchannel, parked.method_count,
+ parked.resume_address);
+ }
+ }
+ parked_methods.clear();
dma_pushbuffer.pop();
dma_pushbuffer_subindex = 0;
} else {
@@ -111,20 +185,23 @@ bool DmaPusher::Step() {
return true;
}
-void DmaPusher::ProcessCommands(std::span<const CommandHeader> commands) {
+size_t DmaPusher::ProcessCommands(std::span<const CommandHeader> commands, size_t word_offset) {
for (size_t index = 0; index < commands.size();) {
// Data word of methods command
if (dma_state.method_count && dma_state.non_incrementing) {
auto const& command_header = commands[index]; //must ref (MUltiMethod re)
- dma_state.dma_word_offset = u32(index * sizeof(u32));
- const u32 max_write = u32(std::min<std::size_t>(index + dma_state.method_count, commands.size()) - index);
+ dma_state.dma_word_offset = u64((word_offset + index) * sizeof(u32));
+ u32 max_write = u32(std::min<std::size_t>(index + dma_state.method_count, commands.size()) - index);
+ if (g_nx2_ampere && dma_state.method < non_puller_methods) {
+ max_write = 1; // a semaphore acquire may suspend the channel after any word
+ }
CallMultiMethod(&command_header.argument, max_write);
dma_state.method_count -= max_write;
dma_state.is_last_call = true;
index += max_write;
} else if (dma_state.method_count) {
auto const command_header = commands[index]; //can copy
- dma_state.dma_word_offset = u32(index * sizeof(u32));
+ dma_state.dma_word_offset = u64((word_offset + index) * sizeof(u32));
dma_state.is_last_call = dma_state.method_count <= 1;
CallMethod(command_header.argument);
dma_state.method += !dma_state.non_incrementing ? 1 : 0;
@@ -163,6 +240,54 @@ void DmaPusher::ProcessCommands(std::span<const CommandHeader> commands) {
}
index++;
}
+ if (puller.HasPendingAcquire()) {
+ return index;
+ }
+ }
+ return commands.size();
+}
+
+// NVN on Switch 2 ends GPFIFO entries in the middle of a method's data and calls another command
+// buffer (e.g. a compute dispatch) before the entry that carries the rest, which starts right
+// where the first one ended. An entry starting anywhere else is such a call: run it with a clean
+// method state and resume the unfinished method when the stream continues at that address.
+void DmaPusher::ResolveSplitMethod(GPUVAddr entry_address) {
+ const auto continued = std::ranges::find(parked_methods, entry_address,
+ &ParkedMethod::resume_address);
+ std::optional<ParkedMethod> resume;
+ if (continued != parked_methods.end()) {
+ resume = *continued;
+ parked_methods.erase(continued);
+ }
+ if (dma_state.method_count != 0 && entry_address != split_method_end) {
+ if (parked_methods.size() == MaxParkedMethods) {
+ static std::atomic<u32> reported{};
+ if (reported.fetch_add(1) < 8) {
+ const auto& oldest = parked_methods.front();
+ LOG_WARNING(HW_GPU,
+ "Dropping parked method {:#x} (sub {}, {} words left, resume {:#x})",
+ oldest.method, oldest.subchannel, oldest.method_count,
+ oldest.resume_address);
+ }
+ parked_methods.erase(parked_methods.begin());
+ }
+ parked_methods.push_back(ParkedMethod{
+ .method = dma_state.method,
+ .subchannel = dma_state.subchannel,
+ .method_count = dma_state.method_count,
+ .non_incrementing = dma_state.non_incrementing,
+ .increment_once = dma_increment_once,
+ .resume_address = split_method_end,
+ });
+ dma_state.method_count = 0;
+ dma_increment_once = false;
+ }
+ if (resume && dma_state.method_count == 0) {
+ dma_state.method = resume->method;
+ dma_state.subchannel = resume->subchannel;
+ dma_state.method_count = resume->method_count;
+ dma_state.non_incrementing = resume->non_incrementing;
+ dma_increment_once = resume->increment_once;
}
}
@@ -172,7 +297,28 @@ void DmaPusher::SetState(const CommandHeader& command_header) {
dma_state.method_count = command_header.method_count;
}
+// EDEN_NX2_SEM_WATCH=<gpu address>: log every method whose argument is the low word of an
+// address in the 16 bytes there (finds the engine and method that release a semaphore).
+static const u32 g_semaphore_watch = [] {
+ const char* value = std::getenv("EDEN_NX2_SEM_WATCH");
+ return value ? static_cast<u32>(std::strtoull(value, nullptr, 16)) : 0U;
+}();
+
+// EDEN_NX2_PB_CHECK also keeps the last parsed entries, for DumpCurrentEntry.
+static const bool g_record_parsed_entries = std::getenv("EDEN_NX2_PB_CHECK") != nullptr;
+
+void DmaPusher::WatchArgument(u32 method, u32 argument) const {
+ const u32 watch = g_semaphore_watch;
+ if ((argument & ~0xfU) == watch) {
+ LOG_INFO(HW_GPU, "Semaphore watch: channel {} subchannel {} method {:#x} argument {:#x}",
+ channel_state.bind_id, dma_state.subchannel, method, argument);
+ }
+}
+
void DmaPusher::CallMethod(u32 argument) {
+ if (g_semaphore_watch != 0) [[unlikely]] {
+ WatchArgument(dma_state.method, argument);
+ }
if (dma_state.method < non_puller_methods) {
puller.CallPullerMethod(*this, Engines::Puller::MethodCall{
dma_state.method,
@@ -182,6 +328,10 @@ void DmaPusher::CallMethod(u32 argument) {
});
} else {
auto subchannel = subchannels[dma_state.subchannel];
+ if (subchannel == nullptr) {
+ ReportUnboundSubchannel();
+ return;
+ }
if (!subchannel->execution_mask[dma_state.method]) {
subchannel->method_sink.emplace_back(dma_state.method, argument);
} else {
@@ -193,16 +343,128 @@ void DmaPusher::CallMethod(u32 argument) {
}
void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) {
+ if (g_semaphore_watch != 0) [[unlikely]] {
+ for (u32 i = 0; i < num_methods; ++i) {
+ WatchArgument(dma_state.method + (dma_state.non_incrementing ? 0 : i), base_start[i]);
+ }
+ }
if (dma_state.method < non_puller_methods) {
puller.CallMultiMethod(*this, dma_state.method, dma_state.subchannel, base_start, num_methods, dma_state.method_count);
} else {
auto subchannel = subchannels[dma_state.subchannel];
+ if (subchannel == nullptr) {
+ ReportUnboundSubchannel();
+ return;
+ }
subchannel->ConsumeSink(system);
subchannel->current_dma_segment = dma_state.dma_get + dma_state.dma_word_offset;
subchannel->CallMultiMethod(system, dma_state.method, base_start, num_methods, dma_state.method_count);
}
}
+void DmaPusher::ReportUnboundSubchannel() {
+ // Methods sent to a subchannel with no engine (no SetObject yet) are dropped.
+ static std::atomic<u32> reported{};
+ if (reported.fetch_add(1) < 8) {
+ LOG_WARNING(HW_GPU, "Dropping method {:#x} on unbound subchannel {}", dma_state.method,
+ dma_state.subchannel);
+ }
+}
+
+void DmaPusher::CheckSubmitHash(const CommandListHeader& header, u64 submit_hash) {
+ std::vector<u32> words(header.size);
+ memory_manager.ReadBlockUnsafe(header.addr, words.data(), words.size() * sizeof(u32));
+ static std::atomic<u64> checked{};
+ static std::atomic<u64> changed{};
+ ++checked;
+ if (HashPushbufferWords(words) != submit_hash) {
+ if (changed.fetch_add(1) < 16) {
+ LOG_WARNING(HW_GPU, "pushbuffer entry {:#x}+{:#x} changed between submit and execution",
+ header.addr.Value(), header.size.Value());
+ }
+ }
+ if ((checked % 20000) == 0) {
+ LOG_WARNING(HW_GPU, "pushbuffer check: {} of {} entries changed after submit",
+ changed.load(), checked.load());
+ }
+}
+
+void DmaPusher::RecordParsedEntry(const CommandListHeader& header,
+ std::span<const CommandHeader> commands) {
+ if (!g_record_parsed_entries) {
+ return;
+ }
+ u64 hash = 0xcbf29ce484222325ULL;
+ for (const CommandHeader& command : commands) {
+ hash = (hash ^ command.argument) * 0x100000001b3ULL;
+ }
+ parsed_entries[parsed_entry_count++ % parsed_entries.size()] = {header.addr, static_cast<u32>(header.size), hash};
+}
+
+void DmaPusher::DumpCurrentEntry(const char* reason) {
+ // Entries whose memory changed after they were parsed (the CPU rewrote them meanwhile).
+ for (size_t i = 0; i < std::min(parsed_entry_count, parsed_entries.size()); ++i) {
+ const auto& entry = parsed_entries[(parsed_entry_count - 1 - i) % parsed_entries.size()];
+ std::vector<u32> words(entry.size);
+ memory_manager.ReadBlockUnsafe(entry.address, words.data(), words.size() * sizeof(u32));
+ u64 hash = 0xcbf29ce484222325ULL;
+ for (const u32 word : words) {
+ hash = (hash ^ word) * 0x100000001b3ULL;
+ }
+ LOG_WARNING(HW_GPU, " parsed -{}: {:#x}+{:#x} {}", i, entry.address, entry.size,
+ hash == entry.hash ? "unchanged" : "CHANGED since parsed");
+ }
+ // EDEN_NX2_PB_DUMP=<file>: also write the whole command list (u64 address, u32 size and the
+ // current words of every entry, then the current entry index) for offline replay.
+ if (static const char* const path = std::getenv("EDEN_NX2_PB_DUMP");
+ path != nullptr && !dma_pushbuffer.empty()) {
+ if (std::FILE* file = std::fopen(path, "wb")) {
+ const auto& lists = dma_pushbuffer.front().command_lists;
+ const u64 current = dma_pushbuffer_subindex;
+ const u64 count = lists.size();
+ std::fwrite(&count, sizeof(count), 1, file);
+ std::fwrite(&current, sizeof(current), 1, file);
+ for (const CommandListHeader& header : lists) {
+ const u64 address = header.addr;
+ const u32 size = static_cast<u32>(header.size);
+ std::vector<u32> words(size);
+ memory_manager.ReadBlockUnsafe(address, words.data(), words.size() * sizeof(u32));
+ std::fwrite(&address, sizeof(address), 1, file);
+ std::fwrite(&size, sizeof(size), 1, file);
+ std::fwrite(words.data(), sizeof(u32), words.size(), file);
+ }
+ std::fclose(file);
+ }
+ }
+ LOG_WARNING(HW_GPU, "pushbuffer dump ({}): method={:#x} sub={} count={} non_inc={} get={:#x}",
+ reason, dma_state.method, dma_state.subchannel, dma_state.method_count,
+ dma_state.non_incrementing, dma_state.dma_get);
+ if (dma_pushbuffer.empty()) {
+ return;
+ }
+ const auto& lists = dma_pushbuffer.front().command_lists;
+ const size_t first = dma_pushbuffer_subindex >= 2 ? dma_pushbuffer_subindex - 2 : 0;
+ for (size_t i = first; i <= dma_pushbuffer_subindex && i < lists.size(); ++i) {
+ const CommandListHeader& header = lists[i];
+ LOG_WARNING(HW_GPU, " entry {} of {}: addr={:#x} size={:#x}", i, lists.size(),
+ header.addr.Value(), header.size.Value());
+ for (u64 word = 0; word < std::min<u64>(header.size, 0x80); word += 8) {
+ std::string gpu_line;
+ std::string cpu_line;
+ for (u64 j = word; j < std::min<u64>(header.size, word + 8); ++j) {
+ const GPUVAddr address = header.addr + j * sizeof(u32);
+ const u32 gpu_value = memory_manager.Read<u32>(address);
+ const u32 cpu_value = system.ApplicationMemory().Read32(address);
+ gpu_line += fmt::format(" {:08x}", gpu_value);
+ cpu_line += gpu_value == cpu_value ? std::string(" ........")
+ : fmt::format(" {:08x}", cpu_value);
+ }
+ LOG_WARNING(HW_GPU, " +{:03x}:{}", word * 4, gpu_line);
+ LOG_WARNING(HW_GPU, " cpu :{}", cpu_line);
+ }
+ }
+}
+
void DmaPusher::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) {
rasterizer = rasterizer_;
}
diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h
index 62af628511..bbcc345c84 100644
--- a/src/video_core/dma_pusher.h
+++ b/src/video_core/dma_pusher.h
@@ -8,6 +8,7 @@
#include <array>
#include <condition_variable>
+#include <optional>
#include <span>
#include <vector>
#include <boost/container/small_vector.hpp>
@@ -114,8 +115,19 @@ struct CommandList final {
boost::container::small_vector<CommandListHeader, 512> command_lists;
boost::container::small_vector<CommandHeader, 512> prefetch_command_list;
+ /// EDEN_NX2_PB_CHECK: hash of each entry's words when it was submitted.
+ std::vector<u64> submit_hashes;
};
+/// FNV-1a over pushbuffer words (EDEN_NX2_PB_CHECK).
+inline u64 HashPushbufferWords(std::span<const u32> words) {
+ u64 hash = 0xcbf29ce484222325ULL;
+ for (const u32 word : words) {
+ hash = (hash ^ word) * 0x100000001b3ULL;
+ }
+ return hash;
+}
+
/// @brief The DmaPusher class implements DMA submission to FIFOs, providing an area of memory that the
/// emulated app fills with commands and tells PFIFO to process. The pushbuffers are then assembled
/// into a "command stream" consisting of 32-bit words that make up "commands".
@@ -132,6 +144,14 @@ public:
void DispatchCalls();
+ /// Switch 2: true while the channel is suspended on a semaphore acquire.
+ [[nodiscard]] bool HasPendingAcquire() const {
+ return puller.HasPendingAcquire();
+ }
+ [[nodiscard]] bool CanResumeAcquire() {
+ return puller.CanResumeAcquire(*this);
+ }
+
void BindSubchannel(Engines::EngineInterface* engine, u32 subchannel_id, Engines::EngineTypes engine_type) {
subchannels[subchannel_id] = engine;
subchannel_type[subchannel_id] = engine_type;
@@ -143,12 +163,26 @@ private:
static constexpr u32 non_puller_methods = 0x40;
static constexpr u32 max_subchannels = 8;
bool Step();
- void ProcessCommands(std::span<const CommandHeader> commands);
+ /// Returns how many words were processed: fewer than given when a semaphore acquire
+ /// suspended the channel.
+ size_t ProcessCommands(std::span<const CommandHeader> commands, size_t word_offset = 0);
+ size_t command_resume_offset{}; ///< Words of the current entry done before a suspension
void SetState(const CommandHeader& command_header);
void CallMethod(u32 argument);
void CallMultiMethod(const u32* base_start, u32 num_methods);
+ void ReportUnboundSubchannel();
+ void WatchArgument(u32 method, u32 argument) const;
+
+public:
+ /// Logs the GPFIFO entry being processed (and the one before it), read through the GPU MMU
+ /// and straight from guest memory, to find pushbuffer desyncs. Debug aid for Switch 2.
+ void DumpCurrentEntry(const char* reason);
+ void CheckSubmitHash(const CommandListHeader& header, u64 submit_hash);
+ void RecordParsedEntry(const CommandListHeader& header, std::span<const CommandHeader> commands);
+
+private:
public:
Common::ScratchBuffer<CommandHeader> command_headers; ///< Buffer for list of commands fetched at once
@@ -172,6 +206,31 @@ public:
Control::ChannelState& channel_state;
DmaState dma_state{};
+
+ // Switch 2: methods whose data continues in a later GPFIFO entry, set aside while other
+ // command buffers run (NVN interleaves several, so more than one can be pending).
+ struct ParkedMethod {
+ u32 method;
+ u32 subchannel;
+ u32 method_count;
+ bool non_incrementing;
+ bool increment_once;
+ GPUVAddr resume_address; ///< Where the entry that continues the method starts
+ };
+ void ResolveSplitMethod(GPUVAddr entry_address);
+ static constexpr size_t MaxParkedMethods = 16;
+
+ // Debug aid: hashes of the words of the last entries as they were parsed.
+ struct ParsedEntry {
+ GPUVAddr address;
+ u32 size;
+ u64 hash;
+ };
+ std::array<ParsedEntry, 64> parsed_entries{};
+ size_t parsed_entry_count{};
+ std::vector<ParkedMethod> parked_methods;
+ GPUVAddr split_method_end{}; ///< End of the last entry that ended inside a method
+
std::array<Engines::EngineInterface*, max_subchannels> subchannels{};
std::array<Engines::EngineTypes, max_subchannels> subchannel_type;
diff --git a/src/video_core/engines/engine_upload.cpp b/src/video_core/engines/engine_upload.cpp
index ba21eec4c4..0ffdae8036 100644
--- a/src/video_core/engines/engine_upload.cpp
+++ b/src/video_core/engines/engine_upload.cpp
@@ -27,6 +27,7 @@ void State::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) {
void State::ProcessExec(const bool is_linear_) {
write_offset = 0;
+ flushed_offset = 0;
copy_size = regs.line_length_in * regs.line_count;
inner_buffer.resize_destructive(copy_size);
is_linear = is_linear_;
@@ -39,10 +40,29 @@ void State::ProcessData(const u32 data, const bool is_last_call) {
if (!is_last_call) {
return;
}
+ if (native_partial && is_linear && regs.line_count == 1) {
+ const u32 count = write_offset - flushed_offset;
+ if (count) {
+ rasterizer->AccelerateInlineToMemory(regs.dest.Address() + flushed_offset, count,
+ std::span<const u8>{inner_buffer.data() + flushed_offset, count});
+ flushed_offset = write_offset;
+ }
+ return;
+ }
ProcessData(inner_buffer);
}
void State::ProcessData(const u32* data, size_t num_data) {
+ if (native_partial && is_linear && regs.line_count == 1) {
+ const auto count = std::min<size_t>(num_data * sizeof(u32), copy_size - write_offset);
+ if (count) {
+ rasterizer->AccelerateInlineToMemory(regs.dest.Address() + write_offset, count,
+ std::span<const u8>{reinterpret_cast<const u8*>(data), count});
+ write_offset += static_cast<u32>(count);
+ flushed_offset = write_offset;
+ }
+ return;
+ }
std::span<const u8> read_buffer(reinterpret_cast<const u8*>(data), num_data * sizeof(u32));
ProcessData(read_buffer);
}
diff --git a/src/video_core/engines/engine_upload.h b/src/video_core/engines/engine_upload.h
index 21bf8aeb46..55a82f938b 100644
--- a/src/video_core/engines/engine_upload.h
+++ b/src/video_core/engines/engine_upload.h
@@ -62,6 +62,7 @@ public:
explicit State(MemoryManager& memory_manager_, Registers& regs_);
~State();
+ void SetNativePartialTransfers(bool enabled) { native_partial = enabled; }
void ProcessExec(bool is_linear_);
void ProcessData(u32 data, bool is_last_call);
void ProcessData(const u32* data, size_t num_data);
@@ -80,6 +81,8 @@ public:
private:
void ProcessData(std::span<const u8> read_buffer);
+ bool native_partial{};
+ u32 flushed_offset{};
u32 write_offset = 0;
u32 copy_size = 0;
Common::ScratchBuffer<u8> inner_buffer;
diff --git a/src/video_core/engines/fermi_2d.cpp b/src/video_core/engines/fermi_2d.cpp
index aeff5e156f..35b2c69801 100644
--- a/src/video_core/engines/fermi_2d.cpp
+++ b/src/video_core/engines/fermi_2d.cpp
@@ -4,6 +4,7 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <cstdlib>
#include "common/assert.h"
#include "common/logging.h"
#include "video_core/engines/fermi_2d.h"
@@ -59,6 +60,27 @@ void Fermi2D::ConsumeSinkImpl(Core::System& system) {
}
void Fermi2D::Blit() {
+ if (static const u64 trace_destination = [] {
+ const char* value = std::getenv("EDEN_NX2_TRACE_DST");
+ return value != nullptr ? std::strtoull(value, nullptr, 16) : 0ULL;
+ }();
+ trace_destination != 0 && regs.dst.Address() >= trace_destination &&
+ regs.dst.Address() < trace_destination + (32ULL << 20)) {
+ static u32 traced_destination{};
+ if (traced_destination++ < 40) {
+ LOG_INFO(HW_GPU, "nx2 2D blit to traced dst: {:#x} {}x{} -> {:#x} {}x{}",
+ regs.src.Address(), regs.src.width, regs.src.height, regs.dst.Address(),
+ regs.dst.width, regs.dst.height);
+ }
+ }
+ if (static const bool trace = std::getenv("EDEN_NX2_RT_TRACE") != nullptr; trace) {
+ static u32 traced{};
+ if (traced++ < 64) {
+ LOG_INFO(HW_GPU, "nx2 2D blit {:#x} {}x{} -> {:#x} {}x{}", regs.src.Address(),
+ regs.src.width, regs.src.height, regs.dst.Address(), regs.dst.width,
+ regs.dst.height);
+ }
+ }
LOG_DEBUG(HW_GPU, "called. source address={:#x}, destination address={:#x}", regs.src.Address(), regs.dst.Address());
UNIMPLEMENTED_IF_MSG(regs.operation != Operation::SrcCopy, "Operation is not copy");
diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp
index 96a12539a2..7792bc9b5d 100644
--- a/src/video_core/engines/kepler_compute.cpp
+++ b/src/video_core/engines/kepler_compute.cpp
@@ -4,13 +4,25 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <algorithm>
+#include <atomic>
#include <bitset>
+#include <cstdlib>
+#include <fstream>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
#include "common/assert.h"
#include "common/logging.h"
+#include "shader_recompiler/exception.h"
#include "core/core.h"
+#include "video_core/nx2.h"
+#include "video_core/engines/fermi_2d.h"
#include "video_core/engines/kepler_compute.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/memory_manager.h"
+#include "video_core/native_query_trace.h"
#include "video_core/rasterizer_interface.h"
#include "video_core/textures/decoders.h"
@@ -20,6 +32,13 @@ KeplerCompute::KeplerCompute(MemoryManager& memory_manager_)
: memory_manager{memory_manager_}
, upload_state{memory_manager, regs.upload}
{
+ // EDEN_NX2_QMD_TRACE: log the QMD scheduler's loads, dependency counts and launches.
+ if (std::getenv("EDEN_NX2_QMD_TRACE")) {
+ native_qmd.trace = [this](std::string_view event, u64 address, s32 value) {
+ LOG_INFO(HW_GPU, "QMD trace {} {:#x} {} (engine {} as {})", event, address, value,
+ fmt::ptr(this), memory_manager.GetID());
+ };
+ }
execution_mask.reset();
execution_mask[KEPLER_COMPUTE_REG_INDEX(exec_upload)] = true;
execution_mask[KEPLER_COMPUTE_REG_INDEX(data_upload)] = true;
@@ -35,13 +54,347 @@ void KeplerCompute::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_)
void KeplerCompute::ConsumeSinkImpl(Core::System& system) {
for (auto [method, value] : method_sink) {
- regs.reg_array[method] = value;
+ if (method < Regs::NUM_REGS) {
+ regs.reg_array[method] = value;
+ }
}
method_sink.clear();
}
+// SET_REPORT_SEMAPHORE_A-D (0x6c0-0x6c3): address high/low, payload, then operation in bits 1:0
+// (release / acquire / report only / trap), reduction enable bit 3 with the op in bits 11:9, and
+// a one-word (no timestamp) structure in bit 28. NVN2 releases queue syncs from compute this way.
+void KeplerCompute::ProcessNativeReportSemaphore(u32 argument) {
+ const GPUVAddr address = (GPUVAddr{native_registers[0x6c0] & 0x1ffffff} << 32) |
+ native_registers[0x6c1];
+ const u32 payload = native_registers[0x6c2];
+ const u32 operation = argument & 3;
+ const bool one_word = (argument >> 28) & 1;
+ static u32 logged{};
+ if (logged++ < 32) {
+ LOG_INFO(HW_GPU, "Native compute report semaphore {:#x} = {:#x}, control {:#x}", address,
+ payload, argument);
+ }
+ if (operation != 0 && operation != 2) {
+ LOG_WARNING(HW_GPU, "Native compute report semaphore operation {} not supported", operation);
+ return;
+ }
+ if ((argument >> 3) & 1) {
+ const u32 reduction = (argument >> 9) & 7;
+ rasterizer->SignalFence([this, address, payload, reduction] {
+ const u32 old_value = memory_manager.Read<u32>(address);
+ const u32 value = reduction == 0 ? old_value + payload
+ : reduction == 1 ? std::min(old_value, payload)
+ : reduction == 2 ? std::max(old_value, payload)
+ : reduction == 3 ? (old_value >= payload ? 0 : old_value + 1)
+ : reduction == 4 ? (old_value == 0 || old_value > payload ? payload : old_value - 1)
+ : reduction == 5 ? (old_value & payload)
+ : reduction == 6 ? (old_value | payload)
+ : (old_value ^ payload);
+ memory_manager.Write<u32>(address, value);
+ });
+ return;
+ }
+ rasterizer->Query(address, VideoCommon::QueryType::Payload,
+ one_word ? VideoCommon::QueryPropertiesFlags::IsAFence
+ : VideoCommon::QueryPropertiesFlags::HasTimeout,
+ payload, 0);
+}
+
+bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call) {
+ if (native_context) {
+ if (method == 0x45 || method == 0x47) native_compute_uploaded = true;
+ native_macro = native_compute_uploaded ? &native_context->compute_engine : &native_context->engine;
+ native_macro_positions = native_compute_uploaded ? std::span<u32>{native_context->compute_positions} : std::span<u32>{native_context->positions};
+ }
+ if (method >= native_registers.size()) {
+ LOG_ERROR(HW_GPU, "Native compute method out of range: {:#x}", method);
+ return true;
+ }
+ if (method >= 0xe00) {
+ if (!native_executing_macro) {
+ if (method & 1) { LOG_ERROR(HW_GPU, "Native compute macro starts on argument method"); return true; }
+ native_executing_macro = method;
+ }
+ native_parameters.push_back(argument);
+ if (!is_last_call) return true;
+ const u32 entry = ((*native_executing_macro - 0xe00) / 2) % native_macro_positions.size();
+ native_executing_macro_logged = native_executing_macro;
+ native_executing_macro.reset();
+ static const u32 watch = [] {
+ const char* value = std::getenv("EDEN_NX2_SEM_WATCH");
+ return value ? static_cast<u32>(std::strtoull(value, nullptr, 16)) : 0U;
+ }();
+ const bool watched = watch != 0 && std::ranges::any_of(native_parameters,
+ [](u32 parameter) { return (parameter & ~0xfU) == watch; });
+ if (watched) {
+ LOG_INFO(HW_GPU, "Semaphore watch: compute macro start={:#x} parameters={:08x}",
+ native_macro_positions[entry], fmt::join(native_parameters, ","));
+ }
+ // EDEN_NX2_MME_DUMP: write the compute macro code once per engine, and log each macro
+ // call with its entry and parameters (diagnostics).
+ if (static const bool mme_dump = std::getenv("EDEN_NX2_MME_DUMP") != nullptr; mme_dump) {
+ if (!native_mme_dumped && native_macro_positions[entry] == 0xda) {
+ native_mme_dumped = true;
+ std::ofstream dump(fmt::format("native-compute-mme-{}.bin", fmt::ptr(this)),
+ std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native_macro->native_code.data()),
+ sizeof(native_macro->native_code));
+ }
+ LOG_INFO(HW_GPU, "MME call engine {} method {:#x} start={:#x} parameters={:08x}",
+ fmt::ptr(this), *native_executing_macro_logged, native_macro_positions[entry],
+ fmt::join(native_parameters, ","));
+ }
+ if (native_data_ram) native_macro->native_interpreter.active_dram = native_data_ram;
+ const u32 dram0_before = (*native_macro->native_interpreter.active_dram)[0x27];
+ const u32 start = native_macro_positions[entry];
+ const auto emit = [&](u32 address, u32 value) {
+ static u32 diagnostic_writes{};
+ static const bool log_emits = std::getenv("EDEN_NX2_SEM_WATCH_EMITS") != nullptr;
+ if (watched && log_emits) LOG_INFO(HW_GPU, "Semaphore watch: compute macro emit {:#x} = {:#x}", address, value);
+ if (diagnostic_writes++ < 4096) LOG_DEBUG(HW_GPU, "Native compute MME emit method={:#x}, value={:#x}", address, value);
+ if (address >= 0xe00) return false;
+ CallMethod(system, address, value, true);
+ return true;
+ };
+ // Repeated calls replay their writes, as for 3D macros (macro_memo.h).
+ static const bool use_memo = [] {
+ const char* value = std::getenv("EDEN_NX2_MME_MEMO");
+ return value == nullptr || value[0] != '0';
+ }();
+ auto& memo = native_macro->native_memo;
+ if (use_memo && !watched) {
+ memo.Synchronize(native_macro->native_code_generation);
+ if (std::remove_reference_t<decltype(memo)>::Effects effects;
+ memo.Find(start, native_parameters, *native_macro->native_interpreter.active_dram,
+ effects)) {
+ auto& data_ram = *native_macro->native_interpreter.active_dram;
+ for (size_t i = 0; i < effects.data_ram_writes.size(); i += 2) {
+ data_ram[effects.data_ram_writes[i]] = effects.data_ram_writes[i + 1];
+ }
+ for (size_t i = 0; i < effects.writes.size(); i += 2) {
+ emit(effects.writes[i], effects.writes[i + 1]);
+ }
+ native_parameters.clear();
+ return true;
+ }
+ native_macro->native_writes.clear();
+ }
+ // The interpreter consumes native_parameters as its FIFO: keep the call's own parameters.
+ native_memo_parameters.assign(native_parameters.begin(), native_parameters.end());
+ const u64 code_generation = native_macro->native_code_generation;
+ const auto result = native_macro->native_interpreter.Execute(native_macro->native_code,
+ start, native_parameters,
+ [&](u32 address) { return address < native_registers.size() ? native_registers[address] : 0U; },
+ [&](u32 address, u32 value) {
+ if (use_memo) {
+ native_macro->native_writes.emplace_back(address, value);
+ }
+ return emit(address, value);
+ });
+ const auto& interpreter = native_macro->native_interpreter;
+ if (use_memo && !watched && result.success && !interpreter.read_registers &&
+ !interpreter.reread_data_ram && code_generation == native_macro->native_code_generation) {
+ memo.Store(start, native_memo_parameters, interpreter.data_ram_reads,
+ interpreter.data_ram_writes, native_macro->native_writes);
+ }
+ if (static const bool mme_dump = std::getenv("EDEN_NX2_MME_DUMP") != nullptr;
+ mme_dump && (*native_macro->native_interpreter.active_dram)[0x27] != dram0_before) {
+ LOG_INFO(HW_GPU, "MME data RAM[0x27] {:#x} -> {:#x} by engine {} macro start={:#x}",
+ dram0_before, (*native_macro->native_interpreter.active_dram)[0x27], fmt::ptr(this),
+ native_macro_positions[entry]);
+ }
+ if (!result.success) {
+ native_macro->native_execution_failed = true;
+ if (static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr;
+ diagnostics) {
+ std::ofstream dump("native-compute-mme.bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native_macro->native_code.data()), sizeof(native_macro->native_code));
+ }
+ LOG_ERROR(HW_GPU, "Native MME failed: compute start={:#x}, pc={:#x}, consumed={}/{}, error={}, parameters={:08x}",
+ native_macro_positions[entry], result.pc, result.consumed, native_parameters.size(), result.error,
+ fmt::join(native_parameters, ","));
+ }
+ native_parameters.clear();
+ return true;
+ }
+ native_registers[method] = argument;
+ if (method == 0x6c3) { // SET_REPORT_SEMAPHORE_D: the compute class's semaphore release
+ ProcessNativeReportSemaphore(argument);
+ return true;
+ }
+ if (method == 0x5a || (method >= 0xc6 && method <= 0x108)) {
+ VideoCommon::TraceNativeQuery(12, memory_manager.GetID(), method, 0, 0, argument, 0);
+ }
+ if (method == 0x107) {
+ const GPUVAddr address = ((GPUVAddr{native_registers[0xc6]} << 32) |
+ native_registers[0xc7]) << 8;
+ const GPUVAddr program = (GPUVAddr{native_registers[0xc8 + 49] & 0x1ffff} << 32) |
+ native_registers[0xc8 + 48];
+ const u32 control = native_registers[0xa7];
+ LOG_DEBUG(HW_GPU, "Native inline QMD: address={:#x}, program={:#x}, control={:#x}, grid={}/{}/{}",
+ address, program, control, native_registers[0xc8 + 12],
+ native_registers[0xc8 + 13] & 0xffff, native_registers[0xc8 + 14] & 0xffff);
+ if (static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr;
+ diagnostics) {
+ std::ofstream dump("native-inline-qmd.bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native_registers.data() + 0xc8), 256);
+ std::ofstream registers("native-inline-compute-registers.bin", std::ios::binary);
+ registers.write(reinterpret_cast<const char*>(native_registers.data()), sizeof(native_registers));
+ if (memory_manager.IsFullyMappedRange(program, 0x10000)) {
+ std::vector<u8> code(0x10000);
+ memory_manager.ReadBlockUnsafe(program, code.data(), code.size());
+ std::ofstream shader("native-inline-compute-shader.bin", std::ios::binary);
+ shader.write(reinterpret_cast<const char*>(code.data()), code.size());
+ }
+ }
+ if (!memory_manager.IsFullyMappedRange(address, 256)) {
+ native_macro->native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Unmapped native inline QMD destination");
+ return true;
+ }
+ rasterizer->AccelerateInlineToMemory(address, 256,
+ std::span<const u8>{reinterpret_cast<const u8*>(native_registers.data() + 0xc8), 256});
+ if ((control & (1U << 24)) || !(native_registers[0xcc] & (1U << 12))) {
+ const u32 previous = native_registers[0xad];
+ native_registers[0xad] = static_cast<u32>(address >> 8);
+ ProcessNativeMethod(system, 0xb0, 3, true);
+ native_registers[0xad] = previous;
+ }
+ return true;
+ }
+ switch (method) {
+ case 0x45: native_macro->ClearCode(argument); return true;
+ case 0x46: native_macro->AddCode(0, argument); return true;
+ case 0x47: native_bind_pointer = argument; return true;
+ case 0x48:
+ if (native_bind_pointer < native_macro_positions.size()) native_macro_positions[native_bind_pointer++] = argument;
+ else LOG_ERROR(HW_GPU, "Native compute macro bind out of bounds");
+ return true;
+ case 0xae: case 0xaf: case 0xb0: {
+ const GPUVAddr qmd_address = GPUVAddr{native_registers[0xad]} << 8;
+ VideoCommon::TraceNativeQuery(9, memory_manager.GetID(), qmd_address,
+ memory_manager.GpuToCpuAddress(qmd_address).value_or(0), method, argument, 0);
+ if (method == 0xae || (method == 0xaf && !(argument & 2)) ||
+ (method == 0xb0 && ((argument & 15) == 0 || (argument & 15) == 1))) return true;
+ static const bool qmd_diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr;
+ if ((argument & 15) == 12 && qmd_diagnostics) {
+ static u32 captured{};
+ if (captured++ == 0 && memory_manager.IsFullyMappedRange(qmd_address, 256)) {
+ std::array<u32, 64> descriptor{};
+ memory_manager.ReadBlockUnsafe(qmd_address, descriptor.data(), sizeof(descriptor));
+ std::ofstream dump("native-increment-qmd.bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(descriptor.data()), sizeof(descriptor));
+ LOG_WARNING(HW_GPU, "Native increment dependency descriptor: address={:#x}, words={:08x}",
+ qmd_address, fmt::join(descriptor, ","));
+ }
+ }
+ const auto error = native_qmd.Signal(qmd_address, argument,
+ [&](u64 address, NativeQmdScheduler::Descriptor& descriptor) {
+ if (!memory_manager.IsFullyMappedRange(address, sizeof(descriptor))) return false;
+ memory_manager.ReadBlockUnsafe(address, descriptor.data(), sizeof(descriptor));
+ if (static u32 logged{}; (descriptor[53] & (1U << 23)) && logged < 8) {
+ ++logged;
+ LOG_INFO(HW_GPU, "Native QMD {:#x} releases: {:08x} | {:08x} | {:08x}", address,
+ fmt::join(std::span{descriptor}.subspan(24, 4), ","),
+ fmt::join(std::span{descriptor}.subspan(28, 4), ","),
+ fmt::join(std::span{descriptor}.subspan(52, 4), ","));
+ }
+ for (u32 slot = 0; slot < 2; ++slot) {
+ const u32 flags = descriptor[25 + slot * 4];
+ if (!(flags & (1U << 23))) continue;
+ const GPUVAddr destination = (GPUVAddr{flags & 0xff} << 32) | descriptor[24 + slot * 4];
+ VideoCommon::TraceNativeQuery(7 + slot, memory_manager.GetID(), destination,
+ memory_manager.GpuToCpuAddress(destination).value_or(0),
+ descriptor[4], descriptor[26 + slot * 4], flags);
+ }
+ return true;
+ },
+ [&](u64 address, u32 payload, u32 flags) {
+ if (!memory_manager.IsFullyMappedRange(address, sizeof(payload))) return false;
+ if (flags & (1U << 26)) {
+ rasterizer->SignalFence([this, address, payload] {
+ const u32 value = memory_manager.Read<u32>(address) + payload;
+ memory_manager.Write<u32>(address, value);
+ if (const auto device = memory_manager.GpuToCpuAddress(address)) {
+ Tegra::Nx2SemaphoreWaiters::Get().OnWrite(*device, value);
+ }
+ static const u64 watch = [] {
+ const char* text = std::getenv("EDEN_NX2_SEM_WATCH");
+ return text ? u64{std::strtoull(text, nullptr, 16)} : u64{};
+ }();
+ if (watch != 0 && (address & ~u64{0xf}) == watch) {
+ LOG_INFO(HW_GPU, "Semaphore watch: QMD release add {:#x} -> {:#x}", address, value);
+ }
+ VideoCommon::TraceNativeQuery(1, memory_manager.GetID(), address,
+ memory_manager.GpuToCpuAddress(address).value_or(0), 0, value, 1U << 26);
+ });
+ } else if (flags & (1U << 19)) {
+ // NVN2 tracks QMD completion with a max release (bit 19) into a word other
+ // releases also advance: a plain write would move it backwards and leave
+ // GEQ acquires waiting forever.
+ rasterizer->SignalFence([this, address, payload] {
+ const u32 value = std::max(memory_manager.Read<u32>(address), payload);
+ memory_manager.Write<u32>(address, value);
+ if (const auto device = memory_manager.GpuToCpuAddress(address)) {
+ Tegra::Nx2SemaphoreWaiters::Get().OnWrite(*device, value);
+ }
+ });
+ } else {
+ static const u64 watch = [] {
+ const char* text = std::getenv("EDEN_NX2_SEM_WATCH");
+ return text ? u64{std::strtoull(text, nullptr, 16)} : u64{};
+ }();
+ if (watch != 0 && (address & ~u64{0xf}) == (watch & ~u64{0xf})) {
+ LOG_INFO(HW_GPU, "Semaphore watch: QMD release {:#x} = {:#x} flags {:#x}", address, payload, flags);
+ }
+ rasterizer->Query(address, VideoCommon::QueryType::Payload,
+ VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+ }
+ return true;
+ }, [&](const auto& descriptor) { return DispatchNativeCompute(descriptor); });
+ if (error.empty()) {
+ LOG_DEBUG(HW_GPU, "Native QMD control action={:#x}, QMD={:#x}", argument, qmd_address);
+ return true;
+ }
+ native_macro->native_execution_failed = true;
+ std::array<u32, 128> qmd{};
+ if (memory_manager.IsFullyMappedRange(qmd_address, sizeof(qmd))) {
+ memory_manager.ReadBlockUnsafe(qmd_address, qmd.data(), sizeof(qmd));
+ }
+ // Log a failing program's first few dispatches only (QMD words 48-49 hold its code
+ // address); it fails the same way every frame.
+ static std::unordered_map<u64, u32> failures;
+ const u64 program = (u64{qmd[49]} << 32) | qmd[48];
+ if (failures[program ^ std::hash<std::string>{}(std::string{error})]++ >= 2) {
+ return true;
+ }
+ LOG_ERROR(HW_GPU, "Native QMD failed: {}", error);
+ if (qmd_diagnostics && memory_manager.IsFullyMappedRange(qmd_address, sizeof(qmd))) {
+ std::ofstream dump("native-qmd.bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(qmd.data()), sizeof(qmd));
+ std::ofstream macros("native-dispatch-mme.bin", std::ios::binary);
+ macros.write(reinterpret_cast<const char*>(native_macro->native_code.data()), sizeof(native_macro->native_code));
+ std::ofstream registers("native-dispatch-registers.bin", std::ios::binary);
+ registers.write(reinterpret_cast<const char*>(native_registers.data()), sizeof(native_registers));
+ }
+ LOG_ERROR(HW_GPU, "Native compute dispatch unsupported: method={:#x}, value={:#x}, QMD={:#x}, words={:08x}", method, argument,
+ qmd_address, fmt::join(qmd, ","));
+ return true;
+ }
+ default:
+ // The inline upload register block is shared with the older class.
+ if (method < Regs::NUM_REGS) regs.reg_array[method] = argument;
+ return method != KEPLER_COMPUTE_REG_INDEX(exec_upload) && method != KEPLER_COMPUTE_REG_INDEX(data_upload);
+ }
+}
+
void KeplerCompute::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) {
- ASSERT_MSG(method < Regs::NUM_REGS, "Invalid KeplerCompute register, increase the size of the Regs structure");
+ if (native_mode && ProcessNativeMethod(system, method, method_argument, is_last_call)) return;
+ if (method >= Regs::NUM_REGS) {
+ LOG_ERROR(HW_GPU, "Compute register out of range: {:#x}", method);
+ return;
+ }
regs.reg_array[method] = method_argument;
@@ -85,6 +438,10 @@ void KeplerCompute::CallMethod(Core::System& system, u32 method, u32 method_argu
}
void KeplerCompute::CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) {
+ if (native_mode) {
+ for (u32 i = 0; i < amount; ++i) CallMethod(system, method, base_start[i], methods_pending - i <= 1);
+ return;
+ }
switch (method) {
case KEPLER_COMPUTE_REG_INDEX(data_upload):
upload_address = current_dma_segment;
@@ -100,6 +457,223 @@ void KeplerCompute::CallMultiMethod(Core::System& system, u32 method, const u32*
}
}
+bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& qmd) {
+ launch_description = {};
+ launch_description.grid_dim_x.Assign(qmd[12]);
+ launch_description.grid_dim_y.Assign(qmd[13] & 0xffff);
+ launch_description.grid_dim_z.Assign(qmd[14] & 0xffff);
+ launch_description.shared_alloc.Assign(qmd[17] & 0x3ffff);
+ launch_description.block_dim_x.Assign(qmd[18] >> 16);
+ launch_description.block_dim_y.Assign(qmd[19] & 0xffff);
+ launch_description.block_dim_z.Assign(qmd[19] >> 16);
+ launch_description.const_buffer_enable_mask.Assign(qmd[20] & 0xff);
+ launch_description.local_pos_alloc.Assign(qmd[23] & 0xffffff);
+ launch_description.local_crs_alloc.Assign(qmd[50] & 0xffffff);
+ regs.code_loc.address_low = qmd[48];
+ regs.code_loc.address_high = qmd[49] & 0x1ffff;
+ for (u32 slot = 0; slot < NumConstBuffers; ++slot) {
+ const u32 high = qmd[33 + slot * 2];
+ if (high & 0x1ff00) {
+ LOG_ERROR(HW_GPU, "Native compute constant address exceeds supported GPU width");
+ return false;
+ }
+ auto& buffer = launch_description.const_buffer_config[slot];
+ buffer.address_low = qmd[32 + slot * 2];
+ buffer.address_high.Assign(high & 0xff);
+ buffer.size.Assign(((high >> 19) & 0x1fff) << 4);
+ }
+ if (!launch_description.block_dim_x || !launch_description.block_dim_y || !launch_description.block_dim_z)
+ return false;
+ try {
+ rasterizer->DispatchCompute();
+ } catch (const Shader::Exception& error) {
+ static std::unordered_set<std::string> logged;
+ if (logged.insert(error.what()).second) {
+ LOG_ERROR(HW_GPU, "Native compute shader rejected: {}", error.what());
+ }
+ if (std::string_view{error.what()}.find("matrix multiply") != std::string_view::npos) {
+ DescribeNativeResources(qmd);
+ // The console runs this on tensor cores; stand in with a plain upscale of its input.
+ return UpscaleInsteadOfNativeProgram();
+ }
+ return false;
+ }
+ return true;
+}
+
+// Logs the images a rejected program's handles (constant bank 0, bytes 0x80-0x11c) point to, so a
+// substitute (e.g. an upscale blit for a tensor-core pass) can find its inputs and output.
+void KeplerCompute::DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd) {
+ static u32 described{};
+ if (described++ >= 2) return;
+ const auto& cbuf = launch_description.const_buffer_config[0];
+ const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low;
+ LOG_INFO(HW_GPU, "Native resources: cbuf0 {:#x} size {:#x}, grid {}x{}x{}, block {}x{}x{}, tic pool {:#x}",
+ cbuf_address, cbuf.size.Value(), qmd[12], qmd[13] & 0xffff, qmd[14] & 0xffff,
+ launch_description.block_dim_x.Value(), launch_description.block_dim_y.Value(),
+ launch_description.block_dim_z.Value(), regs.tic.Address());
+ for (u32 offset = 0x80; offset < 0x120; offset += 4) {
+ const u32 handle = memory_manager.Read<u32>(cbuf_address + offset);
+ const u32 tic_index = handle & 0xfffff;
+ if (handle == 0 || tic_index > regs.tic.limit) continue;
+ const auto tic = GetTICEntry(tic_index);
+ LOG_INFO(HW_GPU, " handle @{:#x} = {:#x}: tic {} address {:#x} {}x{} format {:#x} types {}/{}/{}/{}",
+ offset, handle, tic_index, tic.Address(), tic.Width(), tic.Height(),
+ static_cast<u32>(tic.format.Value()), static_cast<u32>(tic.r_type.Value()),
+ static_cast<u32>(tic.g_type.Value()), static_cast<u32>(tic.b_type.Value()),
+ static_cast<u32>(tic.a_type.Value()));
+ }
+}
+
+std::atomic<u32> g_nx2_trace_draws_after_upscale{};
+
+// Title 4's upscaler reads the tonemapped frame through the texture handle at cbuf0+0x90 and
+// writes the output resolution image through the surface handle at cbuf0+0xa0 (from its
+// TLD/SUST operands). Blit one to the other with bilinear filtering when both are plain 2D images
+// of the same format and the output is larger.
+bool KeplerCompute::UpscaleInsteadOfNativeProgram() {
+ const auto& cbuf = launch_description.const_buffer_config[0];
+ const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low;
+ const auto image = [&](u32 offset) -> std::optional<Texture::TICEntry> {
+ const u32 tic_index = memory_manager.Read<u32>(cbuf_address + offset) & 0xfffff;
+ if (tic_index > regs.tic.limit) return std::nullopt;
+ return GetTICEntry(tic_index);
+ };
+ const auto input = image(0x90), output = image(0xa0);
+ if (!input || !output || input->format != output->format || input->Address() == 0 ||
+ output->Address() == 0 || output->Width() <= input->Width() || !input->IsBlockLinear() ||
+ !output->IsBlockLinear())
+ return false;
+ using RTFormat = Tegra::RenderTargetFormat;
+ RTFormat format{};
+ switch (input->format) {
+ case Texture::TextureFormat::A2B10G10R10: format = RTFormat::A2B10G10R10_UNORM; break;
+ case Texture::TextureFormat::A8B8G8R8: format = RTFormat::A8B8G8R8_UNORM; break;
+ case Texture::TextureFormat::B10G11R11: format = RTFormat::B10G11R11_FLOAT; break;
+ case Texture::TextureFormat::R16G16B16A16: format = RTFormat::R16G16B16A16_FLOAT; break;
+ default: return false;
+ }
+ const auto surface = [&](const Texture::TICEntry& tic) {
+ Fermi2D::Surface result{};
+ result.format = format;
+ result.linear = Fermi2D::MemoryLayout::BlockLinear;
+ result.block_width.Assign(tic.block_width);
+ result.block_height.Assign(tic.block_height);
+ result.block_depth.Assign(tic.block_depth);
+ result.depth = 1;
+ result.width = tic.Width();
+ result.height = tic.Height();
+ result.addr_upper = static_cast<u32>(tic.Address() >> 32);
+ result.addr_lower = static_cast<u32>(tic.Address());
+ return result;
+ };
+ const Fermi2D::Config config{
+ .operation = Fermi2D::Operation::SrcCopy,
+ .filter = Fermi2D::Filter::Bilinear,
+ .must_accelerate = true,
+ .dst_x0 = 0,
+ .dst_y0 = 0,
+ .dst_x1 = static_cast<s32>(output->Width()),
+ .dst_y1 = static_cast<s32>(output->Height()),
+ .src_x0 = 0,
+ .src_y0 = 0,
+ .src_x1 = static_cast<s32>(input->Width()),
+ .src_y1 = static_cast<s32>(input->Height()),
+ };
+ // Debug: does the input hold anything? Download it once and count non-zero words.
+ static const u32 sample_at = [] {
+ const char* value = std::getenv("EDEN_NX2_SAMPLE_AT");
+ return value ? static_cast<u32>(std::strtoul(value, nullptr, 10)) : 20U;
+ }();
+ if (static u32 sampled{}; sampled++ == sample_at) {
+ const auto sample = [&](const Texture::TICEntry& tic) {
+ const u64 size = u64{tic.Width()} * tic.Height() * 4;
+ if (const auto device = memory_manager.GpuToCpuAddress(tic.Address())) {
+ rasterizer->FlushRegion(*device, size, VideoCommon::CacheType::All);
+ }
+ u32 nonzero{}, samples{};
+ u64 sum{};
+ for (u64 offset = 0; offset < size; offset += 4096 + 4) {
+ const u32 word = memory_manager.Read<u32>(tic.Address() + offset);
+ ++samples;
+ nonzero += word != 0;
+ sum += (word & 0x3ff) + ((word >> 10) & 0x3ff) + ((word >> 20) & 0x3ff);
+ }
+ LOG_INFO(HW_GPU, "Upscale fallback image {:#x}: {}/{} sampled words non-zero, mean channel {}",
+ tic.Address(), nonzero, samples, samples ? sum / (samples * 3) : 0);
+ };
+ sample(*input);
+ sample(*output);
+ // Every 2560x1440 target the in-game passes draw to (from EDEN_NX2_RT_TRACE).
+ for (const GPUVAddr address : {0x8116a03000ULL, 0x8117ae5000ULL, 0x8114b62000ULL,
+ 0x8115a63000ULL, 0x8117904000ULL, 0x8114a02000ULL}) {
+ Texture::TICEntry fake = *input;
+ const u64 size = 2560ULL * 1440 * 4;
+ if (const auto device = memory_manager.GpuToCpuAddress(address)) {
+ rasterizer->FlushRegion(*device, size, VideoCommon::CacheType::All);
+ }
+ u32 nonzero{}, samples{}, distinct{};
+ u32 last{};
+ for (u64 offset = 0; offset < size; offset += 4096 + 4) {
+ const u32 word = memory_manager.Read<u32>(address + offset);
+ ++samples;
+ nonzero += word != 0;
+ distinct += word != last;
+ last = word;
+ }
+ LOG_INFO(HW_GPU, "Upscale fallback target {:#x}: {}/{} non-zero, {} changes", address, nonzero,
+ samples, distinct);
+ (void)fake;
+ }
+ }
+ auto source = surface(*input);
+ // EDEN_NX2_UPSCALE_SOURCE=<gpu address>: blit from another same-sized image (debug aid).
+ if (static const u64 override_source = [] {
+ const char* value = std::getenv("EDEN_NX2_UPSCALE_SOURCE");
+ return value ? std::strtoull(value, nullptr, 16) : 0ULL;
+ }();
+ override_source != 0) {
+ source.addr_upper = static_cast<u32>(override_source >> 32);
+ source.addr_lower = static_cast<u32>(override_source);
+ if (const char* value = std::getenv("EDEN_NX2_UPSCALE_SOURCE_FORMAT")) {
+ source.format = static_cast<RTFormat>(std::strtoul(value, nullptr, 16));
+ }
+ }
+ const bool done = rasterizer->AccelerateSurfaceCopy(source, surface(*output), config);
+ if (static u32 checked{}; checked++ == sample_at) {
+ const u64 size = u64{output->Width()} * output->Height() * 4;
+ if (const auto device = memory_manager.GpuToCpuAddress(output->Address())) {
+ rasterizer->FlushRegion(*device, size, VideoCommon::CacheType::All);
+ }
+ u32 nonzero{}, samples{}, changes{}, last{};
+ for (u64 offset = 0; offset < size; offset += 4096 + 4) {
+ const u32 word = memory_manager.Read<u32>(output->Address() + offset);
+ ++samples;
+ nonzero += word != 0;
+ changes += word != last;
+ last = word;
+ }
+ LOG_INFO(HW_GPU, "Upscale fallback output after blit: {}/{} non-zero, {} changes", nonzero, samples,
+ changes);
+ }
+ // Debug: trace the draws that follow one upscale (EDEN_NX2_RT_TRACE_AFTER_UPSCALE).
+ static const u32 trace_at = [] {
+ const char* value = std::getenv("EDEN_NX2_TRACE_AFTER_UPSCALE");
+ return value ? static_cast<u32>(std::strtoul(value, nullptr, 10)) : 30U;
+ }();
+ if (static u32 count{}; ++count == trace_at) g_nx2_trace_draws_after_upscale = 40;
+ static u32 logged{};
+ if (logged++ < 4) {
+ LOG_INFO(HW_GPU, "Upscale fallback {:#x} {}x{} block {}/{}/{} -> {:#x} {}x{} block {}/{}/{} levels {}: {}",
+ input->Address(), input->Width(), input->Height(), input->block_width.Value(),
+ input->block_height.Value(), input->block_depth.Value(), output->Address(),
+ output->Width(), output->Height(), output->block_width.Value(),
+ output->block_height.Value(), output->block_depth.Value(), output->max_mip_level.Value(),
+ done ? "blitted" : "not accelerated");
+ }
+ return done;
+}
+
void KeplerCompute::ProcessLaunch() {
const GPUVAddr launch_desc_loc = regs.launch_desc_loc.Address();
memory_manager.ReadBlockUnsafe(launch_desc_loc, &launch_description,
diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h
index 078d3d45be..5ebf8edf38 100644
--- a/src/video_core/engines/kepler_compute.h
+++ b/src/video_core/engines/kepler_compute.h
@@ -7,6 +7,7 @@
#pragma once
#include <array>
+#include <atomic>
#include <cstddef>
#include <optional>
#include <vector>
@@ -15,6 +16,8 @@
#include "common/common_types.h"
#include "video_core/engines/engine_interface.h"
#include "video_core/engines/engine_upload.h"
+#include "video_core/macro.h"
+#include "video_core/native_qmd.h"
#include "video_core/textures/texture.h"
namespace Core {
@@ -31,6 +34,9 @@ class RasterizerInterface;
namespace Tegra::Engines {
+/// Draws left to trace after an upscale fallback (debug aid).
+extern std::atomic<u32> g_nx2_trace_draws_after_upscale;
+
/**
* This Engine is known as GK104_Compute. Documentation can be found in:
* https://github.com/envytools/envytools/blob/master/rnndb/graph/gk104_compute.xml
@@ -51,6 +57,24 @@ public:
/// Binds a rasterizer to this engine.
void BindRasterizer(VideoCore::RasterizerInterface* rasterizer);
+ /// The channel's MME data RAM, shared with its 3D engine.
+ void SetNativeDataRam(std::array<u32, 0xc00>* data_ram) { native_data_ram = data_ram; }
+ void SetNativeMacroMode(MacroEngine* shared_macro, std::span<u32> positions, NativeMacroContext* context = nullptr) {
+ if (native_context != context) native_compute_uploaded = false;
+ native_context = context;
+ native_mode = shared_macro != nullptr;
+ upload_state.SetNativePartialTransfers(native_mode);
+ native_macro = shared_macro;
+ native_macro_positions = positions;
+ if (native_mode) execution_mask.set();
+ else {
+ execution_mask.reset();
+ execution_mask[KEPLER_COMPUTE_REG_INDEX(exec_upload)] = true;
+ execution_mask[KEPLER_COMPUTE_REG_INDEX(data_upload)] = true;
+ execution_mask[KEPLER_COMPUTE_REG_INDEX(launch)] = true;
+ }
+ }
+
static constexpr std::size_t NumConstBuffers = 8;
struct Regs {
@@ -211,7 +235,29 @@ public:
return indirect_compute;
}
+ bool IsNativeMode() const { return native_mode; }
+
private:
+ bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor);
+ void ProcessNativeReportSemaphore(u32 argument);
+ void DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd);
+ bool UpscaleInsteadOfNativeProgram();
+ bool ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call);
+ NativeQmdScheduler native_qmd;
+ std::array<u32, 0xc00>* native_data_ram{};
+ bool native_mme_dumped{};
+ std::optional<u32> native_executing_macro_logged;
+ bool native_mode{};
+ NativeMacroContext* native_context{};
+ bool native_compute_uploaded{};
+ MacroEngine* native_macro{};
+ std::array<u32, 0x1000> native_registers{};
+ std::span<u32> native_macro_positions;
+ std::vector<u32> native_parameters;
+ std::vector<u32> native_memo_parameters; ///< The current call's parameters, for the memo
+ std::optional<u32> native_executing_macro;
+ u32 native_bind_pointer{};
+
void ProcessLaunch();
void ConsumeSinkImpl(Core::System& system) override;
diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp
index e7da9d9138..60efef9b8e 100644
--- a/src/video_core/engines/maxwell_3d.cpp
+++ b/src/video_core/engines/maxwell_3d.cpp
@@ -5,6 +5,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <cstring>
+#include <limits>
#include <optional>
#include "common/assert.h"
#include "common/bit_util.h"
@@ -253,7 +254,10 @@ u32 Maxwell3D::GetMaxCurrentVertices() {
const auto& limit = regs.vertex_stream_limits[index];
const GPUVAddr gpu_addr_begin = array.Address();
const GPUVAddr gpu_addr_end = limit.Address() + 1;
- const u32 address_size = static_cast<u32>(gpu_addr_end - gpu_addr_begin);
+ const u64 stream_size = GetNativeMacroEngine().native_turing
+ ? regs.NativeVertexStreamSize(index)
+ : gpu_addr_end - gpu_addr_begin;
+ const u32 address_size = static_cast<u32>(std::min<u64>(stream_size, std::numeric_limits<u32>::max()));
num_vertices = (std::max)(
num_vertices, address_size / (std::max)(attribute.SizeInBytes(), array.stride.Value()));
break;
@@ -305,6 +309,9 @@ void Maxwell3D::ProcessDirtyRegisters(u32 method, u32 argument) {
}
void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argument, bool is_last_call) {
+ if (GetNativeMacroEngine().native_turing && method == MAXWELL3D_REG_INDEX(clear_surface)) {
+ LOG_DEBUG(HW_GPU, "Native GPU surface clear: value={:#x}", argument);
+ }
switch (method) {
case MAXWELL3D_REG_INDEX(wait_for_idle):
return rasterizer->WaitForIdle();
@@ -312,9 +319,9 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume
shadow_state.shadow_ram_control = static_cast<Regs::ShadowRamControl>(nonshadow_argument);
return;
case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
- return macro_engine.ClearCode(regs.load_mme.instruction_ptr);
+ return GetNativeMacroEngine().ClearCode(regs.load_mme.instruction_ptr);
case MAXWELL3D_REG_INDEX(load_mme.instruction):
- return macro_engine.AddCode(regs.load_mme.instruction_ptr, argument);
+ return GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr, argument);
case MAXWELL3D_REG_INDEX(load_mme.start_address):
return ProcessMacroBind(argument);
case MAXWELL3D_REG_INDEX(falcon[4]):
@@ -378,19 +385,34 @@ void Maxwell3D::CallMacroMethod(Core::System& system, u32 method, const std::vec
// Lookup the macro offset
const u32 entry =
- ((method - MacroRegistersStart) >> 1) % static_cast<u32>(macro_positions.size());
+ ((method - MacroRegistersStart) >> 1) % static_cast<u32>(GetNativeMacroPositions().size());
// Execute the current macro.
- macro_engine.Execute(system, *this, macro_positions[entry], parameters);
+ GetNativeMacroEngine().Execute(system, *this, GetNativeMacroPositions()[entry], parameters);
draw_manager.DrawDeferred(*this);
}
+// EDEN_NX2_3D_METHODS: log the first write to each 3D register (finds methods Eden doesn't model).
+static const bool g_trace_first_method_writes = std::getenv("EDEN_NX2_3D_METHODS") != nullptr;
+
+void Maxwell3D::TraceFirstMethodWrite(u32 method, u32 argument) {
+ static std::array<bool, Regs::NUM_REGS> seen{};
+ if (method < seen.size() && !seen[method]) {
+ seen[method] = true;
+ LOG_INFO(HW_GPU, "3D method first write: {:#x} (byte {:#x}) = {:#x}", method, method * 4, argument);
+ }
+}
+
void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) {
// It is an error to write to a register other than the current macro's ARG register before
// it has finished execution.
+ if (executing_macro != 0 && method != executing_macro + 1) {
+ DumpRecentWrites(method);
+ }
if (executing_macro != 0) {
ASSERT(method == executing_macro + 1);
}
+ RecordWrite(method, method_argument, 1, is_last_call ? 1 : 2);
// Methods after 0xE00 are special, they're actually triggers for some microcode that was
// uploaded to the GPU during initialization.
@@ -398,6 +420,9 @@ void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument
ProcessMacro(system, method, &method_argument, 1, is_last_call);
return;
}
+ if (g_trace_first_method_writes) [[unlikely]] {
+ TraceFirstMethodWrite(method, method_argument);
+ }
ASSERT(method < Regs::NUM_REGS && "Invalid Maxwell3D register, increase the size of the Regs structure");
const u32 argument = ProcessShadowRam(method, method_argument);
@@ -405,7 +430,24 @@ void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument
ProcessMethodCall(method, argument, method_argument, is_last_call);
}
+void Maxwell3D::DumpRecentWrites(u32 method) {
+ static u32 dumps{};
+ if (dumps++ >= 4) {
+ return;
+ }
+ LOG_ERROR(HW_GPU, "Write to {:#x} while macro {:#x} collects parameters ({} so far); last writes:",
+ method, executing_macro, macro_params.size());
+ for (size_t i = 0; i < recent_writes.size(); ++i) {
+ const auto& w = recent_writes[(recent_write_index + i) % recent_writes.size()];
+ if (w.amount != 0) {
+ LOG_ERROR(HW_GPU, " method {:#x} arg {:#x} amount {} pending {}", w.method, w.argument,
+ w.amount, w.pending);
+ }
+ }
+}
+
void Maxwell3D::CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) {
+ RecordWrite(method, amount ? base_start[0] : 0, amount, methods_pending);
// Methods after 0xE00 are special, they're actually triggers for some microcode that was
// uploaded to the GPU during initialization.
if (method >= MacroRegistersStart) {
@@ -453,11 +495,11 @@ void Maxwell3D::CallMultiMethod(Core::System& system, u32 method, const u32* bas
}
void Maxwell3D::ProcessMacroUpload(u32 data) {
- macro_engine.AddCode(regs.load_mme.instruction_ptr++, data);
+ GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr++, data);
}
void Maxwell3D::ProcessMacroBind(u32 data) {
- macro_positions[regs.load_mme.start_address_ptr++] = data;
+ GetNativeMacroPositions()[regs.load_mme.start_address_ptr++] = data;
}
void Maxwell3D::ProcessFirmwareCall4() {
@@ -586,6 +628,9 @@ void Maxwell3D::ProcessCounterReset() {
}
void Maxwell3D::ProcessSyncPoint() {
+ if (GetNativeMacroEngine().native_turing) {
+ LOG_DEBUG(HW_GPU, "Native 3D syncpoint increment: id={}, raw={:#x}", regs.sync_info.sync_point.Value(), regs.reg_array[MAXWELL3D_REG_INDEX(sync_info)]);
+ }
const u32 sync_point = regs.sync_info.sync_point.Value();
[[maybe_unused]] const u32 cache_flush = regs.sync_info.clean_l2.Value();
rasterizer->SignalSyncPoint(sync_point);
@@ -595,6 +640,15 @@ void Maxwell3D::ProcessCBBind(size_t stage_index) {
// Bind the buffer currently in CB_ADDRESS to the specified index in the desired shader
// stage.
const auto& bind_data = regs.bind_groups[stage_index];
+ static std::array<std::array<bool, 32>, 5> seen{};
+ if (!seen[stage_index][bind_data.shader_slot]) {
+ seen[stage_index][bind_data.shader_slot] = true;
+ LOG_INFO(HW_GPU, "Constant buffer bind: stage {} slot {} address {:#x} size {:#x}", stage_index,
+ bind_data.shader_slot.Value(), regs.const_buffer.Address(), regs.const_buffer.size);
+ }
+ if (bind_data.shader_slot >= Regs::MaxConstBuffers) {
+ return; // slot field is 5 bits; the state array holds 18
+ }
auto& buffer = state.shader_stages[stage_index].const_buffers[bind_data.shader_slot];
buffer.enabled = bind_data.valid.Value() != 0;
buffer.address = regs.const_buffer.Address();
diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h
index 0cab2ae986..82e716b822 100644
--- a/src/video_core/engines/maxwell_3d.h
+++ b/src/video_core/engines/maxwell_3d.h
@@ -2605,6 +2605,12 @@ public:
u32 stop_shader_mask;
};
+ // Native C597/C797 use byte counts at 0x600 rather than legacy end addresses.
+ u64 NativeVertexStreamSize(std::size_t index) const {
+ const std::size_t method = 0x600 / sizeof(u32) + index * 2;
+ return (u64{reg_array[method] & 0xffU} << 32) | reg_array[method + 1];
+ }
+
// clang-format off
union {
struct {
@@ -3129,6 +3135,26 @@ public:
/// Reads a register value located at the input method address
u32 GetRegisterValue(u32 method) const;
+ void TraceFirstMethodWrite(u32 method, u32 argument);
+
+ /// GPU memory, for Turing macros that read memory into their FIFO or data RAM.
+ MemoryManager& GetMemoryManager() {
+ return memory_manager;
+ }
+
+ void SetNativeTuringMacroMode(bool enabled) {
+ GetNativeMacroEngine().native_turing = enabled;
+ upload_state.SetNativePartialTransfers(enabled);
+ }
+ void SetNativeMacroContext(std::shared_ptr<NativeMacroContext> context) {
+ native_macro_context = std::move(context);
+ }
+ MacroEngine& GetNativeMacroEngine() { return native_macro_context ? native_macro_context->engine : macro_engine; }
+ /// This channel's MME data RAM (shared with its compute engine).
+ std::array<u32, 0xc00>& GetNativeDataRam() { return native_data_ram; }
+ std::span<u32> GetNativeMacroPositions() { return native_macro_context ? std::span<u32>{native_macro_context->positions} : std::span<u32>{macro_positions}; }
+
+
/// Write the value to the register identified by method.
void CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) override;
@@ -3260,10 +3286,25 @@ private:
/// Macro method that is currently being executed / being fed parameters.
u32 executing_macro = 0;
+
+ struct RecentWrite {
+ u32 method;
+ u32 argument;
+ u32 amount;
+ u32 pending;
+ };
+ std::array<RecentWrite, 48> recent_writes{};
+ size_t recent_write_index = 0;
+ void RecordWrite(u32 method, u32 argument, u32 amount, u32 pending) {
+ recent_writes[recent_write_index++ % recent_writes.size()] = {method, argument, amount, pending};
+ }
+ void DumpRecentWrites(u32 method);
/// Parameters that have been submitted to the macro call so far.
std::vector<u32> macro_params;
/// Interpreter for the macro codes uploaded to the GPU.
+ std::shared_ptr<NativeMacroContext> native_macro_context;
+ std::array<u32, 0xc00> native_data_ram{};
MacroEngine macro_engine;
Upload::State upload_state;
diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp
index ed4675ed12..0b0541799b 100644
--- a/src/video_core/engines/maxwell_dma.cpp
+++ b/src/video_core/engines/maxwell_dma.cpp
@@ -4,6 +4,7 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <cstdlib>
#include "common/algorithm.h"
#include "common/assert.h"
#include "common/logging.h"
@@ -42,7 +43,11 @@ void MaxwellDMA::ConsumeSinkImpl(Core::System& system) {
}
void MaxwellDMA::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) {
- ASSERT_MSG(method < NUM_REGS, "Invalid MaxwellDMA register");
+ if (method >= NUM_REGS) {
+ // Ampere copy (Switch 2) can use methods past this register file; don't write past it.
+ LOG_DEBUG(HW_GPU, "Ignoring MaxwellDMA method {:#x} past the register file", method);
+ return;
+ }
regs.reg_array[method] = method_argument;
@@ -59,6 +64,32 @@ void MaxwellDMA::CallMultiMethod(Core::System& system, u32 method, const u32* ba
void MaxwellDMA::Launch() {
LOG_TRACE(Render_OpenGL, "DMA copy {:#x} -> {:#x}", static_cast<GPUVAddr>(regs.offset_in), GPUVAddr(regs.offset_out));
+ // EDEN_NX2_TRACE_DST=<hex>: also log copies writing within 32 MiB above that GPU address.
+ static const u64 trace_destination = [] {
+ const char* value = std::getenv("EDEN_NX2_TRACE_DST");
+ return value != nullptr ? std::strtoull(value, nullptr, 16) : 0ULL;
+ }();
+ if (trace_destination != 0 && GPUVAddr(regs.offset_out) >= trace_destination &&
+ GPUVAddr(regs.offset_out) < trace_destination + (32ULL << 20)) {
+ static u32 traced_destination{};
+ if (traced_destination++ < 40) {
+ LOG_INFO(HW_GPU,
+ "nx2 DMA to traced dst: {:#x} -> {:#x} lines={} length={} pitch in={:#x} "
+ "out={:#x} launch={:#x} src_layout={} dst_layout={}",
+ static_cast<GPUVAddr>(regs.offset_in), GPUVAddr(regs.offset_out),
+ regs.line_count, regs.line_length_in, regs.pitch_in, regs.pitch_out,
+ regs.reg_array[0xc0], regs.launch_dma.src_memory_layout.Value() == LaunchDMA::MemoryLayout::PITCH ? "pitch" : "block",
+ regs.launch_dma.dst_memory_layout.Value() == LaunchDMA::MemoryLayout::PITCH ? "pitch" : "block");
+ }
+ }
+ if (static const bool trace = std::getenv("EDEN_NX2_RT_TRACE") != nullptr; trace) {
+ static u32 traced{};
+ if (traced++ < 64) {
+ LOG_INFO(HW_GPU, "nx2 DMA {:#x} -> {:#x} lines={} length={} launch={:#x}",
+ static_cast<GPUVAddr>(regs.offset_in), GPUVAddr(regs.offset_out),
+ regs.line_count, regs.line_length_in, regs.reg_array[0xc0]);
+ }
+ }
// TODO(Subv): Perform more research and implement all features of this engine.
const LaunchDMA& launch = regs.launch_dma;
@@ -219,7 +250,7 @@ void MaxwellDMA::CopyBlockLinearToPitch() {
UnswizzleSubrect(tmp_write_buffer, tmp_read_buffer, bytes_per_pixel, width, height, depth,
x_offset, src_params.origin.y, x_elements, regs.line_count, block_height,
- block_depth, dst_operand.pitch);
+ block_depth, dst_operand.pitch, memory_manager.UsesNativeTiling());
}
void MaxwellDMA::CopyPitchToBlockLinear() {
@@ -284,7 +315,7 @@ void MaxwellDMA::CopyPitchToBlockLinear() {
// If the input is linear and the output is tiled, swizzle the input and copy it over.
SwizzleSubrect(tmp_write_buffer, tmp_read_buffer, bytes_per_pixel, width, height, depth,
x_offset, dst_params.origin.y, x_elements, regs.line_count, block_height,
- block_depth, regs.pitch_in);
+ block_depth, regs.pitch_in, memory_manager.UsesNativeTiling());
}
void MaxwellDMA::CopyBlockLinearToBlockLinear() {
@@ -337,11 +368,11 @@ void MaxwellDMA::CopyBlockLinearToBlockLinear() {
UnswizzleSubrect(intermediate_buffer, tmp_read_buffer, bytes_per_pixel, src_width, src.height,
src.depth, src_x_offset, src.origin.y, x_elements, regs.line_count,
- src.block_size.height, src.block_size.depth, pitch);
+ src.block_size.height, src.block_size.depth, pitch, memory_manager.UsesNativeTiling());
SwizzleSubrect(tmp_write_buffer, intermediate_buffer, bytes_per_pixel, dst_width, dst.height,
dst.depth, dst_x_offset, dst.origin.y, x_elements, regs.line_count,
- dst.block_size.height, dst.block_size.depth, pitch);
+ dst.block_size.height, dst.block_size.depth, pitch, memory_manager.UsesNativeTiling());
}
void MaxwellDMA::ReleaseSemaphore() {
diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp
index 0b05d31718..b607f20161 100644
--- a/src/video_core/engines/puller.cpp
+++ b/src/video_core/engines/puller.cpp
@@ -4,6 +4,9 @@
// SPDX-FileCopyrightText: 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
+#include <atomic>
+#include <cstdlib>
+
#include "common/assert.h"
#include "common/logging.h"
#include "common/settings.h"
@@ -18,6 +21,7 @@
#include "video_core/engines/puller.h"
#include "video_core/gpu.h"
#include "video_core/memory_manager.h"
+#include "video_core/nx2.h"
#include "video_core/rasterizer_interface.h"
namespace Tegra::Engines {
@@ -27,14 +31,102 @@ namespace {
constexpr u32 Gf100BindClassMask = 0xffff;
constexpr u32 Gf100BindValidMask = 0x1f0000 | Gf100BindClassMask;
+// Reads a semaphore word, flushing a GPU-written copy out of the buffer cache first.
+u32 ReadSemaphoreWord(DmaPusher& dma_pusher, GPUVAddr address) {
+ if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address);
+ device && dma_pusher.rasterizer->MustFlushRegion(*device, sizeof(u32),
+ VideoCommon::CacheType::BufferCache)) {
+ dma_pusher.rasterizer->FlushRegion(*device, sizeof(u32),
+ VideoCommon::CacheType::BufferCache);
+ }
+ return dma_pusher.memory_manager.Read<u32>(address);
+}
+
} // Anonymous namespace
+bool Puller::PendingAcquire::Matches(u32 current) const {
+ switch (static_cast<GpuSemaphoreOperation>(operation)) {
+ case GpuSemaphoreOperation::AcquireEqual:
+ return current == value;
+ case GpuSemaphoreOperation::AcquireGequal:
+ return current >= value;
+ case GpuSemaphoreOperation::AcquireMask:
+ return (current & value) != 0;
+ default:
+ return true;
+ }
+}
+
+bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) {
+ if (!pending_acquire) {
+ return true;
+ }
+ const u32 current = ReadSemaphoreWord(dma_pusher, pending_acquire->address);
+ if (Nx2SemaphoreWaiters::Get().Latched(this)) {
+ if (static std::atomic<u32> rescued{};
+ !pending_acquire->Matches(current) && rescued.fetch_add(1) < 64) {
+ LOG_INFO(HW_GPU, "Channel {} met semaphore {:#x} value {:#x} at its release (now {:#x})",
+ dma_pusher.channel_state.bind_id, pending_acquire->address,
+ pending_acquire->value, current);
+ }
+ return true;
+ }
+ if (pending_acquire->Matches(current)) {
+ return true;
+ }
+ if (!pending_acquire->reported &&
+ std::chrono::steady_clock::now() - pending_acquire->since > std::chrono::seconds{2}) {
+ pending_acquire->reported = true;
+ const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address);
+ LOG_WARNING(HW_GPU, "Channel {} waits on semaphore {:#x} (op {}, {:#x}, now {:#x}) for 2 s; "
+ "device {:#x}, buffer cache holds GPU data: {}, words {:08x} {:08x} {:08x} {:08x}",
+ dma_pusher.channel_state.bind_id, pending_acquire->address,
+ pending_acquire->operation, pending_acquire->value, current, device.value_or(0),
+ device && dma_pusher.rasterizer->MustFlushRegion(*device, 16, VideoCommon::CacheType::All),
+ dma_pusher.memory_manager.Read<u32>(pending_acquire->address & ~GPUVAddr{0xf}),
+ dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 4),
+ dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 8),
+ dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 12));
+ }
+ return false;
+}
+
+bool Puller::ResumeAcquire(DmaPusher& dma_pusher) {
+ if (!CanResumeAcquire(dma_pusher)) {
+ return false;
+ }
+ pending_acquire.reset();
+ Nx2SemaphoreWaiters::Get().Remove(this);
+ return true;
+}
+
void Puller::ProcessBindMethod(DmaPusher& dma_pusher, const MethodCall& method_call) {
LOG_DEBUG(HW_GPU, "Binding subchannel {} to engine {:#x}", method_call.subchannel, method_call.argument);
u32 engine = method_call.argument;
if ((engine & ~Gf100BindClassMask) != 0 && (engine & ~Gf100BindValidMask) == 0) {
engine &= Gf100BindClassMask;
}
+ // Switch 2 binds Ampere classes; run them on the Maxwell engines (same method layout for
+ // most state). Class 0 unbinds the subchannel.
+ switch (engine & 0xFFFF) {
+ case 0xC797: // AMPERE_B
+ engine = static_cast<u32>(EngineID::MAXWELL_B);
+ break;
+ case 0xC7C0: // AMPERE_COMPUTE_B
+ engine = static_cast<u32>(EngineID::KEPLER_COMPUTE_B);
+ break;
+ case 0xC7B5: // AMPERE_DMA_COPY_B
+ engine = static_cast<u32>(EngineID::MAXWELL_DMA_COPY_A);
+ break;
+ case 0x0:
+ if (g_nx2_ampere) {
+ LOG_DEBUG(HW_GPU, "Switch 2 unbinds subchannel {}", method_call.subchannel);
+ return;
+ }
+ break;
+ default:
+ break;
+ }
const auto engine_id = static_cast<EngineID>(engine);
bound_engines[method_call.subchannel] = engine_id;
@@ -82,10 +174,64 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) {
void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
const auto semaphoreOperationMask = 0xF;
const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask);
+ // EDEN_NX2_SEM_WATCH=<gpu address>: log every release/acquire on the 16 bytes there.
+ static const GPUVAddr watch = [] {
+ const char* value = std::getenv("EDEN_NX2_SEM_WATCH");
+ return value ? GPUVAddr{std::strtoull(value, nullptr, 16)} : GPUVAddr{};
+ }();
+ if (watch != 0 && (regs.semaphore_address.SemaphoreAddress() & ~GPUVAddr{0xf}) == watch) {
+ LOG_INFO(HW_GPU, "Semaphore watch: channel {} as {} device {:#x} op {} {:#x} value {:#x} (now {:#x})",
+ dma_pusher.channel_state.bind_id, dma_pusher.memory_manager.GetID(),
+ dma_pusher.memory_manager.GpuToCpuAddress(regs.semaphore_address.SemaphoreAddress()).value_or(0),
+ static_cast<u32>(op),
+ regs.semaphore_address.SemaphoreAddress(), regs.semaphore_sequence,
+ dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress()));
+ }
if (op == GpuSemaphoreOperation::WriteLong) {
const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()};
const u32 payload = regs.semaphore_sequence;
- dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::HasTimeout, payload, 0);
+ // RELEASE_SIZE (bit 24) set means a 4-byte release without the timestamp; writing 16
+ // bytes there clobbers the words next to it (Switch 2 packs semaphores tightly).
+ const auto properties = g_nx2_ampere && (regs.semaphore_trigger & (1U << 24)) != 0
+ ? VideoCommon::QueryPropertiesFlags::IsAFence
+ : VideoCommon::QueryPropertiesFlags::HasTimeout;
+ dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0);
+ if (const GPUVAddr mirror = g_nx2_ampere ? Nx2SemaphoreMirror(sequence_address) : 0;
+ mirror != 0) {
+ LOG_DEBUG(HW_GPU, "Switch 2 semaphore {:#x}={:#x} mirrored to {:#x}", sequence_address, payload, mirror);
+ dma_pusher.rasterizer->Query(mirror, VideoCommon::QueryType::Payload, properties, payload, 0);
+ }
+ } else if (g_nx2_ampere) {
+ // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has
+ // written the commands this channel runs next: suspend the channel until the value is
+ // met instead of reading ahead (approach from changes.patch).
+ PendingAcquire acquire{
+ .address = regs.semaphore_address.SemaphoreAddress(),
+ .value = regs.semaphore_sequence,
+ .operation = static_cast<u32>(op),
+ .since = std::chrono::steady_clock::now(),
+ .reported = false,
+ };
+ if (op != GpuSemaphoreOperation::AcquireEqual && op != GpuSemaphoreOperation::AcquireGequal &&
+ op != GpuSemaphoreOperation::AcquireMask) {
+ LOG_ERROR(HW_GPU, "Invalid semaphore operation {}", static_cast<u32>(op));
+ return;
+ }
+ if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) {
+ return;
+ }
+ // Registered before the fences run, so a release among them that meets the value counts
+ // even if a later one overwrites it.
+ auto& waiters = Nx2SemaphoreWaiters::Get();
+ if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address)) {
+ waiters.Add(this, *device, acquire.value, acquire.operation);
+ }
+ dma_pusher.rasterizer->ReleaseFences();
+ if (!waiters.Latched(this) && !acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) {
+ pending_acquire = acquire;
+ } else {
+ waiters.Remove(this);
+ }
} else {
do {
const u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress());
@@ -137,8 +283,43 @@ void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) {
}
}
+// Ampere SEM_EXECUTE (method 0x1B): operation in bits 2:0 on the semaphore at
+// SEM_ADDR_HI:SEM_ADDR_LO (methods 0x18/0x17) with payload SEM_PAYLOAD_LO (0x19).
+void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) {
+ const GPUVAddr address = (GPUVAddr{regs.reg_array[0x18] & 0x1FFFFFF} << 32) |
+ (regs.reg_array[0x17] & ~3U);
+ const u32 payload = regs.reg_array[0x19];
+ const u32 operation = argument & 7;
+ if (operation == 1) { // RELEASE
+ dma_pusher.rasterizer->Query(address, VideoCommon::QueryType::Payload,
+ VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+ return;
+ }
+ // Acquires: check once after flushing pending fences; never block the GPU thread on them.
+ dma_pusher.rasterizer->ReleaseFences();
+ const u32 word = dma_pusher.memory_manager.Read<u32>(address);
+ const bool ready = operation == 0 ? word == payload
+ : operation == 2 ? word >= payload
+ : operation == 3 ? static_cast<s32>(word - payload) >= 0
+ : operation == 4 ? (word & payload) != 0
+ : operation == 5 ? (word & payload) == 0
+ : true;
+ if (!ready) {
+ static std::atomic<u32> reported{};
+ if (reported.fetch_add(1) < 8) {
+ LOG_WARNING(HW_GPU, "SEM_EXECUTE op {} at {:#x}: {:#x} vs {:#x} not met, continuing",
+ operation, address, word, payload);
+ }
+ }
+}
+
/// Calls a GPU puller method.
void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_call) {
+ static const bool nx2_trace = std::getenv("EDEN_NX2_PULLER_TRACE") != nullptr;
+ if (nx2_trace) {
+ LOG_INFO(HW_GPU, "nx2 puller sub={} method={:#x} arg={:#x}", method_call.subchannel,
+ method_call.method, method_call.argument);
+ }
regs.reg_array[method_call.method] = method_call.argument;
const auto method = static_cast<BufferMethods>(method_call.method);
@@ -172,7 +353,7 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
break;
}
case BufferMethods::MemOpA: {
- LOG_ERROR(HW_GPU, "Memory Operation A");
+ LOG_TRACE(HW_GPU, "Memory Operation A");
break;
}
case BufferMethods::MemOpB: {
@@ -182,15 +363,22 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
}
case BufferMethods::MemOpC:
case BufferMethods::MemOpD: {
- LOG_ERROR(HW_GPU, "Memory Operation C,D");
+ LOG_TRACE(HW_GPU, "Memory Operation C,D");
break;
}
case BufferMethods::SemaphoreAcquire: {
- ProcessSemaphoreAcquire(dma_pusher);
+ // On Ampere host classes 0x17..0x1A are SEM_ADDR_LO/HI and SEM_PAYLOAD_LO/HI.
+ if (!g_nx2_ampere) {
+ ProcessSemaphoreAcquire(dma_pusher);
+ }
break;
}
case BufferMethods::SemaphoreRelease: {
- ProcessSemaphoreRelease(dma_pusher);
+ if (g_nx2_ampere) {
+ ProcessSemaphoreExecute(dma_pusher, method_call.argument);
+ } else {
+ ProcessSemaphoreRelease(dma_pusher);
+ }
break;
}
case BufferMethods::Yield: {
@@ -200,13 +388,27 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
}
default:
LOG_ERROR(HW_GPU, "Special puller engine method {:X} not implemented", method);
+ if (static bool dumped = false; g_nx2_ampere && !dumped) {
+ dumped = true;
+ dma_pusher.DumpCurrentEntry("unknown host method");
+ }
break;
}
}
/// Calls a GPU engine method.
+static bool Nx2MethodTrace() {
+ static const bool trace = std::getenv("EDEN_NX2_METHOD_TRACE") != nullptr;
+ return trace;
+}
+
void Puller::CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_call) {
const EngineID engine = bound_engines[method_call.subchannel];
+ if (Nx2MethodTrace()) {
+ LOG_INFO(HW_GPU, "nx2 method sub={} engine={:04X} method={:#x} arg={:#x}",
+ method_call.subchannel, static_cast<u32>(engine), method_call.method,
+ method_call.argument);
+ }
switch (engine) {
case EngineID::FERMI_TWOD_A:
dma_pusher.channel_state.payload->fermi_2d.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall());
@@ -235,6 +437,10 @@ void Puller::CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
/// Calls a GPU engine multivalue method.
void Puller::CallEngineMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending) {
const EngineID engine = bound_engines[subchannel];
+ if (Nx2MethodTrace()) {
+ LOG_INFO(HW_GPU, "nx2 multimethod sub={} engine={:04X} method={:#x} amount={} first={:#x}",
+ subchannel, static_cast<u32>(engine), method, amount, base_start[0]);
+ }
switch (engine) {
case EngineID::FERMI_TWOD_A:
dma_pusher.channel_state.payload->fermi_2d.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending);
diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h
index e8994f5640..2b9e2c5df3 100644
--- a/src/video_core/engines/puller.h
+++ b/src/video_core/engines/puller.h
@@ -7,7 +7,9 @@
#pragma once
#include <array>
+#include <chrono>
#include <cstddef>
+#include <optional>
#include <vector>
#include "common/bit_field.h"
#include "common/common_funcs.h"
@@ -70,6 +72,24 @@ public:
BitField<8, 24, u32> syncpoint_id;
};
+ /// Switch 2: a semaphore acquire this channel is suspended on.
+ struct PendingAcquire {
+ GPUVAddr address;
+ u32 value;
+ u32 operation;
+ std::chrono::steady_clock::time_point since;
+ bool reported;
+
+ [[nodiscard]] bool Matches(u32 current) const;
+ };
+
+ [[nodiscard]] bool HasPendingAcquire() const {
+ return pending_acquire.has_value();
+ }
+ [[nodiscard]] bool CanResumeAcquire(DmaPusher& dma_pusher);
+ /// Clears the pending acquire once it is met; false while the channel has to keep waiting.
+ bool ResumeAcquire(DmaPusher& dma_pusher);
+
void CallMethod(DmaPusher& dma_pusher, const MethodCall& method_call);
void CallMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending);
void BindRasterizer(DmaPusher& dma_pusher, VideoCore::RasterizerInterface* rasterizer);
@@ -125,8 +145,11 @@ private:
void ProcessSemaphoreAcquire(DmaPusher& dma_pusher);
void ProcessSemaphoreRelease(DmaPusher& dma_pusher);
void ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher);
+ void ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument);
[[nodiscard]] bool ExecuteMethodOnEngine(DmaPusher& dma_pusher, u32 method);
+ std::optional<PendingAcquire> pending_acquire;
+
/// Mapping of command subchannels to their bound engine ids
std::array<EngineID, 8> bound_engines{};
diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h
index 776c01e3fa..eb72529da0 100644
--- a/src/video_core/fence_manager.h
+++ b/src/video_core/fence_manager.h
@@ -68,6 +68,10 @@ public:
}
void SyncOperation(std::function<void()>&& func) {
+ // Not only the GPU thread queues these (query cache work runs elsewhere too), while
+ // SignalFence moves the list into pending_operations: unguarded, the list was corrupted
+ // and the fencing thread crashed walking it. Its own lock, held for no callbacks.
+ std::scoped_lock lock{uncommitted_guard};
uncommitted_operations.emplace_back(std::move(func));
}
@@ -77,15 +81,20 @@ public:
if constexpr (!can_async_check) {
TryReleasePendingFences<false>();
}
- CommitAsyncFlushes();
+ const u8 committed_flushes = CommitAsyncFlushes();
TFence new_fence = CreateFence(!should_flush);
if constexpr (can_async_check) {
guard.lock();
}
- if (delay_fence) {
- uncommitted_operations.emplace_back(std::move(func));
+ {
+ std::scoped_lock lock{uncommitted_guard};
+ if (delay_fence) {
+ uncommitted_operations.emplace_back(std::move(func));
+ }
+ pending_operations.emplace_back(std::move(uncommitted_operations));
+ uncommitted_operations.clear(); // moved-from: make it empty explicitly
}
- pending_operations.emplace_back(std::move(uncommitted_operations));
+ pending_flushes.push_back(committed_flushes);
QueueFence(new_fence);
if (!delay_fence) {
func();
@@ -178,7 +187,8 @@ private:
return;
}
}
- PopAsyncFlushes();
+ PopAsyncFlushes(pending_flushes.front());
+ pending_flushes.pop_front();
auto operations = std::move(pending_operations.front());
pending_operations.pop_front();
for (auto& operation : operations) {
@@ -199,6 +209,7 @@ private:
TFence current_fence;
std::deque<std::function<void()>> current_operations;
+ u8 current_flushes{};
while (!stop_token.stop_requested()) {
{
std::unique_lock lock(guard);
@@ -208,13 +219,15 @@ private:
}
current_fence = std::move(fences.front());
current_operations = std::move(pending_operations.front());
+ current_flushes = pending_flushes.front();
fences.pop();
pending_operations.pop_front();
+ pending_flushes.pop_front();
}
if (!current_fence->IsStubbed()) {
WaitFence(current_fence);
}
- PopAsyncFlushes();
+ PopAsyncFlushes(current_flushes);
for (auto& operation : current_operations) {
operation();
}
@@ -237,30 +250,55 @@ private:
query_cache.HasUncommittedFlushes();
}
- void PopAsyncFlushes() {
- {
- std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
+ static constexpr u8 TextureFlushes = 1;
+ static constexpr u8 BufferFlushes = 2;
+
+ /// committed: what CommitAsyncFlushes returned for this fence.
+ void PopAsyncFlushes(u8 committed) {
+ if (committed & TextureFlushes) {
+ std::scoped_lock lock{texture_cache.mutex};
texture_cache.PopAsyncFlushes();
+ }
+ if (committed & BufferFlushes) {
+ std::scoped_lock lock{buffer_cache.mutex};
buffer_cache.PopAsyncFlushes();
}
query_cache.PopAsyncFlushes();
}
- void CommitAsyncFlushes() {
+ /// Returns which caches committed downloads for this fence (TextureFlushes, BufferFlushes).
+ /// Switch 2 titles signal a fence per compute dispatch (thousands a second), mostly with
+ /// nothing or only buffer data to download: the fencing thread then takes only the lock of
+ /// the cache with work, instead of holding both while it copies (the GPU thread's draws and
+ /// uploads waited for it).
+ u8 CommitAsyncFlushes() {
+ u8 committed = 0;
{
- std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
- texture_cache.CommitAsyncFlushes();
- buffer_cache.CommitAsyncFlushes();
+ std::scoped_lock lock{texture_cache.mutex};
+ if (texture_cache.HasAsyncFlushWork()) {
+ texture_cache.CommitAsyncFlushes();
+ committed |= TextureFlushes;
+ }
+ }
+ {
+ std::scoped_lock lock{buffer_cache.mutex};
+ if (buffer_cache.HasUncommittedFlushes()) {
+ buffer_cache.CommitAsyncFlushes();
+ committed |= BufferFlushes;
+ }
}
query_cache.CommitAsyncFlushes();
+ return committed;
}
std::queue<TFence> fences;
std::deque<std::function<void()>> uncommitted_operations;
std::deque<std::deque<std::function<void()>>> pending_operations;
+ std::deque<u8> pending_flushes; ///< Per fence: the caches that committed downloads
std::mutex guard;
std::mutex ring_guard;
+ std::mutex uncommitted_guard; ///< Guards uncommitted_operations
std::condition_variable cv;
std::jthread fence_thread;
diff --git a/src/video_core/framebuffer_config.h b/src/video_core/framebuffer_config.h
index 1fc70583f9..bb041cf12f 100644
--- a/src/video_core/framebuffer_config.h
+++ b/src/video_core/framebuffer_config.h
@@ -37,6 +37,7 @@ struct FramebufferConfig {
Service::android::BufferTransformFlags transform_flags{};
Common::Rectangle<int> crop_rect{};
BlendMode blending{};
+ bool native_tiling{};
u32 layer_stack_mask{Service::Nvnflinger::DefaultLayerStackMask};
};
diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp
index 1621b24125..6441b9511a 100644
--- a/src/video_core/gpu.cpp
+++ b/src/video_core/gpu.cpp
@@ -206,6 +206,10 @@ struct GPU::Impl {
gpu_thread.SubmitList(channel, std::move(entries), is_async);
}
+ void RunInOrder(std::function<void()>&& func) {
+ gpu_thread.Run(std::move(func), is_async);
+ }
+
/// Notify rasterizer that any caches of the specified region should be flushed to Switch memory
void FlushRegion(DAddr addr, u64 size) {
gpu_thread.FlushRegion(addr, size, is_async);
@@ -407,6 +411,10 @@ u64 GPU::RequestFlush(DAddr addr, std::size_t size) {
});
}
+u64 GPU::RunOnGpuThread(std::function<void()>&& action) {
+ return impl->RequestSyncOperation(std::move(action));
+}
+
u64 GPU::CurrentSyncRequestFence() const {
return impl->CurrentSyncRequestFence();
}
@@ -530,6 +538,10 @@ void GPU::FlushRegion(DAddr addr, u64 size) {
impl->FlushRegion(addr, size);
}
+void GPU::RunInOrder(std::function<void()>&& func) {
+ impl->RunInOrder(std::move(func));
+}
+
void GPU::InvalidateRegion(DAddr addr, u64 size, bool preserve_gpu_writes) {
impl->InvalidateRegion(addr, size, preserve_gpu_writes);
}
diff --git a/src/video_core/gpu.h b/src/video_core/gpu.h
index 974d214e10..3a6546b41b 100644
--- a/src/video_core/gpu.h
+++ b/src/video_core/gpu.h
@@ -6,6 +6,7 @@
#pragma once
+#include <functional>
#include <memory>
#include "common/bit_field.h"
@@ -163,6 +164,9 @@ public:
/// Request a host GPU memory flush from the CPU.
[[nodiscard]] u64 RequestFlush(DAddr addr, std::size_t size);
+ /// Runs an action on the GPU thread between command lists (returns its sync fence).
+ u64 RunOnGpuThread(std::function<void()>&& action);
+
/// Obtains current flush request fence id.
[[nodiscard]] u64 CurrentSyncRequestFence() const;
@@ -247,6 +251,9 @@ public:
/// Notify rasterizer that any caches of the specified region should be flushed to Switch memory
void FlushRegion(DAddr addr, u64 size);
+ /// Runs func on the GPU thread after the command lists submitted so far.
+ void RunInOrder(std::function<void()>&& func);
+
/// Notify rasterizer that any caches of the specified region should be invalidated
void InvalidateRegion(DAddr addr, u64 size, bool preserve_gpu_writes = false);
diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp
index a01375e7be..72da1904e1 100644
--- a/src/video_core/gpu_thread.cpp
+++ b/src/video_core/gpu_thread.cpp
@@ -4,6 +4,9 @@
// SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <chrono>
+#include <thread>
+
#include "common/assert.h"
#include "common/scope_exit.h"
#include "common/settings.h"
@@ -14,6 +17,7 @@
#include "video_core/dma_pusher.h"
#include "video_core/gpu.h"
#include "video_core/gpu_thread.h"
+#include "video_core/nx2_profile.h"
#include "video_core/host1x/host1x.h"
#include "video_core/renderer_base.h"
@@ -35,18 +39,52 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten
auto current_context = context.Acquire();
CommandDataContainer next;
+ // When a pending channel last moved or a command arrived (see the polling below).
+ auto last_progress = std::chrono::steady_clock::now();
+ namespace Profile = VideoCommon::Nx2Profile;
while (!stop_token.stop_requested()) {
- state.queue.PopWait(next, stop_token);
+ Profile::Report();
+ if (scheduler.HasPendingChannels()) {
+ // Suspended channels wait on semaphores the CPU releases: poll them meanwhile.
+ {
+ Profile::Scope scope{Profile::Zone::ResumePending};
+ scheduler.ResumePending(system.GPU());
+ }
+ if (!state.queue.TryPop(next)) {
+ // The CPU usually releases the semaphore soon after: poll for half a
+ // millisecond before sleeping, or every wait also waits out a host sleep
+ // (title 1 waits ~25 times a second).
+ Profile::Scope scope{Profile::Zone::PendingSleep};
+ if (std::chrono::steady_clock::now() - last_progress <
+ std::chrono::microseconds{500}) {
+ std::this_thread::yield();
+ } else {
+ std::this_thread::sleep_for(std::chrono::milliseconds{1});
+ }
+ continue;
+ }
+ } else {
+ Profile::Scope scope{Profile::Zone::Idle};
+ state.queue.PopWait(next, stop_token);
+ }
if (stop_token.stop_requested()) {
break;
}
+ last_progress = std::chrono::steady_clock::now();
if (auto* submit_list = std::get_if<SubmitListCommand>(&next.data)) {
+ Profile::Scope scope{Profile::Zone::Push};
scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries));
} else if (std::holds_alternative<GPUTickCommand>(next.data)) {
+ Profile::Scope scope{Profile::Zone::Tick};
system.GPU().TickWork();
+ } else if (auto* run = std::get_if<RunCommand>(&next.data)) {
+ Profile::Scope scope{Profile::Zone::Run};
+ run->func();
} else if (const auto* flush = std::get_if<FlushRegionCommand>(&next.data)) {
+ Profile::Scope scope{Profile::Zone::FlushInvalidate};
renderer.ReadRasterizer()->FlushRegion(flush->addr, flush->size);
} else if (const auto* invalidate = std::get_if<InvalidateRegionCommand>(&next.data)) {
+ Profile::Scope scope{Profile::Zone::FlushInvalidate};
renderer.ReadRasterizer()->OnCacheInvalidation(invalidate->addr, invalidate->size);
} else {
ASSERT(false);
@@ -77,6 +115,10 @@ void ThreadManager::TickGPU(bool is_async) {
PushCommand(GPUTickCommand(), false, is_async);
}
+void ThreadManager::Run(std::function<void()>&& func, bool is_async) {
+ PushCommand(RunCommand{std::move(func)}, false, is_async);
+}
+
void ThreadManager::InvalidateRegion(DAddr addr, u64 size) {
rasterizer->OnCacheInvalidation(addr, size);
}
diff --git a/src/video_core/gpu_thread.h b/src/video_core/gpu_thread.h
index 631a680db4..796c0d89f6 100644
--- a/src/video_core/gpu_thread.h
+++ b/src/video_core/gpu_thread.h
@@ -6,6 +6,7 @@
#pragma once
+#include <functional>
#include <atomic>
#include <condition_variable>
#include <mutex>
@@ -76,9 +77,14 @@ struct FlushAndInvalidateRegionCommand final {
/// Command to make the gpu look into pending requests
struct GPUTickCommand final {};
+/// Command to run a function on the GPU thread after the commands queued before it
+struct RunCommand final {
+ std::function<void()> func;
+};
+
using CommandData =
std::variant<std::monostate, SubmitListCommand, FlushRegionCommand, InvalidateRegionCommand,
- FlushAndInvalidateRegionCommand, GPUTickCommand>;
+ FlushAndInvalidateRegionCommand, GPUTickCommand, RunCommand>;
struct CommandDataContainer {
CommandDataContainer() = default;
@@ -125,6 +131,9 @@ public:
void TickGPU(bool is_async);
+ /// Runs func on the GPU thread once the commands queued before it have been processed.
+ void Run(std::function<void()>&& func, bool is_async);
+
void NotifyShutdown();
private:
diff --git a/src/video_core/host1x/gpu_device_memory_manager.h b/src/video_core/host1x/gpu_device_memory_manager.h
index a9f249991f..1b4f57fe24 100644
--- a/src/video_core/host1x/gpu_device_memory_manager.h
+++ b/src/video_core/host1x/gpu_device_memory_manager.h
@@ -14,7 +14,9 @@ namespace Tegra {
struct MaxwellDeviceMethods;
struct MaxwellDeviceTraits {
- static constexpr size_t device_virtual_bits = 34;
+ // 64 GiB: Switch 2 titles map most of their memory twice plus GPU-only regions (title 1
+ // ran out of 16 GiB).
+ static constexpr size_t device_virtual_bits = 36;
using DeviceInterface = typename VideoCore::RasterizerInterface;
using DeviceMethods = MaxwellDeviceMethods;
};
diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt
index c4e7ac3f6f..8082e830d6 100644
--- a/src/video_core/host_shaders/CMakeLists.txt
+++ b/src/video_core/host_shaders/CMakeLists.txt
@@ -15,6 +15,7 @@ set(GLSL_INCLUDES
set(SHADER_FILES
${CMAKE_CURRENT_SOURCE_DIR}/astc_decoder.comp
+ ${CMAKE_CURRENT_SOURCE_DIR}/bc_encoder.comp
${CMAKE_CURRENT_SOURCE_DIR}/blit_color_float.frag
${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d.comp
${CMAKE_CURRENT_SOURCE_DIR}/blit_color_msaa.frag
diff --git a/src/video_core/host_shaders/astc_decoder.comp b/src/video_core/host_shaders/astc_decoder.comp
index e059c683ff..01bec11f17 100644
--- a/src/video_core/host_shaders/astc_decoder.comp
+++ b/src/video_core/host_shaders/astc_decoder.comp
@@ -33,6 +33,7 @@ UNIFORM(3) uint block_size;
UNIFORM(4) uint x_shift;
UNIFORM(5) uint block_height;
UNIFORM(6) uint block_height_mask;
+UNIFORM(7) uint native_tiling;
END_PUSH_CONSTANTS
struct EncodingData {
@@ -1404,7 +1405,12 @@ uint SwizzleOffset(uvec2 pos) {
void main() {
uvec3 pos = gl_GlobalInvocationID;
pos.x <<= BYTES_PER_BLOCK_LOG2;
- const uint swizzle = SwizzleOffset(pos.xy);
+ uint swizzle = SwizzleOffset(pos.xy);
+ if (native_tiling != 0u) {
+ // Switch 2 GOBs swap offset bits 5 and 6 (see block_linear_unswizzle_2d.comp).
+ const uint difference = ((swizzle >> 5u) ^ (swizzle >> 6u)) & 1u;
+ swizzle ^= (difference << 5u) | (difference << 6u);
+ }
const uint block_y = pos.y >> GOB_SIZE_Y_SHIFT;
uint offset = 0;
diff --git a/src/video_core/host_shaders/bc_encoder.comp b/src/video_core/host_shaders/bc_encoder.comp
new file mode 100644
index 0000000000..3a87e92dcb
--- /dev/null
+++ b/src/video_core/host_shaders/bc_encoder.comp
@@ -0,0 +1,183 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Compresses one level of an RGBA8 image (the ASTC decoder's output) to BC1 or BC3, one 4x4 block
+// per invocation, so re-compressed ASTC never round-trips through the CPU. Colour endpoints are
+// the block's bounding box, oriented along the colours' covariance and inset by 1/16; each texel
+// takes the nearest palette entry. Values are encoded as stored (sRGB data stays sRGB, like the
+// CPU path).
+
+#version 460 core
+
+layout (local_size_x = 8, local_size_y = 8, local_size_z = 1) in;
+
+layout (set = 0, binding = 0, rgba8) uniform readonly image2DArray source;
+
+layout (std430, set = 0, binding = 1) writeonly buffer Output {
+ uvec2 blocks[];
+};
+
+layout (push_constant) uniform PushConstants {
+ uint blocks_x;
+ uint blocks_y;
+ uint width;
+ uint height;
+ uint bc3;
+ uint out_offset; // in 8-byte units
+};
+
+uint Pack565(vec3 color) {
+ const uvec3 q = uvec3(round(clamp(color, 0.0, 1.0) * vec3(31.0, 63.0, 31.0)));
+ return (q.r << 11) | (q.g << 5) | q.b;
+}
+
+vec3 Unpack565(uint value) {
+ const uvec3 q = uvec3((value >> 11) & 31u, (value >> 5) & 63u, value & 31u);
+ const uvec3 e = uvec3((q.r << 3) | (q.r >> 2), (q.g << 2) | (q.g >> 4), (q.b << 3) | (q.b >> 2));
+ return vec3(e) / 255.0;
+}
+
+float Distance2(vec3 a, vec3 b) {
+ const vec3 d = a - b;
+ return dot(d, d);
+}
+
+// BC1 colour block. With punch_through (BC1 only), texels with alpha < 0.5 use the transparent
+// index of the three-colour mode; otherwise the four-colour mode is used (BC3 always decodes so).
+uvec2 EncodeColor(in vec4 texels[16], bool punch_through) {
+ vec3 lo = vec3(1.0);
+ vec3 hi = vec3(0.0);
+ for (int i = 0; i < 16; ++i) {
+ if (punch_through && texels[i].a < 0.5) {
+ continue;
+ }
+ lo = min(lo, texels[i].rgb);
+ hi = max(hi, texels[i].rgb);
+ }
+ if (any(greaterThan(lo, hi))) {
+ // Every texel transparent: colour0 == colour1 selects three-colour mode, index 3.
+ return uvec2(0u, 0xFFFFFFFFu);
+ }
+ const vec3 center = (lo + hi) * 0.5;
+ const vec3 extent = hi - lo;
+ const int major = extent.g >= extent.r && extent.g >= extent.b ? 1 : (extent.r >= extent.b ? 0 : 2);
+ vec3 covariance = vec3(0.0);
+ for (int i = 0; i < 16; ++i) {
+ if (punch_through && texels[i].a < 0.5) {
+ continue;
+ }
+ const vec3 d = texels[i].rgb - center;
+ covariance += d * d[major];
+ }
+ vec3 e0 = hi;
+ vec3 e1 = lo;
+ for (int c = 0; c < 3; ++c) {
+ if (covariance[c] < 0.0) {
+ e0[c] = lo[c];
+ e1[c] = hi[c];
+ }
+ }
+ const vec3 inset = (e0 - e1) / 16.0;
+ e0 -= inset;
+ e1 += inset;
+
+ uint q0 = Pack565(e0);
+ uint q1 = Pack565(e1);
+ // Four-colour mode needs colour0 > colour1, three-colour mode colour0 <= colour1.
+ if (punch_through ? q0 > q1 : q0 < q1) {
+ const uint t = q0;
+ q0 = q1;
+ q1 = t;
+ }
+ if (!punch_through && q0 == q1) {
+ return uvec2(q0 | (q1 << 16), 0u);
+ }
+ vec3 palette[4];
+ palette[0] = Unpack565(q0);
+ palette[1] = Unpack565(q1);
+ if (punch_through) {
+ palette[2] = (palette[0] + palette[1]) * 0.5;
+ palette[3] = vec3(0.0);
+ } else {
+ palette[2] = (2.0 * palette[0] + palette[1]) / 3.0;
+ palette[3] = (palette[0] + 2.0 * palette[1]) / 3.0;
+ }
+ const int entries = punch_through ? 3 : 4;
+ uint indices = 0u;
+ for (int i = 0; i < 16; ++i) {
+ uint best = 0u;
+ if (punch_through && texels[i].a < 0.5) {
+ best = 3u;
+ } else {
+ float best_distance = Distance2(texels[i].rgb, palette[0]);
+ for (int p = 1; p < entries; ++p) {
+ const float distance = Distance2(texels[i].rgb, palette[p]);
+ if (distance < best_distance) {
+ best_distance = distance;
+ best = uint(p);
+ }
+ }
+ }
+ indices |= best << (2 * i);
+ }
+ return uvec2(q0 | (q1 << 16), indices);
+}
+
+// BC3 alpha block: eight-value mode (alpha0 > alpha1), indices 0 = alpha0, 1 = alpha1, 2..7 the
+// ramp from alpha0 to alpha1.
+uvec2 EncodeAlpha(in vec4 texels[16]) {
+ float lo = 1.0;
+ float hi = 0.0;
+ for (int i = 0; i < 16; ++i) {
+ lo = min(lo, texels[i].a);
+ hi = max(hi, texels[i].a);
+ }
+ const uint a0 = uint(round(hi * 255.0));
+ const uint a1 = uint(round(lo * 255.0));
+ uint bits_lo = a0 | (a1 << 8);
+ uint bits_hi = 0u;
+ if (a0 == a1) {
+ return uvec2(bits_lo, bits_hi);
+ }
+ for (uint i = 0u; i < 16u; ++i) {
+ const float t = (texels[i].a * 255.0 - float(a1)) / float(a0 - a1);
+ const uint q = uint(clamp(round(t * 7.0), 0.0, 7.0));
+ const uint index = q == 7u ? 0u : (q == 0u ? 1u : 8u - q);
+ const uint bit = 16u + 3u * i;
+ if (bit < 32u) {
+ bits_lo |= index << bit;
+ if (bit > 29u) {
+ bits_hi |= index >> (32u - bit);
+ }
+ } else {
+ bits_hi |= index << (bit - 32u);
+ }
+ }
+ return uvec2(bits_lo, bits_hi);
+}
+
+void main() {
+ const uvec3 id = gl_GlobalInvocationID;
+ if (id.x >= blocks_x || id.y >= blocks_y) {
+ return;
+ }
+ vec4 texels[16];
+ const ivec2 base = ivec2(id.xy) * 4;
+ const ivec2 last = ivec2(width, height) - 1;
+ bool any_transparent = false;
+ for (int y = 0; y < 4; ++y) {
+ for (int x = 0; x < 4; ++x) {
+ // Edge blocks repeat the last row/column so padding doesn't widen the endpoints.
+ const ivec2 coord = min(base + ivec2(x, y), last);
+ texels[y * 4 + x] = imageLoad(source, ivec3(coord, int(id.z)));
+ any_transparent = any_transparent || texels[y * 4 + x].a < 0.5;
+ }
+ }
+ const uint block = (id.z * blocks_y + id.y) * blocks_x + id.x;
+ if (bc3 != 0u) {
+ blocks[out_offset + block * 2u] = EncodeAlpha(texels);
+ blocks[out_offset + block * 2u + 1u] = EncodeColor(texels, false);
+ } else {
+ blocks[out_offset + block] = EncodeColor(texels, any_transparent);
+ }
+}
diff --git a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp
index 2a3446ca66..bbe73acd1a 100644
--- a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp
+++ b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp
@@ -39,6 +39,7 @@ UNIFORM(4) uint block_size;
UNIFORM(5) uint x_shift;
UNIFORM(6) uint block_height;
UNIFORM(7) uint block_height_mask;
+UNIFORM(8) uint native_tiling;
END_PUSH_CONSTANTS
#if HAS_EXTENDED_TYPES
@@ -77,7 +78,12 @@ uint SwizzleTable(uint pos) {
uint SwizzleOffset(uvec2 pos) {
pos = pos & SWIZZLE_MASK;
- return SwizzleTable(pos.y * 64 + pos.x);
+ uint offset = SwizzleTable(pos.y * 64 + pos.x);
+ if (native_tiling != 0) {
+ uint difference = ((offset >> 5) ^ (offset >> 6)) & 1;
+ offset ^= (difference << 5) | (difference << 6);
+ }
+ return offset;
}
uvec4 ReadTexel(uint offset) {
diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp
index 34e4f93307..8dfaf0cd1a 100644
--- a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp
+++ b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp
@@ -41,6 +41,7 @@ UNIFORM(6) uint block_height;
UNIFORM(7) uint block_height_mask;
UNIFORM(8) uint block_depth;
UNIFORM(9) uint block_depth_mask;
+UNIFORM(10) uint native_tiling;
END_PUSH_CONSTANTS
#if HAS_EXTENDED_TYPES
@@ -79,7 +80,12 @@ uint SwizzleTable(uint pos) {
uint SwizzleOffset(uvec2 pos) {
pos = pos & SWIZZLE_MASK;
- return SwizzleTable(pos.y * 64 + pos.x);
+ uint offset = SwizzleTable(pos.y * 64 + pos.x);
+ if (native_tiling != 0) {
+ uint difference = ((offset >> 5) ^ (offset >> 6)) & 1;
+ offset ^= (difference << 5) | (difference << 6);
+ }
+ return offset;
}
uvec4 ReadTexel(uint offset) {
diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp
index e084837f25..28b1dcf599 100644
--- a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp
+++ b/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp
@@ -42,7 +42,7 @@ layout(push_constant) uniform PushConstants {
uint block_depth; // Offset 48
uint block_depth_mask; // Offset 52
- int _pad; // Offset 56
+ uint native_tiling; // Offset 56
ivec3 destination; // Offset 60
} pc;
@@ -59,6 +59,7 @@ BEGIN_PUSH_CONSTANTS
UNIFORM(8) uint block_depth;
UNIFORM(9) uint block_depth_mask;
UNIFORM(10) uvec3 blocks_dim;
+ UNIFORM(11) uint native_tiling;
END_PUSH_CONSTANTS
#define pc // Map pc prefix to nothing for OpenGL compatibility
#endif
@@ -103,7 +104,12 @@ uint SwizzleTable(uint pos) {
// --- Helpers ---
uint SwizzleOffset(uvec2 pos) {
pos &= SWIZZLE_MASK;
- return SwizzleTable(pos.y * 64u + pos.x);
+ uint offset = SwizzleTable(pos.y * 64 + pos.x);
+ if (pc.native_tiling != 0) {
+ uint difference = ((offset >> 5) ^ (offset >> 6)) & 1;
+ offset ^= (difference << 5) | (difference << 6);
+ }
+ return offset;
}
uvec4 ReadTexel(uint offset) {
diff --git a/src/video_core/macro.cpp b/src/video_core/macro.cpp
index 86f528af15..3422cb16fa 100644
--- a/src/video_core/macro.cpp
+++ b/src/video_core/macro.cpp
@@ -4,6 +4,9 @@
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <array>
+#include <cstdlib>
+#include <unordered_set>
#include <cstring>
#include <fstream>
#include <optional>
@@ -25,6 +28,7 @@
#include "video_core/engines/maxwell_3d.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/dirty_flags.h"
+#include "video_core/memory_manager.h"
#include "video_core/rasterizer_interface.h"
#include "video_core/macro.h"
@@ -32,6 +36,7 @@
#include "common/bit_field.h"
#include "common/logging.h"
#include "video_core/engines/maxwell_3d.h"
+#include "video_core/nx2_profile.h"
namespace Tegra {
@@ -1339,6 +1344,173 @@ static void Dump(u64 hash, std::span<const u32> code, bool decompiled = false) {
void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u32 method,
std::span<const u32> parameters) {
+ VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro};
+ if (native_turing) {
+ maxwell3d.RefreshParameters();
+ const auto context = reinterpret_cast<std::uintptr_t>(this);
+ static std::unordered_set<std::uintptr_t> captured_contexts;
+ static const bool capture = std::getenv("EDEN_NATIVE_FRAME_CAPTURE") != nullptr;
+ if (capture && captured_contexts.insert(context).second) {
+ std::ofstream dump("native-graphics-mme-" + std::to_string(context) + ".bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code));
+ }
+ // EDEN_NX2_MME_LOG: log every macro call (runs ~750,000 times a second in title 1).
+ if (static const bool log_calls = std::getenv("EDEN_NX2_MME_LOG") != nullptr; log_calls) {
+ LOG_DEBUG(HW_GPU, "Native MME context={:#x}", context);
+ LOG_DEBUG(HW_GPU, "Native MME start={:#x}, parameters={}, data={:08x}", method, parameters.size(), fmt::join(parameters, ","));
+ }
+ // Reused between calls (a macro never calls another: methods >= 0xe00 are refused below);
+ // filled only for calls that are interpreted.
+ auto& fifo = native_fifo;
+ native_interpreter.active_dram = &maxwell3d.GetNativeDataRam();
+ // EDEN_NX2_MME_RECORD=<file>: write the first 200000 calls (u32 start, u32 count, the
+ // parameters) and then the macro code to <file>.code, to check interpreter changes
+ // offline against the previous interpreter. EDEN_NX2_MME_RECORD_SKIP=<n>: skip the first
+ // n calls (title 1 makes ~1,000,000 a second in 3D scenes).
+ if (static const char* const record_path = std::getenv("EDEN_NX2_MME_RECORD"); record_path != nullptr) {
+ static std::ofstream record{record_path, std::ios::binary};
+ static u32 recorded{};
+ static u64 to_skip = [] {
+ const char* value = std::getenv("EDEN_NX2_MME_RECORD_SKIP");
+ return value != nullptr ? std::strtoull(value, nullptr, 10) : 0ULL;
+ }();
+ if (to_skip > 0) {
+ --to_skip;
+ } else if (recorded < 200000) {
+ const u32 count = static_cast<u32>(parameters.size());
+ record.write(reinterpret_cast<const char*>(&method), sizeof(method));
+ record.write(reinterpret_cast<const char*>(&count), sizeof(count));
+ record.write(reinterpret_cast<const char*>(parameters.data()), parameters.size_bytes());
+ if (++recorded == 200000) {
+ record.flush();
+ std::ofstream code{std::string{record_path} + ".code", std::ios::binary};
+ code.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code));
+ LOG_INFO(HW_GPU, "Recorded 200000 macro calls to {}", record_path);
+ }
+ }
+ }
+ // A method write of the macro (also what a remembered call replays).
+ bool read_memory = false;
+ const auto emit = [&](u32 address, u32 value) {
+ if (address >= 0xe00) return false;
+ if (address == 0x556) {
+ // SET_RENDER_ENABLE_OVERRIDE: 0 use the render-enable condition, 1 always,
+ // 2 never. Log a histogram every 20000 macro writes (pop-in diagnostics).
+ static std::array<u64, 4> modes{};
+ static u64 enable_writes{};
+ ++modes[value & 3];
+ if (++enable_writes % 20000 == 0) {
+ LOG_INFO(HW_GPU, "Native MME render enable modes: use {} always {} never {} other {}",
+ modes[0], modes[1], modes[2], modes[3]);
+ }
+ }
+ maxwell3d.CallMethod(system, address, value, true);
+ // Turing MME memory reads (methods 0x550 SET_MME_MEM_ADDRESS_A/B, 0x558 data RAM
+ // address, 0x55c MME_DMA_READ into data RAM, 0x560 MME_DMA_READ_FIFOED into
+ // the load FIFO, which the macro then reads after a load barrier).
+ constexpr u32 MemAddressA = 0x154, MemAddressB = 0x155, DataRamAddress = 0x156;
+ constexpr u32 DmaRead = 0x157, DmaReadFifoed = 0x158;
+ if (address == DmaRead || address == DmaReadFifoed) {
+ read_memory = true;
+ const GPUVAddr source = (GPUVAddr{maxwell3d.GetRegisterValue(MemAddressA) & 0x1ffffff} << 32) |
+ maxwell3d.GetRegisterValue(MemAddressB);
+ auto& memory = maxwell3d.GetMemoryManager();
+ const u32 ram = maxwell3d.GetRegisterValue(DataRamAddress);
+ // The words may have just been written by the GPU (e.g. indirect draw arguments
+ // from a culling dispatch) and still be only in host buffers: flush them first,
+ // or the macro draws with an older frame's arguments (objects pop in and out).
+ const size_t read_bytes = size_t{std::min<u32>(value, 0x1000)} * 4;
+ if (memory.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) {
+ memory.FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache);
+ static u64 flushes{};
+ if (++flushes % 2000 == 1) {
+ LOG_INFO(HW_GPU, "Native MME DMA read flushed GPU-written memory ({} times)", flushes);
+ }
+ }
+ static u32 dma_logs{};
+ if (dma_logs < 400) {
+ ++dma_logs;
+ std::array<u32, 8> words{};
+ for (u32 i = 0; i < std::min<u32>(value, 8); ++i) words[i] = memory.Read<u32>(source + u64{i} * 4);
+ LOG_INFO(HW_GPU, "Native MME DMA read{} start={:#x} source={:#x} count={} words={:08x}",
+ address == DmaReadFifoed ? " fifoed" : "", method, source, value, fmt::join(words, ","));
+ }
+ for (u32 i = 0; i < std::min<u32>(value, 0x1000); ++i) {
+ const u32 word = memory.Read<u32>(source + u64{i} * 4);
+ if (address == DmaReadFifoed) {
+ fifo.push_back(word);
+ } else if (ram + i < native_interpreter.active_dram->size()) {
+ if (static const bool mme_dump = std::getenv("EDEN_NX2_MME_DUMP") != nullptr;
+ mme_dump && ram + i == 0x27 && (*native_interpreter.active_dram)[0x27] != word) {
+ LOG_INFO(HW_GPU, "MME data RAM[0x27] {:#x} -> {:#x} by 3D engine {} (DMA from {:#x})",
+ (*native_interpreter.active_dram)[0x27], word, fmt::ptr(&maxwell3d), source);
+ }
+ (*native_interpreter.active_dram)[ram + i] = word;
+ }
+ }
+ }
+ return true;
+ };
+ // EDEN_NX2_MME_MEMO=0: interpret every call (see macro_memo.h).
+ static const bool use_memo = [] {
+ const char* value = std::getenv("EDEN_NX2_MME_MEMO");
+ return value == nullptr || value[0] != '0';
+ }();
+ if (use_memo) {
+ // EDEN_NX2_MME_MEMO_STATS: log the memo's counters every 2^24 calls.
+ if (static const bool memo_stats = std::getenv("EDEN_NX2_MME_MEMO_STATS") != nullptr;
+ memo_stats && (++native_calls & 0xffffff) == 0) {
+ const auto& stats = native_memo.GetStatistics();
+ LOG_INFO(HW_GPU,
+ "Macro memo: {} hits, {} misses, {} guard failures, {} stores ({} in "
+ "place), {} clears; {} entries, {} KiB",
+ stats.hits, stats.misses, stats.guard_failures, stats.stores,
+ stats.overwrites, stats.clears, native_memo.Size(),
+ native_memo.ArenaWords() * 4 / 1024);
+ }
+ native_memo.Synchronize(native_code_generation);
+ if (decltype(native_memo)::Effects effects;
+ native_memo.Find(method, parameters, *native_interpreter.active_dram, effects)) {
+ auto& data_ram = *native_interpreter.active_dram;
+ for (size_t i = 0; i < effects.data_ram_writes.size(); i += 2) {
+ data_ram[effects.data_ram_writes[i]] = effects.data_ram_writes[i + 1];
+ }
+ for (size_t i = 0; i < effects.writes.size(); i += 2) {
+ emit(effects.writes[i], effects.writes[i + 1]);
+ }
+ return;
+ }
+ native_writes.clear();
+ }
+ fifo.assign(parameters.begin(), parameters.end());
+ const u64 code_generation = native_code_generation;
+ const auto result = native_interpreter.Execute(native_code, method, fifo,
+ [&](u32 address) { return address < Maxwell3D::Regs::NUM_REGS ? maxwell3d.GetRegisterValue(address) : 0U; },
+ [&](u32 address, u32 value) {
+ if (use_memo) {
+ native_writes.emplace_back(address, value);
+ }
+ return emit(address, value);
+ });
+ // Remember the call if its effects depend only on its parameters and the data RAM words
+ // it read (and the macro code did not change under it).
+ if (use_memo && result.success && !read_memory && !native_interpreter.read_registers &&
+ !native_interpreter.reread_data_ram && code_generation == native_code_generation) {
+ native_memo.Store(method, parameters, native_interpreter.data_ram_reads,
+ native_interpreter.data_ram_writes, native_writes);
+ }
+ if (!result.success) {
+ native_execution_failed = true;
+ if (static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr;
+ diagnostics) {
+ std::ofstream dump("native-mme-" + std::to_string(method) + ".bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code));
+ }
+ LOG_ERROR(HW_GPU, "Native MME failed: start={:#x}, pc={:#x}, consumed={}/{}, error={}",
+ method, result.pc, result.consumed, parameters.size(), result.error);
+ }
+ return;
+ }
const auto execute_variant = [&system, &maxwell3d, &parameters,
method](AnyCachedMacro& cached) {
if (std::holds_alternative<MacroInterpreterImpl>(cached) ||
diff --git a/src/video_core/macro.h b/src/video_core/macro.h
index 90519a5429..a6ca4c4bd2 100644
--- a/src/video_core/macro.h
+++ b/src/video_core/macro.h
@@ -7,6 +7,8 @@
#pragma once
#include <memory>
+#include "video_core/macro_memo.h"
+#include "video_core/macro_turing.h"
#include <span>
#include <variant>
#include <vector>
@@ -223,10 +225,16 @@ struct MacroEngine {
MacroEngine(bool is_interpreted_) noexcept : is_interpreted{is_interpreted_} {}
// Store the uploaded macro code to compile them when they're called.
inline void AddCode(u32 method, u32 data) noexcept {
+ if (native_turing) {
+ if (native_write_offset < native_code.size()) native_code[native_write_offset++] = data;
+ ++native_code_generation;
+ return;
+ }
uploaded_macro_code[method].push_back(data);
}
// Clear the code associated with a method.
inline void ClearCode(u32 method) noexcept {
+ if (native_turing) { native_write_offset = method < 0x1000 ? method * 3 : static_cast<u32>(native_code.size()); return; }
macro_cache.erase(method);
uploaded_macro_code.erase(method);
}
@@ -240,6 +248,25 @@ struct MacroEngine {
::Common::unordered_map<u32, CacheInfo> macro_cache;
::Common::unordered_map<u32, std::vector<u32>> uploaded_macro_code;
bool is_interpreted;
+ bool native_turing{};
+ bool native_execution_failed{};
+ u32 native_write_offset{};
+ std::array<u32, 0x3000> native_code{};
+ TuringMacroInterpreter native_interpreter;
+ std::vector<u32> native_fifo;
+ // Calls replayed instead of interpreted (macro_memo.h), and what the current call wrote.
+ u64 native_code_generation{};
+ BasicMacroMemo<::Common::unordered_map<u64, u32, MacroMemoHash>> native_memo;
+ MacroMemoPairs native_writes;
+ u64 native_calls{};
+
+};
+
+struct NativeMacroContext {
+ MacroEngine engine{true};
+ std::array<u32, 0x80> positions{};
+ MacroEngine compute_engine{true};
+ std::array<u32, 0x80> compute_positions{};
};
} // namespace Tegra
diff --git a/src/video_core/macro_memo.h b/src/video_core/macro_memo.h
new file mode 100644
index 0000000000..0e994fc78a
--- /dev/null
+++ b/src/video_core/macro_memo.h
@@ -0,0 +1,197 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <span>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+namespace Tegra {
+
+using MacroMemoPairs = std::vector<std::pair<std::uint32_t, std::uint32_t>>;
+
+/// The memo's keys are already well mixed hashes: use them as they are (std::hash<u64> is a
+/// byte-wise FNV-1a on MSVC, ~20% of a lookup).
+struct MacroMemoHash {
+ using is_avalanching = std::true_type;
+ std::size_t operator()(std::uint64_t key) const noexcept {
+ return static_cast<std::size_t>(key);
+ }
+};
+
+// The effects of native macro calls that depend on nothing but their start address, their
+// parameters and data RAM words they read (no register or memory reads), so that a repeated
+// call replays its data RAM and method writes instead of being interpreted. In title 1's 3D
+// scenes ~96% of the ~1,000,000 calls a second repeat an earlier one exactly, and they run 7
+// instructions on average: the per-call interpreter cost, not the instructions, was the time.
+// `Map` maps a u64 (the call's hash) to a u32 (its record's offset in the arena).
+template <typename Map>
+class BasicMacroMemo {
+public:
+ /// A remembered call's effects: (address, value) pairs, flattened.
+ struct Effects {
+ std::span<const std::uint32_t> data_ram_writes;
+ std::span<const std::uint32_t> writes; ///< (method, value) in order
+ };
+
+ struct Statistics {
+ std::uint64_t hits{}, misses{}, guard_failures{}, stores{}, overwrites{}, clears{};
+ };
+
+ BasicMacroMemo() {
+ last_by_start.fill({0, NoRecord});
+ }
+
+ /// An earlier call with this start and these parameters whose data RAM reads still hold.
+ bool Find(std::uint32_t start, std::span<const std::uint32_t> parameters,
+ std::span<const std::uint32_t> data_ram, Effects& effects) {
+ // Nearly half the calls repeat the previous call of the same macro: try its record before
+ // hashing (no probe, and the record is still in the cache).
+ auto& last = last_by_start[start % last_by_start.size()];
+ if (last.first == start && last.second != NoRecord &&
+ Match(last.second, start, parameters, data_ram, effects)) {
+ ++statistics.hits;
+ return true;
+ }
+ const auto it = entries.find(Hash(start, parameters));
+ if (it == entries.end()) {
+ ++statistics.misses;
+ return false;
+ }
+ if (!Match(it->second, start, parameters, data_ram, effects)) {
+ ++statistics.guard_failures; // or another call with the same hash
+ return false;
+ }
+ last = {start, it->second};
+ ++statistics.hits;
+ return true;
+ }
+
+private:
+ static constexpr std::uint32_t NoRecord = ~0U;
+
+ /// Whether the record at `offset` is this call with its data RAM reads holding; fills
+ /// `effects` if so.
+ bool Match(std::uint32_t offset, std::uint32_t start, std::span<const std::uint32_t> parameters,
+ std::span<const std::uint32_t> data_ram, Effects& effects) {
+ // Record: start, parameter count, read count, data RAM write count, write count, then
+ // the parameters and the (address, value) pairs.
+ const std::uint32_t* record = arena.data() + offset;
+ const std::uint32_t num_parameters = record[1], num_reads = record[2];
+ const std::uint32_t num_data_ram_writes = record[3], num_writes = record[4];
+ if (record[0] != start || num_parameters != parameters.size() ||
+ !std::equal(parameters.begin(), parameters.end(), record + HeaderWords)) {
+ return false; // another call
+ }
+ const std::uint32_t* reads = record + HeaderWords + num_parameters;
+ for (std::uint32_t i = 0; i < num_reads; ++i) {
+ if (data_ram[reads[i * 2]] != reads[i * 2 + 1]) {
+ return false;
+ }
+ }
+ const std::uint32_t* data_ram_writes = reads + num_reads * 2;
+ effects.data_ram_writes = {data_ram_writes, num_data_ram_writes * 2};
+ effects.writes = {data_ram_writes + num_data_ram_writes * 2, num_writes * 2};
+ return true;
+ }
+
+public:
+ void Store(std::uint32_t start, std::span<const std::uint32_t> parameters,
+ const MacroMemoPairs& data_ram_reads, const MacroMemoPairs& data_ram_writes,
+ const MacroMemoPairs& writes) {
+ ++statistics.stores;
+ const std::uint64_t hash = Hash(start, parameters);
+ // A call stored again (its data RAM reads no longer held) usually has the same shape:
+ // rewrite its record in place, or the arena fills with stale copies.
+ if (const auto it = entries.find(hash); it != entries.end()) {
+ std::uint32_t* record = arena.data() + it->second;
+ if (record[0] == start && record[1] == parameters.size() &&
+ record[2] == data_ram_reads.size() && record[3] == data_ram_writes.size() &&
+ record[4] == writes.size() &&
+ std::equal(parameters.begin(), parameters.end(), record + HeaderWords)) {
+ std::uint32_t* out = record + HeaderWords + parameters.size();
+ for (const MacroMemoPairs* pairs : {&data_ram_reads, &data_ram_writes, &writes}) {
+ for (const auto& [address, value] : *pairs) {
+ *out++ = address;
+ *out++ = value;
+ }
+ }
+ ++statistics.overwrites;
+ return;
+ }
+ }
+ if (arena.size() >= MaxArenaWords) {
+ Clear();
+ ++statistics.clears;
+ }
+ const auto offset = static_cast<std::uint32_t>(arena.size());
+ arena.push_back(start);
+ arena.push_back(static_cast<std::uint32_t>(parameters.size()));
+ arena.push_back(static_cast<std::uint32_t>(data_ram_reads.size()));
+ arena.push_back(static_cast<std::uint32_t>(data_ram_writes.size()));
+ arena.push_back(static_cast<std::uint32_t>(writes.size()));
+ arena.insert(arena.end(), parameters.begin(), parameters.end());
+ for (const MacroMemoPairs* pairs : {&data_ram_reads, &data_ram_writes, &writes}) {
+ for (const auto& [address, value] : *pairs) {
+ arena.push_back(address);
+ arena.push_back(value);
+ }
+ }
+ entries.insert_or_assign(hash, offset);
+ last_by_start[start % last_by_start.size()] = {start, offset};
+ }
+
+ const Statistics& GetStatistics() const {
+ return statistics;
+ }
+
+ std::size_t Size() const {
+ return entries.size();
+ }
+
+ std::size_t ArenaWords() const {
+ return arena.size();
+ }
+
+ /// Forgets every call when the macro code is no longer the code they ran.
+ void Synchronize(std::uint64_t code_generation) {
+ if (code_generation != generation) {
+ generation = code_generation;
+ Clear();
+ }
+ }
+
+private:
+ static constexpr std::size_t HeaderWords = 5;
+ static constexpr std::size_t MaxArenaWords = std::size_t{1} << 22; // 16 MiB
+
+ static std::uint64_t Hash(std::uint32_t start, std::span<const std::uint32_t> parameters) {
+ std::uint64_t hash = 0x9e3779b97f4a7c15ULL ^ (std::uint64_t{start} << 32) ^ parameters.size();
+ for (const std::uint32_t word : parameters) {
+ hash = (hash ^ word) * 0xff51afd7ed558ccdULL;
+ hash ^= hash >> 32;
+ }
+ return hash;
+ }
+
+ void Clear() {
+ arena.clear();
+ entries.clear();
+ last_by_start.fill({0, NoRecord});
+ }
+
+ Map entries;
+ std::vector<std::uint32_t> arena;
+ /// The last record found or stored per macro start (direct mapped).
+ std::array<std::pair<std::uint32_t, std::uint32_t>, 256> last_by_start{};
+ std::uint64_t generation{};
+ Statistics statistics;
+};
+
+} // namespace Tegra
diff --git a/src/video_core/macro_turing.h b/src/video_core/macro_turing.h
new file mode 100644
index 0000000000..600db48119
--- /dev/null
+++ b/src/video_core/macro_turing.h
@@ -0,0 +1,274 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+// Instruction semantics adapted from Mesa mme_tu104_sim.c and mme_tu104.c:
+// Copyright © 2022 Collabora Ltd. SPDX-License-Identifier: MIT
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <span>
+#include <string_view>
+#include <vector>
+
+namespace Tegra {
+
+// TU104/GA10x MME has two ALUs and two outputs per 96-bit instruction.
+class TuringMacroInterpreter {
+public:
+ using Word = std::uint32_t;
+ struct Result {
+ bool success;
+ Word pc;
+ std::size_t consumed;
+ std::string_view error;
+ };
+
+ // One instruction's fields, decoded once and reused while its words stay the same (macro
+ // code runs thousands of times a frame; decoding it every time was most of the interpreter).
+ struct Decoded {
+ std::array<Word, 3> raw{~0U, ~0U, ~0U};
+ std::array<Word, 2> imm, op, dst, src_a, src_b, out_method, out_emit;
+ Word pred_reg, pred_mode;
+ // Lane/output enable masks when the predicate register is non-zero / zero.
+ Word mask_true, mask_false;
+ bool end;
+ std::array<bool, 2> load;
+ };
+
+ static Decoded Decode(const std::array<Word, 3>& bits) {
+ const auto field = [&](Word low, Word count) {
+ std::uint64_t value = bits[low / 32];
+ if (low / 32 < 2) value |= std::uint64_t{bits[low / 32 + 1]} << 32;
+ return Word((value >> (low % 32)) & ((std::uint64_t{1} << count) - 1));
+ };
+ Decoded d;
+ d.raw = bits;
+ d.imm = {field(30, 16), field(66, 16)};
+ d.op = {field(10, 5), field(46, 5)};
+ d.dst = {field(15, 5), field(51, 5)};
+ d.src_a = {field(20, 5), field(56, 5)};
+ d.src_b = {field(25, 5), field(61, 5)};
+ d.out_method = {field(82, 3), field(89, 3)};
+ d.out_emit = {field(85, 4), field(92, 4)};
+ d.pred_reg = field(5, 5);
+ d.pred_mode = field(1, 4);
+ d.end = field(0, 1) != 0;
+ constexpr std::array<std::string_view, 16> predicates{
+ "UUUU", "TTTT", "FFFF", "TTUU", "FFUU", "TFUU", "TUUU", "FUUU",
+ "UUTT", "UUTF", "UUTU", "UUFT", "UUFF", "UUFU", "UUUT", "UUUF"};
+ d.mask_true = d.mask_false = 15;
+ if (d.pred_mode) {
+ d.mask_true = d.mask_false = 0;
+ for (Word i = 0; i < 4; ++i) {
+ if (predicates[d.pred_mode][i] != 'T') d.mask_true |= 1U << i;
+ if (predicates[d.pred_mode][i] != 'F') d.mask_false |= 1U << i;
+ }
+ }
+ for (Word i = 0; i < 2; ++i) {
+ const Word reg = 28 + i, out = 3 + i;
+ d.load[i] = d.pred_reg == reg || d.src_a[0] == reg || d.src_a[1] == reg ||
+ d.src_b[0] == reg || d.src_b[1] == reg || d.out_method[0] == out ||
+ d.out_method[1] == out || d.out_emit[0] == out || d.out_emit[1] == out;
+ }
+ return d;
+ }
+
+ template <typename Read, typename Emit>
+ // `parameters` is the load FIFO; `emit` may append to it (MME_DMA_READ_FIFOED).
+ Result Execute(std::span<const Word> code, Word start,
+ std::vector<Word>& parameters, Read read, Emit emit) {
+ std::array<Word, 24> regs{};
+ Word initialized{}, carry{}, method{}, increment{};
+ Word pc = start, loop_count{}, loop_start{}, loop_end{};
+ std::size_t consumed{};
+ bool end_next{}, stopped{};
+ std::string_view error;
+ const auto fail = [&](std::string_view message) { if (error.empty()) error = message; };
+ if (decoded.size() < code.size() / 3) decoded.resize(code.size() / 3);
+ read_registers = false;
+ reread_data_ram = false;
+ data_ram_reads.clear();
+ data_ram_writes.clear();
+ for (Word steps = 0; steps < 1000000; ++steps) {
+ if (static_cast<std::size_t>(pc) * 3 + 2 >= code.size()) {
+ fail("instruction outside uploaded RAM");
+ break;
+ }
+ const std::array<Word, 3> bits{code[pc * 3 + 2], code[pc * 3 + 1], code[pc * 3]};
+ Decoded& d = decoded[pc];
+ if (d.raw != bits) d = Decode(bits);
+ std::array<Word, 2> load{}, alu{};
+ for (Word i = 0; i < 2; ++i) {
+ if (d.load[i]) {
+ if (consumed >= parameters.size()) { fail("parameter FIFO underflow"); break; }
+ load[i] = parameters[consumed++];
+ }
+ }
+ // Both ALUs read the register file before either destination is committed: their
+ // operands are read up front. Carry chaining remains explicit between the lanes.
+ const auto get = [&](Word reg, Word lane) -> Word {
+ if (reg < 24) {
+ if (!(initialized & (1U << reg))) fail("uninitialized register read");
+ return regs[reg];
+ }
+ switch (reg) {
+ case 24: return 0;
+ case 25: return Word(std::int32_t(std::int16_t(d.imm[lane])));
+ case 26: return Word(std::int32_t(std::int16_t(d.imm[1 - lane])));
+ case 27: return (d.imm[0] << 16) | d.imm[1];
+ case 28: return load[0];
+ case 29: return load[1];
+ default: fail("invalid source register"); return 0;
+ }
+ };
+ const Word mask = d.pred_mode ? (get(d.pred_reg, 0) ? d.mask_true : d.mask_false) : 15;
+ std::array<Word, 2> xs{}, ys{};
+ for (Word lane = 0; lane < 2; ++lane) {
+ if (mask & (1U << lane)) {
+ xs[lane] = get(d.src_a[lane], lane);
+ ys[lane] = get(d.src_b[lane], lane);
+ }
+ }
+ Word next_pc = pc + 1;
+ bool branch_cancels_end{};
+ for (Word lane = 0; lane < 2 && error.empty(); ++lane) {
+ if (!(mask & (1U << lane))) continue;
+ const Word x = xs[lane], y = ys[lane];
+ const Word imm = d.imm[lane];
+ Word value{};
+ const auto condition = [&](Word comparison) {
+ switch (comparison) {
+ case 17: case 25: return std::int32_t(x) < std::int32_t(y);
+ case 18: case 26: return x < y;
+ case 19: case 27: return std::int32_t(x) <= std::int32_t(y);
+ case 20: case 28: return x <= y;
+ default: return x == y;
+ }
+ };
+ switch (d.op[lane]) {
+ case 0: value = x + y; carry = value < x; break;
+ case 1: value = x + y + carry; break;
+ case 2: value = x - y; carry = value > x; break;
+ case 3: value = x - y - carry; break;
+ case 4: case 6: {
+ const std::uint64_t a = d.op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(x))) : x;
+ const std::uint64_t b = d.op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(y))) : y;
+ const auto product = a * b;
+ value = Word(product); carry = Word(product >> 32); break;
+ }
+ case 5: value = carry; break;
+ case 7:
+ // EXTENDED: x = 0x1000 is the load barrier (Mesa mme_tu104_load_barrier),
+ // nothing to do in a sequential interpreter.
+ if (x != 0x1000) fail("unsupported extended operation");
+ break;
+ case 8: value = Word(std::countl_zero(x)); break;
+ case 9: value = x << (y & 31); break;
+ case 10: value = x >> (y & 31); break;
+ case 11: value = Word(std::int32_t(x) >> (y & 31)); break;
+ case 12: value = x & y; break;
+ case 13: value = ~(x & y); break;
+ case 14: value = x | y; break;
+ case 15: value = x ^ y; break;
+ case 16: {
+ const Word dest = (imm >> 10) & 63, count = (imm >> 5) & 31;
+ const Word source = imm & 31;
+ if (dest >= 32) { fail("invalid merge destination"); break; }
+ const Word bits_mask = (1U << count) - 1;
+ value = (x & ~(bits_mask << dest)) | (((y >> source) & bits_mask) << dest);
+ break;
+ }
+ case 17: case 18: case 19: case 20: case 21:
+ value = condition(d.op[lane]) ? ~0U : 0; break;
+ case 22: value = read((x + y) & 0xffff); read_registers = true; break;
+ case 23:
+ if (loop_count || imm < 2) { fail("invalid loop"); break; }
+ // The native SDK emits zero-count loops around FIFO reads.
+ // Unlike Mesa's test simulator, these must skip the body.
+ if (x == 0) { next_pc = pc + imm; break; }
+ loop_count = x - 1; loop_start = pc; loop_end = pc + imm - 1;
+ break;
+ case 24: next_pc = pc + (imm & 0x7fff); break;
+ case 25: case 26: case 27: case 28: case 29:
+ if (condition(d.op[lane]) == bool(imm & 0x8000)) {
+ const auto offset = std::int32_t(imm << 19) >> 19;
+ if (offset == -4096) stopped = true;
+ else {
+ next_pc = Word(std::int32_t(pc) + offset);
+ // Native back edges use bit 14 to suppress the
+ // fallthrough termination when the branch is taken.
+ branch_cancels_end = (imm & 0x4000) != 0;
+ }
+ }
+ break;
+ case 30:
+ if (x >= active_dram->size()) { fail("data RAM read out of bounds"); break; }
+ value = (*active_dram)[x];
+ if (std::ranges::find(data_ram_writes, x, &DataRamAccess::first) != data_ram_writes.end()) {
+ reread_data_ram = true;
+ }
+ data_ram_reads.emplace_back(x, value);
+ break;
+ case 31:
+ if (x >= active_dram->size()) { fail("data RAM write out of bounds"); break; }
+ (*active_dram)[x] = y;
+ data_ram_writes.emplace_back(x, y);
+ break;
+ }
+ alu[lane] = value;
+ if (d.dst[lane] < 24) { regs[d.dst[lane]] = value; initialized |= 1U << d.dst[lane]; }
+ else if (d.dst[lane] != 24) fail("invalid destination register");
+ }
+ const auto output = [&](Word selector) -> Word {
+ switch (selector) {
+ case 1: return alu[0]; case 2: return alu[1];
+ case 3: return load[0]; case 4: return load[1];
+ case 5: return d.imm[0]; case 6: return d.imm[1];
+ case 8: return d.imm[0] >> 12; case 9: return d.imm[1] >> 12;
+ case 10: return (d.imm[0] << 16) | d.imm[1];
+ default: fail("invalid output selector"); return 0;
+ }
+ };
+ for (Word lane = 0; lane < 2 && error.empty(); ++lane) {
+ if (!(mask & (4U << lane))) continue;
+ if (d.out_method[lane]) {
+ const Word address = output(d.out_method[lane]);
+ method = address & 0xfff; increment = (address >> 12) & 15;
+ }
+ if (d.out_emit[lane]) {
+ const Word value = output(d.out_emit[lane]);
+ if (error.empty() && !emit(method, value)) fail("unsupported emitted method");
+ method += increment;
+ }
+ }
+ if (!error.empty()) break;
+ if (end_next || stopped) return {true, pc, consumed, {}};
+ end_next = d.end && !branch_cancels_end;
+ if (loop_count && pc == loop_end) { --loop_count; next_pc = loop_start + 1; }
+ pc = next_pc;
+ }
+ if (error.empty()) error = "instruction limit exceeded";
+ return {false, pc, consumed, error};
+ }
+
+ using DataRam = std::array<Word, 0xc00>;
+ DataRam dram{};
+ // The data RAM DREAD/DWRITE use: a channel's own copy when the caller sets one. Macro code is
+ // shared by the channels of a process, but each channel keeps its own data RAM (title 1's
+ // channels store their own QMD ring base in word 0x27).
+ DataRam* active_dram{&dram};
+ // Decoded instructions by PC, re-decoded whenever their words differ.
+ std::vector<Decoded> decoded;
+ // What the last Execute read besides its parameters, for the macro memo: whether it read a
+ // register (MME_READ), the data RAM words it read and wrote (address, value) in order, and
+ // whether it read a word it had written itself.
+ using DataRamAccess = std::pair<Word, Word>;
+ bool read_registers{};
+ bool reread_data_ram{};
+ std::vector<DataRamAccess> data_ram_reads;
+ std::vector<DataRamAccess> data_ram_writes;
+};
+
+} // namespace Tegra
diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp
index 8ffdcb6276..0eae8fac24 100644
--- a/src/video_core/memory_manager.cpp
+++ b/src/video_core/memory_manager.cpp
@@ -22,6 +22,10 @@
#include "video_core/texture_cache/util.h"
namespace Tegra {
+bool MemoryManager::IsShuttingDown() const {
+ return system.IsShuttingDown();
+}
+
using Tegra::Memory::GuestMemoryFlags;
std::atomic<size_t> MemoryManager::unique_identifier_generator{};
@@ -56,6 +60,10 @@ MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, GPU
: MemoryManager(system_, system_.Host1x().MemoryManager(), address_space_bits_, split_address_, big_page_bits_, page_bits_)
{}
+void MemoryManager::SetNativeSyncpointShim(GPUVAddr address) {
+ native_syncpoint_shim.store(address, std::memory_order_release);
+}
+
MemoryManager::~MemoryManager() = default;
MemoryManager::EntryType MemoryManager::GetEntry(size_t position, bool is_big_page) const {
@@ -231,6 +239,15 @@ std::optional<DAddr> MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t s
template <typename T>
T MemoryManager::Read(GPUVAddr addr) const {
+ if constexpr (sizeof(T) == sizeof(u32)) {
+ const auto shim = native_syncpoint_shim.load(std::memory_order_acquire);
+ if (shim && addr >= shim && addr - shim < NumShimSyncpoints * sizeof(u32) &&
+ Common::IsAligned(addr - shim, sizeof(u32))) {
+ return static_cast<T>(system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(
+ static_cast<u32>((addr - shim) / sizeof(u32))));
+ }
+ }
+
if (auto page_pointer{GetPointer(addr)}; page_pointer) {
// NOTE: Avoid adding any extra logic to this fast-path block
T value;
@@ -238,7 +255,10 @@ T MemoryManager::Read(GPUVAddr addr) const {
return value;
}
- ASSERT(false);
+ static std::atomic<u32> reported{};
+ if (reported.fetch_add(1) < 16) {
+ LOG_ERROR(HW_GPU, "Read{} of unmapped GPU address {:#x}", sizeof(T) * 8, addr);
+ }
return {};
}
@@ -390,6 +410,16 @@ void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::siz
}
void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, const std::size_t size) const {
+ // Small reads within one 4 KiB page (texture and sampler descriptors, several per draw): one
+ // translation and a copy, as Read<T> does, instead of the page walk (~3% of the GPU thread
+ // in title 1). Anything else, unmapped or sparse pages included, takes the full path.
+ constexpr std::size_t SmallPage = 0x1000;
+ if (size <= SmallPage && (gpu_src_addr & (SmallPage - 1)) + size <= SmallPage) {
+ if (const u8* const pointer = GetPointer(gpu_src_addr)) {
+ std::memcpy(dest_buffer, pointer, size);
+ return;
+ }
+ }
ReadBlockImpl(gpu_src_addr, dest_buffer, size, VideoCommon::CacheType::None, true);
}
diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h
index 08978567d2..105cb14e23 100644
--- a/src/video_core/memory_manager.h
+++ b/src/video_core/memory_manager.h
@@ -43,6 +43,19 @@ public:
u64 big_page_bits_ = 16, u64 page_bits_ = 12);
~MemoryManager();
+ // Switch 2: GPU address of the read-only syncpoint shim (u32 per syncpoint); 0 = none.
+ void SetNativeSyncpointShim(GPUVAddr address);
+ static constexpr u32 NumShimSyncpoints = 192; // Tegra::Host1x::SyncpointManager
+
+ /// Switch 2 lays out block-linear surfaces with the console's GOB swizzle.
+ void EnableNativeTiling() {
+ native_tiling = true;
+ }
+ bool UsesNativeTiling() const {
+ return native_tiling;
+ }
+ bool IsShuttingDown() const;
+
static constexpr bool HAS_FLUSH_INVALIDATION = true;
inline size_t GetID() const noexcept {
@@ -178,6 +191,7 @@ private:
GPUVAddr gpu_addr, std::size_t size,
boost::container::small_vector<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result) const;
+ std::atomic<GPUVAddr> native_syncpoint_shim{};
Core::System& system;
MaxwellDeviceMemoryManager& memory;
@@ -220,6 +234,7 @@ private:
boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash{};
boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash2{};
+ bool native_tiling{};
mutable std::mutex guard;
static constexpr size_t continuous_bits = 64;
diff --git a/src/video_core/native_qmd.h b/src/video_core/native_qmd.h
new file mode 100644
index 0000000000..15c59cb7c6
--- /dev/null
+++ b/src/video_core/native_qmd.h
@@ -0,0 +1,133 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+#include <array>
+#include <cstdint>
+#include <functional>
+#include <string_view>
+#include <type_traits>
+#include <unordered_map>
+
+namespace Tegra {
+
+// Native QMD 3.0 dependency scheduling; nonempty grids require a dispatch backend.
+class NativeQmdScheduler {
+public:
+ using Word = std::uint32_t;
+ using Address = std::uint64_t;
+ using Descriptor = std::array<Word, 64>;
+ struct RejectShaderDispatch { bool operator()(const Descriptor&) const { return false; } };
+ // Optional diagnostics: called with "load"/"count"/"defer"/"run" events.
+ std::function<void(std::string_view, Address, std::int32_t)> trace;
+ template <typename Read, typename Release>
+ std::string_view Signal(Address address, Word argument, Read read, Release release) {
+ return Signal(address, argument, read, release, RejectShaderDispatch{});
+ }
+ template <typename Read, typename Release, typename Dispatch>
+ std::string_view Signal(Address address, Word argument, Read read, Release release, Dispatch dispatch) {
+ struct Pending { Address address; Word action; Word count; };
+ std::string_view error;
+ std::array<Pending, 256> pending{};
+ std::size_t count = 1;
+ pending[0] = {address, argument & 15, ((argument >> 14) & 0x3ff) + 1};
+ for (std::size_t i = 0; i < count; ++i) {
+ const auto request = pending[i];
+ if (states.size() >= 4096 && !states.contains(request.address)) return "QMD cache limit";
+ auto& state = states[request.address];
+ if (request.action == 3 || request.action == 2) {
+ if (request.action == 3 || !state.loaded) {
+ if (!read(request.address, state.descriptor)) return "unmapped QMD";
+ const auto& qmd = state.descriptor;
+ if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue";
+ if constexpr (std::is_same_v<Dispatch, RejectShaderDispatch>) {
+ if (qmd[12] && (qmd[13] & 0xffff) && (qmd[14] & 0xffff))
+ return "native shader dispatch required";
+ }
+ state.dependencies = std::int32_t((qmd[4] >> 14) & 0xffff) + state.deferred;
+ if (trace) trace("base", request.address, std::int32_t((qmd[4] >> 14) & 0xffff));
+ if (trace) trace("load", request.address, state.dependencies);
+ state.deferred = 0;
+ state.loaded = true;
+ state.completed = false;
+ }
+ state.scheduled = true;
+ } else if (request.action == 7 || request.action == 12) {
+ const auto delta = request.action == 7 ? -std::int32_t(request.count) : std::int32_t(request.count);
+ if (!state.loaded || state.completed) {
+ state.deferred += delta;
+ if (trace) trace("defer", request.address, state.deferred);
+ continue;
+ }
+ state.dependencies += delta;
+ if (trace) trace("count", request.address, state.dependencies);
+ } else if (request.action == 0 || request.action == 1) {
+ continue;
+ } else {
+ return "unsupported PCAS action";
+ }
+ if (!state.loaded || !state.scheduled || state.completed || state.dependencies > 0) continue;
+ if (state.dependencies < 0) return "QMD dependency underflow";
+ // The SDK patches release payloads while a scheduled QMD is
+ // waiting for its remaining dependencies. Fetch them at execution.
+ if (!read(request.address, state.descriptor)) return "unmapped QMD";
+ const auto qmd = state.descriptor;
+ if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue";
+ // Validate the release slots before making any visible.
+ for (const Word base : ReleaseSlots) {
+ const Word flags = qmd[base + 1];
+ if (!(flags & (1U << 23))) continue;
+ const bool add = (flags & (1U << 26)) != 0;
+ if ((flags & (0x3bU << 24)) || (flags >> 30) != 1 ||
+ (add && (flags & (7U << 20)))) return "unsupported QMD release format";
+ if (add) {
+ if constexpr (!std::is_invocable_r_v<bool, Release, Address, Word, Word>)
+ return "unsupported QMD reduction callback";
+ }
+ }
+ // A program Eden can't run is skipped, but the QMD still completes: its releases
+ // are what other queues and the CPU wait on (the console never fails a launch).
+ const bool dispatched = !(qmd[12] && (qmd[13] & 0xffff) && (qmd[14] & 0xffff)) || dispatch(qmd);
+ if (!dispatched) error = "native shader dispatch required";
+ if (trace) trace("run", request.address, dispatched ? 1 : 0);
+ state.completed = true;
+ for (const Word base : ReleaseSlots) {
+ const Word flags = qmd[base + 1];
+ if (!(flags & (1U << 23))) continue;
+ const Address destination = (Address{flags & 0xff} << 32) | qmd[base];
+ bool released{};
+ if constexpr (std::is_invocable_r_v<bool, Release, Address, Word, Word>) {
+ released = release(destination, qmd[base + 2], flags);
+ } else {
+ released = release(destination, qmd[base + 2]);
+ }
+ if (!released) return "unmapped QMD release";
+ }
+ if (qmd[16] & 1) {
+ const Word action = (qmd[16] >> 1) & 7;
+ if (action != 4 && action != 3 && action != 1) return "unsupported dependent QMD action";
+ if (count == pending.size()) return "QMD chain limit";
+ pending[count++] = {Address{qmd[15]} << 8, action == 4 ? 7U : action == 3 ? 3U : 2U, 1};
+ }
+ if (qmd[16] & (1U << 5)) return "second dependent QMD unsupported";
+ }
+ return error;
+ }
+private:
+ // Release slots (address low, flags | address high, payload). NVN2 patches a third one at
+ // word 52 for queue syncs (e.g. a compute queue releasing a semaphore another queue waits on).
+ static constexpr std::array<Word, 3> ReleaseSlots{24, 28, 52};
+
+ struct State {
+ Descriptor descriptor{};
+ std::int32_t dependencies{};
+ std::int32_t deferred{};
+ bool loaded{};
+ bool scheduled{};
+ bool completed{};
+ };
+ // One table for every channel: a QMD's dependency counter is state of the QMD itself, and
+ // title 1 schedules a QMD on one channel while another decrements it (a decrement that
+ // arrived first was kept by the other channel's scheduler and the QMD never launched).
+ static inline std::unordered_map<Address, State> states;
+};
+} // namespace Tegra
diff --git a/src/video_core/native_query_trace.h b/src/video_core/native_query_trace.h
new file mode 100644
index 0000000000..036c3de936
--- /dev/null
+++ b/src/video_core/native_query_trace.h
@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <cstdlib>
+#include <fstream>
+#include <iomanip>
+#include <mutex>
+#include "common/common_types.h"
+
+namespace VideoCommon {
+namespace NativeQueryDiagnostics {
+struct State {
+ State() : file{"native-query-trace.csv", std::ios::trunc} {
+ file << "event,asid,gpu,cpu,type,value,flags\n";
+ }
+ std::mutex mutex;
+ std::ofstream file;
+ size_t records{};
+};
+inline State& GetState() {
+ static State state;
+ return state;
+}
+} // namespace NativeQueryDiagnostics
+
+// Events: 0=request, 1=written, 2=invalidated, 3=unsynchronized,
+// 4=channel suspended, 5=channel resumed, 6=buffer readback,
+// 7/8=QMD release slots observed, 9=PCAS request,
+// 10=DMA destination (value=source, flags=line bytes), 11=inline write.
+// Keep this opt-in and bounded.
+inline void TraceNativeQuery(u32 event, u64 asid, u64 gpu, u64 cpu, u32 type, u64 value, u32 flags) {
+ static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr;
+ if (!diagnostics) return;
+ auto& state = NativeQueryDiagnostics::GetState();
+ const std::scoped_lock lock{state.mutex};
+ if (state.records++ < 1000000 || event == 4 || event == 5) {
+ state.file << event << ',' << asid << ',' << std::hex << gpu << ',' << cpu << ','
+ << type << ',' << value << ',' << flags << std::dec << '\n';
+ }
+ if (event == 4 || event == 5) state.file.flush();
+}
+} // namespace VideoCommon
diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h
new file mode 100644
index 0000000000..5b6d89abdd
--- /dev/null
+++ b/src/video_core/nx2.h
@@ -0,0 +1,143 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <mutex>
+#include <vector>
+
+#include "common/common_types.h"
+
+namespace Tegra {
+
+// Set once a Switch 2 (Ampere, NVN2) driver is talking to the GPU. Its command streams use
+// Ampere classes and the Turing+ macro ISA, which Eden's Maxwell engines don't implement.
+inline std::atomic<bool> g_nx2_ampere{false};
+
+// NVN2 queues release their progress values into the second page of a two-page GPU-accessible
+// region and GPU acquires read the same offsets in the first page (the CPU polls the second).
+// Something on the console copies one into the other; until that is known, the puller mirrors
+// releases that land in a second page. Every queue has such a region, so keep a small table.
+inline constexpr size_t Nx2MaxSemaphoreRegions = 32;
+inline std::array<std::atomic<u64>, Nx2MaxSemaphoreRegions> g_nx2_semaphore_regions{}; // starts
+
+inline void Nx2AddSemaphoreRegion(u64 start) {
+ for (auto& region : g_nx2_semaphore_regions) {
+ if (region.load() == start) {
+ return;
+ }
+ }
+ for (auto& region : g_nx2_semaphore_regions) {
+ u64 expected = 0;
+ if (region.compare_exchange_strong(expected, start)) {
+ return;
+ }
+ }
+}
+
+/// The region's memory was destroyed: stop mirroring into it.
+inline void Nx2RemoveSemaphoreRegion(u64 start) {
+ for (auto& region : g_nx2_semaphore_regions) {
+ u64 expected = start;
+ region.compare_exchange_strong(expected, 0);
+ }
+}
+
+/// GPU VA in the first page that mirrors a release at `address`, or 0.
+inline u64 Nx2SemaphoreMirror(u64 address) {
+ for (const auto& region : g_nx2_semaphore_regions) {
+ const u64 start = region.load(std::memory_order_relaxed);
+ if (start == 0) {
+ continue;
+ }
+ if (address >= start + 0x1000 && address < start + 0x2000) {
+ return address - 0x1000;
+ }
+ }
+ return 0;
+}
+
+/// Channels suspended on a semaphore acquire. A console channel polls and sees every release;
+/// Eden writes releases in bursts when fences signal and resumes waiters only between command
+/// lists, so an acquire-equal could miss its value. Writes latch the waiters they satisfy.
+class Nx2SemaphoreWaiters {
+public:
+ // Puller GpuSemaphoreOperation values.
+ static constexpr u32 AcquireEqual = 0x1;
+ static constexpr u32 AcquireGequal = 0x4;
+ static constexpr u32 AcquireMask = 0x8;
+
+ static Nx2SemaphoreWaiters& Get() {
+ static Nx2SemaphoreWaiters waiters;
+ return waiters;
+ }
+
+ void Add(const void* owner, u64 device_address, u32 value, u32 operation) {
+ std::scoped_lock lk{mutex};
+ std::erase_if(waiters, [owner](const Waiter& w) { return w.owner == owner; });
+ waiters.push_back({owner, device_address, value, operation, false});
+ count.store(waiters.size(), std::memory_order_release);
+ }
+
+ void Remove(const void* owner) {
+ if (count.load(std::memory_order_acquire) == 0) {
+ return;
+ }
+ std::scoped_lock lk{mutex};
+ std::erase_if(waiters, [owner](const Waiter& w) { return w.owner == owner; });
+ count.store(waiters.size(), std::memory_order_release);
+ }
+
+ [[nodiscard]] bool Latched(const void* owner) {
+ if (count.load(std::memory_order_acquire) == 0) {
+ return false;
+ }
+ std::scoped_lock lk{mutex};
+ return std::ranges::any_of(waiters,
+ [owner](const Waiter& w) { return w.owner == owner && w.latched; });
+ }
+
+ /// A GPU write of `value` to the u32 at `device_address`.
+ void OnWrite(u64 device_address, u32 value) {
+ if (count.load(std::memory_order_acquire) == 0) {
+ return;
+ }
+ std::scoped_lock lk{mutex};
+ for (auto& w : waiters) {
+ if (w.address == device_address && !w.latched && Matches(w, value)) {
+ w.latched = true;
+ }
+ }
+ }
+
+private:
+ struct Waiter {
+ const void* owner;
+ u64 address;
+ u32 value;
+ u32 operation;
+ bool latched;
+ };
+
+ static bool Matches(const Waiter& w, u32 current) {
+ switch (w.operation) {
+ case AcquireEqual:
+ return current == w.value;
+ case AcquireGequal:
+ return current >= w.value;
+ case AcquireMask:
+ return (current & w.value) != 0;
+ default:
+ return false;
+ }
+ }
+
+ std::mutex mutex;
+ std::vector<Waiter> waiters;
+ std::atomic<size_t> count{};
+};
+
+} // namespace Tegra
diff --git a/src/video_core/nx2_profile.h b/src/video_core/nx2_profile.h
new file mode 100644
index 0000000000..a162ce16a3
--- /dev/null
+++ b/src/video_core/nx2_profile.h
@@ -0,0 +1,150 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <array>
+#include <chrono>
+#include <cstdlib>
+#include <string>
+
+#include "common/common_types.h"
+#include "common/logging.h"
+#include "common/cpu_features.h"
+
+// EDEN_NX2_GPU_PROFILE: time where the GPU thread spends its time. Each zone counts its own
+// (exclusive) time, nested zones excluded; every 10 s the thread logs each zone's share.
+namespace VideoCommon::Nx2Profile {
+
+enum class Zone : u32 {
+ Other, ///< GPU thread time outside every zone below (register writes, command parsing)
+ Idle,
+ PendingSleep,
+ ResumePending,
+ Push,
+ Tick,
+ Run,
+ FlushInvalidate,
+ Macro,
+ Draw,
+ Clear,
+ Compute,
+ FlushWork,
+ PipelineLookup,
+ Configure,
+ SyncDescriptors,
+ ReadHandles,
+ FillImageViews,
+ TextureBuffers,
+ UpdateBuffers,
+ DescriptorAcquire,
+ BindStageBuffers,
+ PushDescriptors,
+ RenderTargets,
+ ConfigureDraw,
+ DynamicState,
+ DrawRecord,
+ TextureUpload,
+ Count,
+};
+
+inline constexpr std::array<const char*, static_cast<size_t>(Zone::Count)> zone_names{
+ "other", "idle", "pending sleep", "resume pending", "push",
+ "tick", "run", "flush/inval", "macro", "draw",
+ "clear", "compute", "flush work", "pipeline lookup", "configure",
+ "sync descriptors", "read handles", "fill image views", "texture buffers", "update buffers",
+ "descriptor acquire", "bind stage buffers", "push descriptors", "render targets",
+ "configure draw", "dynamic state", "draw record", "texture upload",
+};
+
+inline bool Enabled() {
+ static const bool enabled = std::getenv("EDEN_NX2_GPU_PROFILE") != nullptr;
+ return enabled;
+}
+
+// Nanoseconds; from the host's own counter where the wall clock reads it directly (the zones
+// run ~150,000 times a second, and QueryPerformanceCounter cost ~5% of the GPU thread).
+inline u64 Now() {
+ return static_cast<u64>(Common::g_wall_clock.GetTimeNS().count());
+}
+
+struct ThreadData {
+ std::array<u64, static_cast<size_t>(Zone::Count)> exclusive_ns{};
+ std::array<u64, static_cast<size_t>(Zone::Count)> count{};
+ u64 child_ns{}; ///< Time of the zones nested in the current one
+ u64 window_start{};
+};
+
+inline thread_local ThreadData t_data;
+
+class Scope {
+public:
+ explicit Scope(Zone zone_) : zone{zone_}, enabled{Enabled()} {
+ if (!enabled) {
+ return;
+ }
+ parent_child_ns = t_data.child_ns;
+ t_data.child_ns = 0;
+ start = Now();
+ }
+
+ ~Scope() {
+ if (!enabled) {
+ return;
+ }
+ const u64 elapsed = Now() - start;
+ auto& data = t_data;
+ const auto index = static_cast<size_t>(zone);
+ data.exclusive_ns[index] += elapsed - data.child_ns;
+ ++data.count[index];
+ data.child_ns = parent_child_ns + elapsed;
+ }
+
+ Scope(const Scope&) = delete;
+ Scope& operator=(const Scope&) = delete;
+
+private:
+ Zone zone;
+ bool enabled;
+ u64 start{};
+ u64 parent_child_ns{};
+};
+
+/// Called by the GPU thread between commands: logs and resets the totals every 10 s.
+inline void Report() {
+ if (!Enabled()) {
+ return;
+ }
+ auto& data = t_data;
+ const u64 now = Now();
+ if (data.window_start == 0) {
+ data.window_start = now;
+ data.child_ns = 0;
+ return;
+ }
+ const u64 window = now - data.window_start;
+ if (window < 10'000'000'000ULL) {
+ return;
+ }
+ // Time in no zone at all is the wall time minus the zones' totals.
+ u64 zoned{};
+ for (size_t i = 1; i < data.exclusive_ns.size(); ++i) {
+ zoned += data.exclusive_ns[i];
+ }
+ data.exclusive_ns[0] = window > zoned ? window - zoned : 0;
+ std::string line;
+ for (size_t i = 0; i < data.exclusive_ns.size(); ++i) {
+ if (data.exclusive_ns[i] == 0 && data.count[i] == 0) {
+ continue;
+ }
+ // Per second of wall time: milliseconds spent and calls made.
+ const double ms = static_cast<double>(data.exclusive_ns[i]) * 1000.0 / window;
+ const double calls = static_cast<double>(data.count[i]) * 1e9 / window;
+ line += fmt::format(" | {} {:.1f} ms ({:.0f}/s)", zone_names[i], ms, calls);
+ }
+ LOG_INFO(HW_GPU, "GPU thread per second:{}", line);
+ data = {};
+ data.window_start = now;
+}
+
+} // namespace VideoCommon::Nx2Profile
diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h
index b8c1bc4f45..245ab1996c 100644
--- a/src/video_core/query_cache/query_cache.h
+++ b/src/video_core/query_cache/query_cache.h
@@ -7,6 +7,7 @@
#pragma once
#include <array>
+#include <cstdlib>
#include <deque>
#include <memory>
#include <mutex>
@@ -22,6 +23,7 @@
#include "video_core/gpu.h"
#include "video_core/host1x/gpu_device_memory_manager.h"
#include "video_core/memory_manager.h"
+#include "video_core/nx2.h"
#include "video_core/query_cache/bank_base.h"
#include "video_core/query_cache/query_base.h"
#include "video_core/query_cache/query_cache_base.h"
@@ -233,6 +235,16 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type
QueryPropertiesFlags flags, u32 payload, u32 subreport) {
const bool has_timestamp = True(flags & QueryPropertiesFlags::HasTimeout);
const bool is_fence = True(flags & QueryPropertiesFlags::IsAFence);
+ static const GPUVAddr watch = [] {
+ const char* value = std::getenv("EDEN_NX2_SEM_WATCH");
+ return value ? GPUVAddr{std::strtoull(value, nullptr, 16)} : GPUVAddr{};
+ }();
+ if (static const bool trace = std::getenv("EDEN_NX2_QUERY_TRACE") != nullptr;
+ trace || (watch != 0 && (addr & ~GPUVAddr{0xf}) == watch)) {
+ LOG_INFO(HW_GPU, "nx2 report addr={:#x} type={} flags={:#x} payload={:#x} as={} device={:#x}", addr,
+ static_cast<u32>(counter_type), static_cast<u32>(flags), payload, gpu_memory->GetID(),
+ gpu_memory->GpuToCpuAddress(addr).value_or(0));
+ }
size_t streamer_id = static_cast<size_t>(counter_type);
auto* streamer = impl->streamers[streamer_id];
if (streamer == nullptr) [[unlikely]] {
@@ -262,7 +274,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type
u8* pointer_timestamp = impl->device_memory.template GetPointer<u8>(cpu_addr + 8);
bool is_synced = (Settings::IsGPUFenceBehaviorDefault() ? !Settings::IsGPULevelHigh() : !Settings::IsGPUFenceBehaviorBalanced() && !Settings::IsGPUFenceBehaviorAccurate()) && is_fence;
std::function<void()> operation([this, is_synced, streamer, query_base = query, query_location,
- pointer, pointer_timestamp] {
+ pointer, pointer_timestamp, cpu_addr] {
if (True(query_base->flags & QueryFlagBits::IsInvalidated)) {
if (!is_synced) [[likely]] {
impl->pending_unregister.push_back(query_location);
@@ -287,6 +299,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type
u32 value = static_cast<u32>(query_base->value);
std::memcpy(pointer, &value, sizeof(value));
}
+ Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, static_cast<u32>(query_base->value));
if (!is_synced) [[likely]] {
impl->pending_unregister.push_back(query_location);
}
@@ -303,6 +316,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type
} else {
std::memcpy(pointer, &payload, sizeof(payload));
}
+ Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, payload);
streamer->Free(new_query_id);
return;
}
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp
index 1d77d28c46..4e0972d1d2 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp
@@ -716,7 +716,7 @@ void RasterizerOpenGL::AccelerateInlineToMemory(GPUVAddr address, size_t copy_si
}
gpu_memory->WriteBlockUnsafe(address, memory.data(), copy_size);
{
- std::unique_lock<std::recursive_mutex> lock{buffer_cache.mutex};
+ std::unique_lock lock{buffer_cache.mutex};
if (!buffer_cache.InlineMemory(*cpu_addr, copy_size, memory)) {
buffer_cache.WriteMemory(*cpu_addr, copy_size);
}
diff --git a/src/video_core/renderer_opengl/util_shaders.cpp b/src/video_core/renderer_opengl/util_shaders.cpp
index 7ced8c77b4..5ed96a2037 100644
--- a/src/video_core/renderer_opengl/util_shaders.cpp
+++ b/src/video_core/renderer_opengl/util_shaders.cpp
@@ -96,6 +96,7 @@ void UtilShaders::ASTCDecode(Image& image, const StagingBufferMap& map,
glUniform1ui(4, params.x_shift);
glUniform1ui(5, params.block_height);
glUniform1ui(6, params.block_height_mask);
+ glUniform1ui(7, params.native_tiling);
// ASTC texture data
glBindBufferRange(GL_SHADER_STORAGE_BUFFER, BINDING_INPUT_BUFFER, map.buffer, input_offset,
diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp
index 9f4aaf390f..8a11c356ff 100644
--- a/src/video_core/renderer_vulkan/present/layer.cpp
+++ b/src/video_core/renderer_vulkan/present/layer.cpp
@@ -349,7 +349,7 @@ void Layer::UpdateRawImage(const Tegra::FramebufferConfig& framebuffer, size_t i
if (host_ptr) {
Tegra::Texture::UnswizzleTexture(
mapped_span.subspan(image_offset, linear_size), std::span(host_ptr, tiled_size),
- bytes_per_pixel, framebuffer.width, framebuffer.height, 1, block_height_log2, 0);
+ bytes_per_pixel, framebuffer.width, framebuffer.height, 1, block_height_log2, 0, 1, framebuffer.native_tiling);
buffer.Flush(); // Ensure host writes are visible before the GPU copy.
}
diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp
index ab58560170..b3ebf7b3f6 100644
--- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp
+++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp
@@ -5,6 +5,8 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <array>
+#include <chrono>
+#include <cstdlib>
#include <memory>
#include <numeric>
#include <optional>
@@ -14,9 +16,11 @@
#include "common/assert.h"
#include "common/common_types.h"
+#include "common/logging.h"
#include "common/div_ceil.h"
#include "common/vector_math.h"
#include "video_core/host_shaders/astc_decoder_comp_spv.h"
+#include "video_core/host_shaders/bc_encoder_comp_spv.h"
#include "video_core/host_shaders/queries_prefix_scan_sum_comp_spv.h"
#include "video_core/host_shaders/queries_prefix_scan_sum_nosubgroups_comp_spv.h"
#include "video_core/host_shaders/resolve_conditional_render_comp_spv.h"
@@ -31,6 +35,7 @@
#include "video_core/renderer_vulkan/vk_update_descriptor.h"
#include "video_core/texture_cache/accelerated_swizzle.h"
#include "video_core/texture_cache/types.h"
+#include "video_core/texture_cache/util.h"
#include "video_core/textures/decoders.h"
#include "video_core/vulkan_common/vulkan_device.h"
#include "video_core/vulkan_common/vulkan_wrapper.h"
@@ -183,6 +188,52 @@ struct AstcPushConstants {
u32 x_shift;
u32 block_height;
u32 block_height_mask;
+ u32 native_tiling;
+};
+
+constexpr std::array<VkDescriptorSetLayoutBinding, 2> BC_ENCODER_DESCRIPTOR_SET_BINDINGS{{
+ {
+ .binding = 0,
+ .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
+ .descriptorCount = 1,
+ .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
+ .pImmutableSamplers = nullptr,
+ },
+ {
+ .binding = 1,
+ .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
+ .descriptorCount = 1,
+ .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
+ .pImmutableSamplers = nullptr,
+ },
+}};
+
+constexpr std::array<VkDescriptorUpdateTemplateEntry, 2> BC_ENCODER_DESCRIPTOR_UPDATE_TEMPLATE{{
+ {
+ .dstBinding = 0,
+ .dstArrayElement = 0,
+ .descriptorCount = 1,
+ .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
+ .offset = 0,
+ .stride = sizeof(DescriptorUpdateEntry),
+ },
+ {
+ .dstBinding = 1,
+ .dstArrayElement = 0,
+ .descriptorCount = 1,
+ .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
+ .offset = sizeof(DescriptorUpdateEntry),
+ .stride = sizeof(DescriptorUpdateEntry),
+ },
+}};
+
+struct BcEncoderPushConstants {
+ u32 blocks_x;
+ u32 blocks_y;
+ u32 width;
+ u32 height;
+ u32 bc3;
+ u32 out_offset; // in 8-byte units
};
struct QueriesPrefixScanPushConstants {
@@ -521,16 +572,31 @@ ASTCDecoderPass::~ASTCDecoderPass() = default;
void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map,
std::span<const VideoCommon::SwizzleParameters> swizzles) {
+ const Target target{
+ .image = image.Handle(),
+ .aspect_mask = image.AspectMask(),
+ .initialized = image.ExchangeInitialization(),
+ .level_view = [&image](s32 host_level) { return image.StorageImageView(host_level); },
+ .wait = true,
+ };
+ AssembleInto(image.info, image.host_level_offset, image.guest_size_bytes, target, map,
+ swizzles);
+}
+
+void ASTCDecoderPass::AssembleInto(const VideoCommon::ImageInfo& info, u32 skip,
+ size_t guest_size_bytes, const Target& target,
+ const StagingBufferRef& map,
+ std::span<const VideoCommon::SwizzleParameters> swizzles) {
using namespace VideoCommon::Accelerated;
const std::array<u32, 2> block_dims{
- VideoCore::Surface::DefaultBlockWidth(image.info.format),
- VideoCore::Surface::DefaultBlockHeight(image.info.format),
+ VideoCore::Surface::DefaultBlockWidth(info.format),
+ VideoCore::Surface::DefaultBlockHeight(info.format),
};
scheduler.RequestOutsideRenderPassOperationContext();
const VkPipeline vk_pipeline = *pipeline;
- const VkImageAspectFlags aspect_mask = image.AspectMask();
- const VkImage vk_image = image.Handle();
- const bool is_initialized = image.ExchangeInitialization();
+ const VkImageAspectFlags aspect_mask = target.aspect_mask;
+ const VkImage vk_image = target.image;
+ const bool is_initialized = target.initialized;
scheduler.Record([vk_pipeline, vk_image, aspect_mask,
is_initialized](vk::CommandBuffer cmdbuf) {
const VkImageMemoryBarrier image_barrier{
@@ -557,20 +623,30 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, image_barrier);
cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline);
});
+ // With trimmed host levels the input holds each layer's mip chain from host_level_offset on
+ // (see TextureCache::UploadImageContents), so its layer stride is that tail's size.
+ const u32 input_layer_stride =
+ skip != 0 ? info.layer_stride -
+ static_cast<u32>(VideoCommon::FullUploadSwizzles(info)[skip].buffer_offset)
+ : info.layer_stride;
+ const size_t input_size = skip != 0 ? size_t{input_layer_stride} * info.resources.layers
+ : guest_size_bytes;
for (const VideoCommon::SwizzleParameters& swizzle : swizzles) {
const size_t input_offset = swizzle.buffer_offset + map.offset;
const u32 num_dispatches_x = Common::DivCeil(swizzle.num_tiles.width, 8U);
const u32 num_dispatches_y = Common::DivCeil(swizzle.num_tiles.height, 8U);
- const u32 num_dispatches_z = image.info.resources.layers;
+ const u32 num_dispatches_z = info.resources.layers;
compute_pass_descriptor_queue.Acquire(scheduler, 2);
compute_pass_descriptor_queue.AddBuffer(map.buffer, input_offset,
- image.guest_size_bytes - swizzle.buffer_offset);
- compute_pass_descriptor_queue.AddImage(image.StorageImageView(swizzle.level));
+ input_size - swizzle.buffer_offset);
+ compute_pass_descriptor_queue.AddImage(
+ target.level_view(swizzle.level - static_cast<s32>(skip)));
const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()};
// To unswizzle the ASTC data
- const auto params = MakeBlockLinearSwizzle2DParams(swizzle, image.info);
+ auto params = MakeBlockLinearSwizzle2DParams(swizzle, info);
+ params.layer_stride = input_layer_stride;
ASSERT(params.origin == (std::array<u32, 3>{0, 0, 0}));
ASSERT(params.destination == (std::array<s32, 3>{0, 0, 0}));
ASSERT(params.bytes_per_block_log2 == 4);
@@ -583,6 +659,7 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map,
.x_shift = params.x_shift,
.block_height = params.block_height,
.block_height_mask = params.block_height_mask,
+ .native_tiling = params.native_tiling,
};
const VkDescriptorSet set = descriptor_allocator.Commit();
device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data);
@@ -613,7 +690,74 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map,
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier);
});
- scheduler.Finish();
+ // Each decode ends with the GPU thread waiting for the host GPU to go idle. Measured here
+ // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=1 submits without waiting instead.
+ static const bool flush_only = [] {
+ const char* value = std::getenv("EDEN_NX2_ASTC_FLUSH");
+ return value != nullptr && value[0] == '1';
+ }();
+ if (!target.wait) {
+ return;
+ }
+ const auto start = std::chrono::steady_clock::now();
+ if (flush_only) {
+ scheduler.Flush();
+ } else {
+ scheduler.Finish();
+ }
+ const auto end = std::chrono::steady_clock::now();
+ static u32 decodes{};
+ static std::chrono::nanoseconds waited{};
+ static auto window_start = end;
+ ++decodes;
+ waited += end - start;
+ if (end - window_start >= std::chrono::seconds{5}) {
+ LOG_INFO(Render_Vulkan, "ASTC GPU decodes: {} in {} ms, {} ms waiting ({})", decodes,
+ std::chrono::duration_cast<std::chrono::milliseconds>(end - window_start).count(),
+ std::chrono::duration_cast<std::chrono::milliseconds>(waited).count(),
+ flush_only ? "flush" : "finish");
+ decodes = 0;
+ waited = {};
+ window_start = end;
+ }
+}
+
+BcEncoderPass::BcEncoderPass(const Device& device_, Scheduler& scheduler_,
+ DescriptorPool& descriptor_pool_,
+ ComputePassDescriptorQueue& compute_pass_descriptor_queue_)
+ : ComputePass(device_, scheduler_, descriptor_pool_, BC_ENCODER_DESCRIPTOR_SET_BINDINGS,
+ BC_ENCODER_DESCRIPTOR_UPDATE_TEMPLATE, ASTC_BANK_INFO,
+ COMPUTE_PUSH_CONSTANT_RANGE<sizeof(BcEncoderPushConstants)>, BC_ENCODER_COMP_SPV),
+ scheduler{scheduler_}, compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {}
+
+BcEncoderPass::~BcEncoderPass() = default;
+
+void BcEncoderPass::Encode(VkImageView source, u32 width, u32 height, u32 layers, bool bc3,
+ VkBuffer buffer, VkDeviceSize buffer_offset, VkDeviceSize buffer_size,
+ VkDeviceSize out_offset) {
+ scheduler.RequestOutsideRenderPassOperationContext();
+ compute_pass_descriptor_queue.Acquire(scheduler, 2);
+ compute_pass_descriptor_queue.AddImage(source);
+ compute_pass_descriptor_queue.AddBuffer(buffer, buffer_offset, buffer_size);
+ const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()};
+ const BcEncoderPushConstants uniforms{
+ .blocks_x = Common::DivCeil(width, 4U),
+ .blocks_y = Common::DivCeil(height, 4U),
+ .width = width,
+ .height = height,
+ .bc3 = bc3 ? 1U : 0U,
+ .out_offset = static_cast<u32>(out_offset / 8),
+ };
+ const VkPipeline vk_pipeline = *pipeline;
+ scheduler.Record([this, vk_pipeline, uniforms, layers, descriptor_data](vk::CommandBuffer cmdbuf) {
+ const VkDescriptorSet set = descriptor_allocator.Commit();
+ device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data);
+ cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline);
+ cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {});
+ cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, uniforms);
+ cmdbuf.Dispatch(Common::DivCeil(uniforms.blocks_x, 8U), Common::DivCeil(uniforms.blocks_y, 8U),
+ layers);
+ });
}
constexpr u32 BL3D_BINDING_INPUT_BUFFER = 0;
diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h
index 3d30aa6bdc..5daf4eb3de 100644
--- a/src/video_core/renderer_vulkan/vk_compute_pass.h
+++ b/src/video_core/renderer_vulkan/vk_compute_pass.h
@@ -6,6 +6,7 @@
#pragma once
+#include <functional>
#include <optional>
#include <span>
#include <utility>
@@ -130,6 +131,21 @@ public:
void Assemble(Image& image, const StagingBufferRef& map,
std::span<const VideoCommon::SwizzleParameters> swizzles);
+ /// Where a decode writes: an RGBA8 image with a storage view per (host) level.
+ struct Target {
+ VkImage image;
+ VkImageAspectFlags aspect_mask;
+ bool initialized;
+ std::function<VkImageView(s32 host_level)> level_view;
+ bool wait; ///< Wait for the GPU afterwards (the decoded image itself is sampled).
+ };
+
+ /// Decodes `info`'s ASTC data in `map` into `target`. `skip` leading levels are absent on the
+ /// host (TrimmedLevels) and `guest_size_bytes` is the full guest size.
+ void AssembleInto(const VideoCommon::ImageInfo& info, u32 skip, size_t guest_size_bytes,
+ const Target& target, const StagingBufferRef& map,
+ std::span<const VideoCommon::SwizzleParameters> swizzles);
+
private:
Scheduler& scheduler;
StagingBufferPool& staging_buffer_pool;
@@ -137,6 +153,24 @@ private:
MemoryAllocator& memory_allocator;
};
+/// Compresses an RGBA8 image level to BC1 or BC3 blocks in a buffer (bc_encoder.comp).
+class BcEncoderPass final : public ComputePass {
+public:
+ explicit BcEncoderPass(const Device& device_, Scheduler& scheduler_,
+ DescriptorPool& descriptor_pool_,
+ ComputePassDescriptorQueue& compute_pass_descriptor_queue_);
+ ~BcEncoderPass();
+
+ /// Encodes every layer of one level (`source` is its RGBA8 storage view) into `buffer`,
+ /// `out_offset` bytes past `buffer_offset`; blocks are row-major, layer after layer.
+ void Encode(VkImageView source, u32 width, u32 height, u32 layers, bool bc3, VkBuffer buffer,
+ VkDeviceSize buffer_offset, VkDeviceSize buffer_size, VkDeviceSize out_offset);
+
+private:
+ Scheduler& scheduler;
+ ComputePassDescriptorQueue& compute_pass_descriptor_queue;
+};
+
class BlockLinearUnswizzle3DPass final : public ComputePass {
public:
explicit BlockLinearUnswizzle3DPass(const Device& device_, Scheduler& scheduler_,
diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
index 8b5b0bc9c5..f737ad9f70 100644
--- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
+++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
@@ -29,6 +29,7 @@
#include "video_core/vulkan_common/vulkan_device.h"
#include "video_core/gpu_logging/gpu_logging.h"
#include "common/settings.h"
+#include "video_core/nx2_profile.h"
#if defined(_MSC_VER) && defined(NDEBUG)
#define LAMBDA_FORCEINLINE [[msvc::forceinline]]
@@ -355,7 +356,11 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) {
views.reserve(num_image_elements);
samplers.reserve(num_textures);
- texture_cache.SynchronizeDescriptors(false);
+ namespace Profile = VideoCommon::Nx2Profile;
+ {
+ Profile::Scope scope{Profile::Zone::SyncDescriptors};
+ texture_cache.SynchronizeDescriptors(false);
+ }
buffer_cache.SetUniformBuffersState(enabled_uniform_buffer_masks, &uniform_buffer_sizes);
@@ -432,24 +437,31 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) {
return true;
}};
- if constexpr (Spec::enabled_stages[0]) {
- config_stage(0);
- }
- if constexpr (Spec::enabled_stages[1]) {
- config_stage(1);
- }
- if constexpr (Spec::enabled_stages[2]) {
- config_stage(2);
- }
- if constexpr (Spec::enabled_stages[3]) {
- config_stage(3);
- }
- if constexpr (Spec::enabled_stages[4]) {
- config_stage(4);
+ {
+ Profile::Scope scope{Profile::Zone::ReadHandles};
+ if constexpr (Spec::enabled_stages[0]) {
+ config_stage(0);
+ }
+ if constexpr (Spec::enabled_stages[1]) {
+ config_stage(1);
+ }
+ if constexpr (Spec::enabled_stages[2]) {
+ config_stage(2);
+ }
+ if constexpr (Spec::enabled_stages[3]) {
+ config_stage(3);
+ }
+ if constexpr (Spec::enabled_stages[4]) {
+ config_stage(4);
+ }
}
ASSERT(views.size() == num_image_elements);
ASSERT(samplers.size() == num_textures);
- texture_cache.FillImageViews(std::span(views.data(), views.size()), false, Spec::has_images);
+ {
+ Profile::Scope scope{Profile::Zone::FillImageViews};
+ texture_cache.FillImageViews(std::span(views.data(), views.size()), false,
+ Spec::has_images);
+ }
VideoCommon::ImageViewInOut* texture_buffer_it{views.data()};
const auto bind_stage_info{[&](size_t stage) LAMBDA_FORCEINLINE {
@@ -493,44 +505,62 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) {
texture_buffer_it += Shader::NumDescriptors(info.image_descriptors);
}
}};
- if constexpr (Spec::enabled_stages[0]) {
- bind_stage_info(0);
- }
- if constexpr (Spec::enabled_stages[1]) {
- bind_stage_info(1);
- }
- if constexpr (Spec::enabled_stages[2]) {
- bind_stage_info(2);
- }
- if constexpr (Spec::enabled_stages[3]) {
- bind_stage_info(3);
- }
- if constexpr (Spec::enabled_stages[4]) {
- bind_stage_info(4);
+ {
+ Profile::Scope scope{Profile::Zone::TextureBuffers};
+ if constexpr (Spec::enabled_stages[0]) {
+ bind_stage_info(0);
+ }
+ if constexpr (Spec::enabled_stages[1]) {
+ bind_stage_info(1);
+ }
+ if constexpr (Spec::enabled_stages[2]) {
+ bind_stage_info(2);
+ }
+ if constexpr (Spec::enabled_stages[3]) {
+ bind_stage_info(3);
+ }
+ if constexpr (Spec::enabled_stages[4]) {
+ bind_stage_info(4);
+ }
}
if (regs.transform_feedback_enabled != 0) {
scheduler.RequestOutsideRenderPassOperationContext();
}
- buffer_cache.UpdateGraphicsBuffers(is_indexed);
- buffer_cache.BindHostGeometryBuffers(is_indexed);
+ {
+ Profile::Scope scope{Profile::Zone::UpdateBuffers};
+ buffer_cache.UpdateGraphicsBuffers(is_indexed);
+ buffer_cache.BindHostGeometryBuffers(is_indexed);
+ }
- guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer);
+ {
+ Profile::Scope scope{Profile::Zone::DescriptorAcquire};
+ guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer);
+ }
RescalingPushConstant rescaling;
RenderAreaPushConstant render_area;
const VideoCommon::SamplerId* samplers_it{samplers.data()};
const VideoCommon::ImageViewInOut* views_it{views.data()};
const auto prepare_stage{[&](size_t stage) LAMBDA_FORCEINLINE {
- buffer_cache.BindHostStageBuffers(stage);
- PushImageDescriptors(texture_cache, guest_descriptor_queue, stage_infos[stage], rescaling,
- samplers_it, views_it);
+ {
+ Profile::Scope scope{Profile::Zone::BindStageBuffers};
+ buffer_cache.BindHostStageBuffers(stage);
+ }
+ {
+ Profile::Scope scope{Profile::Zone::PushDescriptors};
+ PushImageDescriptors(texture_cache, guest_descriptor_queue, stage_infos[stage],
+ rescaling, samplers_it, views_it);
+ }
const auto& info{stage_infos[stage]};
if (info.uses_render_area) {
render_area.uses_render_area = true;
- render_area.words = {static_cast<float>(regs.surface_clip.width),
- static_cast<float>(regs.surface_clip.height)};
+ // Switch 2 titles may never program the surface clip; use colour target 0 then.
+ const u32 clip_width = regs.surface_clip.width, clip_height = regs.surface_clip.height;
+ const bool clip_set = clip_width != 0 && clip_height != 0;
+ render_area.words = {static_cast<float>(clip_set ? clip_width : u32{regs.rt[0].width}),
+ static_cast<float>(clip_set ? clip_height : u32{regs.rt[0].height})};
}
}};
if constexpr (Spec::enabled_stages[0]) {
@@ -552,12 +582,16 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) {
buffer_cache.runtime.PostCopyBarrier();
buffer_cache.any_buffer_uploaded = false;
}
- texture_cache.UpdateRenderTargets(false);
- texture_cache.CheckFeedbackLoop(std::span<const VideoCommon::ImageViewInOut>{views.data(),
- views.size()});
+ {
+ Profile::Scope scope{Profile::Zone::RenderTargets};
+ texture_cache.UpdateRenderTargets(false);
+ texture_cache.CheckFeedbackLoop(
+ std::span<const VideoCommon::ImageViewInOut>{views.data(), views.size()});
+ }
if (IsBuilt() && !pipeline) {
return false;
}
+ Profile::Scope scope{Profile::Zone::ConfigureDraw};
return ConfigureDraw(rescaling, render_area);
}
diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
index d4ffcd8317..60bb040984 100644
--- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
@@ -5,6 +5,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
+#include <chrono>
#include <cstddef>
#include <fstream>
#include <iostream>
@@ -15,6 +16,7 @@
#include <bit>
#include <numeric>
#include "common/cityhash.h"
+#include "common/scope_exit.h"
#include "common/fs/fs.h"
#include "common/fs/path_util.h"
#include "common/thread_worker.h"
@@ -930,7 +932,38 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline(
return nullptr;
}
+namespace {
+// Pipelines built on the GPU thread at run time (shader translation; the driver compile is on
+// workers): every one over 2 ms is logged, and the total every 10 s, to tell how much of the
+// stutter on new content is translation.
+void ReportRuntimePipeline(const char* kind, std::chrono::steady_clock::time_point begin) {
+ static std::chrono::steady_clock::duration total{};
+ static u32 count{}, logged{};
+ static auto window = std::chrono::steady_clock::now();
+ const auto now = std::chrono::steady_clock::now();
+ const auto elapsed = now - begin;
+ total += elapsed;
+ ++count;
+ const auto ms = [](auto d) { return std::chrono::duration<double, std::milli>(d).count(); };
+ if (elapsed > std::chrono::milliseconds{2} && logged < 400) {
+ ++logged;
+ LOG_INFO(Render_Vulkan, "{} pipeline built on the GPU thread in {:.1f} ms", kind, ms(elapsed));
+ }
+ if (now - window > std::chrono::seconds{10}) {
+ LOG_INFO(Render_Vulkan, "Runtime pipelines in the last 10 s: {} built, {:.1f} ms on the GPU thread",
+ count, ms(total));
+ total = {};
+ count = 0;
+ window = now;
+ }
+}
+} // Anonymous namespace
+
std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() {
+ const auto begin = std::chrono::steady_clock::now();
+ SCOPE_EXIT {
+ ReportRuntimePipeline("Graphics", begin);
+ };
GraphicsEnvironments environments;
GetGraphicsEnvironments(environments, graphics_key.unique_hashes);
@@ -956,6 +989,10 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() {
std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
const ComputePipelineCacheKey& key, const ShaderInfo* shader) {
+ const auto begin = std::chrono::steady_clock::now();
+ SCOPE_EXIT {
+ ReportRuntimePipeline("Compute", begin);
+ };
const GPUVAddr program_base{kepler_compute->regs.code_loc.Address()};
const auto& qmd{kepler_compute->launch_description};
ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start};
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index 264ffb28b5..429b2d63ef 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -7,6 +7,11 @@
#include <algorithm>
#include <array>
#include <memory>
+#include <cstdio>
+#include <cstdlib>
+#include <tuple>
+#include <set>
+#include <fstream>
#include <mutex>
#include <fmt/format.h>
@@ -14,12 +19,14 @@
#include "video_core/renderer_vulkan/renderer_vulkan.h"
#include "common/assert.h"
+#include "shader_recompiler/exception.h"
#include "common/logging.h"
#include "common/scope_exit.h"
#include "common/settings.h"
#include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/gpu_logging/gpu_logging.h"
#include "video_core/control/channel_state.h"
+#include "video_core/native_query_trace.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/engines/kepler_compute.h"
#include "video_core/engines/maxwell_3d.h"
@@ -42,6 +49,7 @@
#include "video_core/texture_cache/texture_cache_base.h"
#include "video_core/vulkan_common/vulkan_device.h"
#include "video_core/vulkan_common/vulkan_wrapper.h"
+#include "video_core/nx2_profile.h"
namespace Vulkan {
@@ -238,28 +246,296 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) {
SCOPE_EXIT {
gpu.TickWork();
};
- FlushWork();
- gpu_memory->FlushCaching();
-
- GraphicsPipeline* const pipeline{pipeline_cache.CurrentGraphicsPipeline()};
+ bool submitted = false;
+ SCOPE_EXIT {
+ auto& native = maxwell3d->GetNativeMacroEngine();
+ if (native.native_turing && !submitted) {
+ native.native_execution_failed = true;
+ LOG_DEBUG(HW_GPU, "Native graphics draw failed; refusing submission completion");
+ }
+ };
+ namespace Profile = VideoCommon::Nx2Profile;
+ {
+ Profile::Scope scope{Profile::Zone::FlushWork};
+ FlushWork();
+ gpu_memory->FlushCaching();
+ }
+
+ const auto report_failure = [&](std::string_view reason) {
+ static std::set<std::tuple<u64, u64, std::string_view>> seen;
+ const auto& regs = maxwell3d->regs;
+ const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1];
+ const u64 ps = (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1];
+ if (seen.size() < 64 && seen.emplace(vs, ps, reason).second) {
+ LOG_WARNING(HW_GPU, "Draw skipped ({}): vs={:#x} ps={:#x} rt0={:#x} {}x{}", reason, vs, ps,
+ regs.rt[0].Address(), regs.rt[0].width, regs.rt[0].height);
+ }
+ };
+ GraphicsPipeline* const pipeline{[&] {
+ Profile::Scope scope{Profile::Zone::PipelineLookup};
+ return pipeline_cache.CurrentGraphicsPipeline();
+ }()};
if (!pipeline) {
+ report_failure("no pipeline");
return;
}
std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
// update engine as channel may be different.
pipeline->SetEngine(maxwell3d, gpu_memory);
- if (!pipeline->Configure(is_indexed))
+ if (Profile::Scope scope{Profile::Zone::Configure}; !pipeline->Configure(is_indexed)) {
+ report_failure("configure failed");
return;
+ }
- UpdateDynamicStates();
+ {
+ Profile::Scope scope{Profile::Zone::DynamicState};
+ UpdateDynamicStates();
+ }
query_cache.NotifySegment(true);
HandleTransformFeedback();
query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable);
- draw_func();
+ {
+ Profile::Scope scope{Profile::Zone::DrawRecord};
+ draw_func();
+ }
+ submitted = true;
+ static u32 native_capture_count{};
+ if (maxwell3d->GetNativeMacroEngine().native_turing &&
+ native_capture_count < 16 && [] {
+ static const bool capture = std::getenv("EDEN_NATIVE_DRAW_CAPTURE") != nullptr;
+ return capture;
+ }()) {
+ const u32 capture = native_capture_count++;
+ const auto prefix = fmt::format("native-draw-{}", capture);
+ const auto dump = [&](const std::string& name, GPUVAddr address, size_t size) {
+ if (!size || !gpu_memory->IsFullyMappedRange(address, size)) return;
+ std::vector<u8> bytes(size);
+ gpu_memory->ReadBlockUnsafe(address, bytes.data(), bytes.size());
+ std::ofstream file(prefix + name + ".bin", std::ios::binary);
+ file.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+ LOG_INFO(HW_GPU, "Native draw capture {}{}: address={:#x}, bytes={}, nonzero={}",
+ capture, name, address, size,
+ std::count_if(bytes.begin(), bytes.end(), [](u8 byte) { return byte != 0; }));
+ };
+ const auto& regs = maxwell3d->regs;
+ const auto& rt = regs.rt[0];
+ const size_t size = static_cast<size_t>(rt.array_pitch) * 4;
+ if (size && size <= 16 * 1024 * 1024) {
+ if (const auto address = gpu_memory->GpuToCpuAddress(rt.Address(), size)) {
+ LOG_INFO(HW_GPU, "Native draw {} render target physical address={:#x}", capture, *address);
+ texture_cache.DownloadMemory(*address, size);
+ dump("-rt0", rt.Address(), size);
+ }
+ }
+ std::ofstream registers(prefix + "-registers.bin", std::ios::binary);
+ registers.write(reinterpret_cast<const char*>(&regs), sizeof(regs));
+ for (size_t stream = 0; stream < 4; ++stream) {
+ if (regs.vertex_streams[stream].IsEnabled())
+ dump(fmt::format("-vertex-{}", stream), regs.vertex_streams[stream].Address(), 128);
+ }
+ for (size_t stage : {0U, 4U}) {
+ for (size_t bank : {2U, 3U}) {
+ const auto& cb = maxwell3d->state.shader_stages[stage].const_buffers[bank];
+ if (cb.enabled) dump(fmt::format("-cbuf-{}-{}", stage, bank), cb.address, std::min<size_t>(cb.size, 128));
+ }
+ }
+ }
}
+
+namespace {
+// EDEN_NX2_WINDOW_TRACE: describe clears and draws into screen-sized colour targets (debug aid).
+void TraceWindowTarget(const Tegra::Engines::Maxwell3D& maxwell3d, Tegra::MemoryManager& memory,
+ const char* kind, u32 count) {
+ static const bool trace = std::getenv("EDEN_NX2_WINDOW_TRACE") != nullptr;
+ static u32 traced{};
+ const auto& regs = maxwell3d.regs;
+ static const u64 skip = [] {
+ const char* value = std::getenv("EDEN_NX2_WINDOW_TRACE_SKIP");
+ return value != nullptr ? std::strtoull(value, nullptr, 10) : 0ULL;
+ }();
+ static u64 seen_events{};
+ static const u64 only_ps = [] {
+ const char* value = std::getenv("EDEN_NX2_TRACE_PS");
+ return value != nullptr ? std::strtoull(value, nullptr, 16) : 0ULL;
+ }();
+ const u64 pixel_shader =
+ (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1];
+ if (!trace || regs.rt[0].width != 3840 || traced >= 400 || seen_events++ < skip ||
+ (only_ps != 0 && pixel_shader != only_ps)) {
+ return;
+ }
+ ++traced;
+ const auto& vp = regs.viewport_transform[0];
+ const auto& sc = regs.scissor_test[0];
+ LOG_INFO(HW_GPU,
+ "nx2 window {} rt0={:#x} count={} clear={:#x} color=({},{},{},{}) mask_common={} "
+ "mask0={:#x} rt_count={} vp scale=({},{}) translate=({},{}) scissor en={} x={}..{} "
+ "y={}..{} blend0={} vs={:#x} ps={:#x} rasterize={}",
+ kind, regs.rt[0].Address(), count, regs.clear_surface.raw, regs.clear_color[0],
+ regs.clear_color[1], regs.clear_color[2], regs.clear_color[3], regs.color_mask_common,
+ regs.color_mask[0].raw, regs.rt_control.count.Value(), vp.scale_x, vp.scale_y,
+ vp.translate_x, vp.translate_y, sc.enable, sc.min_x.Value(), sc.max_x.Value(),
+ sc.min_y.Value(), sc.max_y.Value(), regs.blend.enable[0],
+ (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1],
+ (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1],
+ regs.rasterize_enable);
+ if (kind[0] != 'd') {
+ return;
+ }
+ // Pixel-stage constant buffer 3: colour constants of the loading-screen passes.
+ if (const auto& cb3 = maxwell3d.state.shader_stages[4].const_buffers[3]; cb3.enabled) {
+ std::array<f32, 8> values{};
+ memory.ReadBlockUnsafe(cb3.address, values.data(), sizeof(values));
+ LOG_INFO(HW_GPU, "nx2 window ps c[3]@{:#x} size={}: {}", cb3.address, cb3.size,
+ fmt::join(values, " "));
+ }
+ static u32 texture_traces{};
+ if (texture_traces++ > 200) {
+ return;
+ }
+ // Texture used by the pixel shader's first bound TEX (handle in c[2][0x20]) and its TIC.
+ const auto& cb = maxwell3d.state.shader_stages[4].const_buffers[2];
+ const u32 handle = cb.enabled ? memory.Read<u32>(cb.address + 0x20) : 0;
+ const u32 tic_index = handle & 0xfffff;
+ const GPUVAddr tic_address = regs.tex_header.Address() + u64{tic_index} * 32;
+ std::array<u32, 8> tic{};
+ memory.ReadBlockUnsafe(tic_address, tic.data(), sizeof(tic));
+ LOG_INFO(HW_GPU,
+ "nx2 window texture: cb2={:#x} en={} handle={:#x} tic pool={:#x} limit={} "
+ "sampler pool={:#x} tic[{}]={:08x}",
+ cb.address, cb.enabled, handle, regs.tex_header.Address(), regs.tex_header.limit,
+ regs.tex_sampler.Address(), tic_index, fmt::join(tic, " "));
+ // First bytes of the texture's memory (block-linear, so just a sample).
+ const GPUVAddr texture = (GPUVAddr{tic[2] & 0xffff} << 32) | tic[1];
+ std::array<u32, 16> texels{};
+ memory.ReadBlockUnsafe(texture, texels.data(), sizeof(texels));
+ LOG_INFO(HW_GPU, "nx2 window texture {:#x} {}x{}: {:08x}", texture, (tic[4] & 0xffff) + 1,
+ (tic[5] & 0xffff) + 1, fmt::join(texels, " "));
+ // EDEN_NX2_TEXTURE_DUMP=<file>: save the first 4 MiB of the last traced texture's memory.
+ if (const char* path = std::getenv("EDEN_NX2_TEXTURE_DUMP"); path != nullptr) {
+ std::vector<u8> bytes(2048u * 10240u);
+ memory.ReadBlockUnsafe(texture, bytes.data(), bytes.size());
+ if (std::FILE* file = std::fopen(path, "wb")) {
+ std::fwrite(bytes.data(), 1, bytes.size(), file);
+ std::fclose(file);
+ }
+ }
+}
+} // Anonymous namespace
+
void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) {
+ VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Draw};
+ TraceWindowTarget(*maxwell3d, *gpu_memory, is_indexed ? "draw-indexed" : "draw",
+ is_indexed ? maxwell3d->draw_manager.draw_state.index_buffer.count
+ : maxwell3d->draw_manager.draw_state.vertex_buffer.count);
+ // EDEN_NX2_CB24_TRACE: for draws whose vertex shader reads bank 24, show the inline
+ // constant buffer upload target (CB_ADDRESS) and its first words.
+ if (static const bool trace = std::getenv("EDEN_NX2_CB24_TRACE") != nullptr; trace) {
+ const auto& regs = maxwell3d->regs;
+ const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1];
+ static u32 logged{};
+ if ((vs == 0x11471c9a00 || vs == 0x11471ca600 || vs == 0x11471cd300) && logged++ < 3) {
+ std::array<u32, 24> words{};
+ gpu_memory->ReadBlockUnsafe(regs.const_buffer.Address(), words.data(), sizeof(words));
+ LOG_INFO(HW_GPU, "bank24 draw vs={:#x}: CB_ADDRESS {:#x} size {:#x} offset {:#x}: {:08x}", vs,
+ regs.const_buffer.Address(), regs.const_buffer.size, regs.const_buffer.offset,
+ fmt::join(words, " "));
+ // Search every bound vertex-stage buffer for render-target-size values.
+ const auto& stage = maxwell3d->state.shader_stages[0];
+ for (u32 slot = 0; slot < Tegra::Engines::Maxwell3D::Regs::MaxConstBuffers; ++slot) {
+ const auto& buffer = stage.const_buffers[slot];
+ if (!buffer.enabled || buffer.size == 0) continue;
+ std::vector<u32> data(std::min<u32>(buffer.size, 0x10000) / 4);
+ gpu_memory->ReadBlockUnsafe(buffer.address, data.data(), data.size() * 4);
+ for (size_t i = 0; i < data.size(); ++i) {
+ const u32 w = data[i];
+ if (w == 0x45200000 || w == 0x44b40000 || w == 0x39cccccd || w == 0x3a360b61 ||
+ w == 0x44a00000 || w == 0x44340000 || w == 2559 || w == 1439) {
+ LOG_INFO(HW_GPU, " slot {} +{:#x} = {:#x}", slot, i * 4, w);
+ }
+ }
+ }
+ }
+ }
+ // EDEN_NX2_TRACE_VS=<address>: describe the next few draws using that vertex shader like the
+ // after-upscale trace does (render target, blend, fragment textures).
+ static const u64 trace_vs = [] {
+ const char* value = std::getenv("EDEN_NX2_TRACE_VS");
+ return value ? std::strtoull(value, nullptr, 16) : 0ULL;
+ }();
+ if (trace_vs != 0) {
+ const auto& regs = maxwell3d->regs;
+ const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1];
+ static u32 traced{};
+ if (vs == trace_vs && traced++ % 200 == 0 && traced < 1000) {
+ Tegra::Engines::g_nx2_trace_draws_after_upscale = 1;
+ }
+ }
+ if (Tegra::Engines::g_nx2_trace_draws_after_upscale > 0) {
+ --Tegra::Engines::g_nx2_trace_draws_after_upscale;
+ const auto& regs = maxwell3d->regs;
+ const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1];
+ const u64 ps = (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1];
+ LOG_INFO(HW_GPU, "after upscale: draw rt0 {:#x} {}x{} fmt {:#x} block {}/{}/{} pitch_linear {} depth {} array_pitch {:#x} ctrl {:#x} vs {:#x} ps {:#x}",
+ regs.rt[0].Address(), regs.rt[0].width, regs.rt[0].height,
+ static_cast<u32>(regs.rt[0].format), regs.rt[0].tile_mode.block_width.Value(),
+ regs.rt[0].tile_mode.block_height.Value(), regs.rt[0].tile_mode.block_depth.Value(),
+ regs.rt[0].tile_mode.is_pitch_linear.Value(), regs.rt[0].depth.Value(),
+ regs.rt[0].array_pitch, regs.rt_control.count.Value(), vs, ps);
+ // Fragment-stage texture handles (bound buffer 2) and the images they name.
+ const auto& handles = maxwell3d->state.shader_stages[4].const_buffers[2];
+ if (handles.enabled) {
+ for (u32 i = 0; i < 24; ++i) {
+ const u32 handle = gpu_memory->Read<u32>(handles.address + i * 4);
+ if (handle == 0 || handle == 0xffffffff) continue;
+ Tegra::Texture::TICEntry tic{};
+ gpu_memory->ReadBlockUnsafe(regs.tex_header.Address() + u64{handle & 0xfffff} * sizeof(tic),
+ &tic, sizeof(tic));
+ LOG_INFO(HW_GPU, " fs handle[{}] = {:#x}: {:#x} {}x{} format {:#x}", i, handle,
+ tic.Address(), tic.Width(), tic.Height(), static_cast<u32>(tic.format.Value()));
+ }
+ }
+ const auto& bt = regs.blend_per_target[0];
+ LOG_INFO(HW_GPU, " blend enable {} per-target {} global op {} src {:#x} dst {:#x} | rt0 op {} src {:#x} dst {:#x} alpha {:#x}/{:#x} mask {:#x}",
+ regs.blend.enable[0], regs.blend_per_target_enabled,
+ static_cast<u32>(regs.blend.color_op), static_cast<u32>(regs.blend.color_source),
+ static_cast<u32>(regs.blend.color_dest), static_cast<u32>(bt.color_op),
+ static_cast<u32>(bt.color_source), static_cast<u32>(bt.color_dest),
+ static_cast<u32>(bt.alpha_source), static_cast<u32>(bt.alpha_dest),
+ regs.color_mask[0].raw);
+ }
+ // EDEN_NX2_SKIP_VS=<address>[,<address>...]: drop draws using those vertex shaders (debug aid).
+ static const std::vector<u64> skip_list = [] {
+ std::vector<u64> list;
+ if (const char* value = std::getenv("EDEN_NX2_SKIP_VS")) {
+ for (char* end = nullptr; *value; value = *end ? end + 1 : end) {
+ list.push_back(std::strtoull(value, &end, 16));
+ if (end == value) break;
+ }
+ }
+ return list;
+ }();
+ if (!skip_list.empty()) {
+ const auto& regs = maxwell3d->regs;
+ const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1];
+ if (std::find(skip_list.begin(), skip_list.end(), vs) != skip_list.end()) return;
+ }
+ // EDEN_NX2_RT_TRACE: log the first draw to every distinct colour target (debug aid).
+ if (static const bool trace = std::getenv("EDEN_NX2_RT_TRACE") != nullptr; trace) {
+ static std::set<std::tuple<GPUVAddr, u32, u32, u64, u64>> seen;
+ const auto& regs = maxwell3d->regs;
+ const auto& rt = regs.rt[0];
+ const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1];
+ const u64 ps = (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1];
+ const auto key = std::make_tuple(rt.Address(), rt.width, static_cast<u32>(rt.format), vs, ps);
+ if (rt.width >= 640 && seen.size() < 600 && seen.insert(key).second) {
+ LOG_INFO(HW_GPU, "nx2 draw target {:#x} {}x{} format={:#x} vs={:#x} ps={:#x} blend={} (pairs so far {})",
+ rt.Address(), rt.width, rt.height, static_cast<u32>(rt.format), vs, ps,
+ regs.blend.enable[0], seen.size());
+ }
+ }
PrepareDraw(is_indexed, [this, is_indexed, instance_count] {
const auto& draw_state = maxwell3d->draw_manager.draw_state;
const u32 num_instances{instance_count};
@@ -293,6 +569,7 @@ void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) {
}
void RasterizerVulkan::DrawIndirect() {
+ VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Draw};
const auto& params = maxwell3d->draw_manager.indirect_state;
buffer_cache.SetDrawIndirect(&params);
PrepareDraw(params.is_indexed, [this, &params] {
@@ -398,6 +675,15 @@ void RasterizerVulkan::DrawTexture() {
}
void RasterizerVulkan::Clear(u32 layer_count) {
+ VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Clear};
+ if (Tegra::Engines::g_nx2_trace_draws_after_upscale > 0) {
+ const auto& regs = maxwell3d->regs;
+ LOG_INFO(HW_GPU, "after upscale: clear rt0 {:#x} {}x{} fmt {:#x} surface {:#x} colour {} {} {} {}",
+ regs.rt[0].Address(), regs.rt[0].width, regs.rt[0].height,
+ static_cast<u32>(regs.rt[0].format), regs.clear_surface.raw, regs.clear_color[0],
+ regs.clear_color[1], regs.clear_color[2], regs.clear_color[3]);
+ }
+ TraceWindowTarget(*maxwell3d, *gpu_memory, "clear", layer_count);
FlushWork();
gpu_memory->FlushCaching();
@@ -583,16 +869,19 @@ void RasterizerVulkan::Clear(u32 layer_count) {
}
void RasterizerVulkan::DispatchCompute() {
+ VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Compute};
FlushWork();
gpu_memory->FlushCaching();
ComputePipeline* const pipeline{pipeline_cache.CurrentComputePipeline()};
if (!pipeline) {
+ if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute pipeline unavailable");
return;
}
std::scoped_lock lock{texture_cache.mutex, buffer_cache.mutex};
if (!pipeline->Configure(*kepler_compute, *gpu_memory, scheduler, buffer_cache,
texture_cache)) {
+ if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute resources unavailable");
return;
}
@@ -942,19 +1231,35 @@ Tegra::Engines::AccelerateDMAInterface& RasterizerVulkan::AccessAccelerateDMA()
void RasterizerVulkan::AccelerateInlineToMemory(GPUVAddr address, size_t copy_size,
std::span<const u8> memory) {
auto cpu_addr = gpu_memory->GpuToCpuAddress(address);
+ if (gpu_memory->UsesNativeTiling()) {
+ u32 first_word{};
+ if (memory.size() >= sizeof(first_word)) std::memcpy(&first_word, memory.data(), sizeof(first_word));
+ VideoCommon::TraceNativeQuery(11, gpu_memory->GetID(), address, cpu_addr.value_or(0),
+ 0, first_word, static_cast<u32>(copy_size));
+ static const GPUVAddr watch = [] {
+ const char* value = std::getenv("EDEN_NX2_SEM_WATCH");
+ return value ? GPUVAddr{std::strtoull(value, nullptr, 16)} : GPUVAddr{};
+ }();
+ if (watch != 0 && address < watch + 16 && address + copy_size > watch) {
+ LOG_INFO(Render_Vulkan, "Semaphore watch: inline write {:#x} size {} first word {:#x}",
+ address, copy_size, first_word);
+ }
+ }
if (!cpu_addr) [[unlikely]] {
gpu_memory->WriteBlock(address, memory.data(), copy_size);
return;
}
gpu_memory->WriteBlockUnsafe(address, memory.data(), copy_size);
+ // One cache at a time: holding both while waiting for either (std::lock) waited longer, the
+ // fencing thread and CPU write traps holding one of them most of the time.
{
- std::unique_lock<std::recursive_mutex> lock{buffer_cache.mutex};
+ std::scoped_lock lock{buffer_cache.mutex};
if (!buffer_cache.InlineMemory(*cpu_addr, copy_size, memory)) {
buffer_cache.WriteMemory(*cpu_addr, copy_size);
}
}
{
- std::scoped_lock lock_texture{texture_cache.mutex};
+ std::scoped_lock lock{texture_cache.mutex};
texture_cache.WriteMemory(*cpu_addr, copy_size);
}
pipeline_cache.InvalidateRegion(*cpu_addr, copy_size);
@@ -969,6 +1274,13 @@ std::optional<FramebufferTextureInfo> RasterizerVulkan::AccelerateDisplay(
std::scoped_lock lock{texture_cache.mutex};
const auto [image_view, scaled] =
texture_cache.TryFindFramebufferImageView(config, framebuffer_addr);
+ static u32 display_trace_count{};
+ static const bool frame_capture = std::getenv("EDEN_NATIVE_FRAME_CAPTURE") != nullptr;
+ if (frame_capture && display_trace_count++ < 64) {
+ LOG_INFO(HW_GPU, "Native display source: address={:#x}, offset={:#x}, dimensions={}x{}, stride={}, accelerated={}, mask={:#x}",
+ framebuffer_addr, config.offset, config.width, config.height, pixel_stride,
+ image_view != nullptr, config.layer_stack_mask);
+ }
if (!image_view) {
return {};
}
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index 583cb6edaf..f5fcdd1380 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -7,6 +7,7 @@
#include <algorithm>
#include <limits>
#include <array>
+#include <cstdlib>
#include <optional>
#include <span>
#include <memory>
@@ -148,10 +149,68 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
return usage;
}
+// An ASTC 2D array whose RGBA8 decode exceeds 1 GiB (title 1's 8192x8192x68 terrain textures).
+// These stay on the GPU decoder as RGBA8 with trimmed levels (TrimmedHostInfo) even when other
+// ASTC images are re-compressed to BC1/BC3 on the CPU: re-compressing them would not fit either.
+[[nodiscard]] bool IsHugeAstcArray(const Device& device, const ImageInfo& info) {
+ if (!IsPixelFormatASTC(info.format) || device.IsOptimalAstcSupported() ||
+ info.type != ImageType::e2D || info.size.depth != 1) {
+ return false;
+ }
+ u64 bytes = 0;
+ for (s32 level = 0; level < info.resources.levels; ++level) {
+ bytes += u64{(std::max)(info.size.width >> level, 1U)} *
+ (std::max)(info.size.height >> level, 1U) * 4;
+ }
+ return bytes * info.resources.layers > (u64{1} << 30);
+}
+
+// The host format of a huge ASTC array: the GPU decoder's RGBA8 (sRGB kept for sampling).
+[[nodiscard]] std::optional<VkFormat> HugeAstcHostFormat(const Device& device, const ImageInfo& info) {
+ if (!IsHugeAstcArray(device, info)) {
+ return std::nullopt;
+ }
+ return VideoCore::Surface::IsPixelFormatSRGB(info.format) ? VK_FORMAT_A8B8G8R8_SRGB_PACK32
+ : VK_FORMAT_A8B8G8R8_UNORM_PACK32;
+}
+
+constexpr std::array<VkFormat, 2> HugeAstcViewFormats{VK_FORMAT_A8B8G8R8_UNORM_PACK32,
+ VK_FORMAT_A8B8G8R8_SRGB_PACK32};
+
+// Opaque ASTC (ImageInfo::astc_opaque) re-compressed to BC1 instead of BC3.
+[[nodiscard]] bool IsOpaqueBc1Astc(const Device& device, const ImageInfo& info) {
+ return info.astc_opaque && IsPixelFormatASTC(info.format) && !IsHugeAstcArray(device, info);
+}
+
+constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM_BLOCK,
+ VK_FORMAT_BC1_RGBA_SRGB_BLOCK};
+
+[[nodiscard]] std::optional<VkFormat> AstcHostFormat(const Device& device, const ImageInfo& info) {
+ if (IsOpaqueBc1Astc(device, info)) {
+ return VideoCore::Surface::IsPixelFormatSRGB(info.format) ? VK_FORMAT_BC1_RGBA_SRGB_BLOCK
+ : VK_FORMAT_BC1_RGBA_UNORM_BLOCK;
+ }
+ return HugeAstcHostFormat(device, info);
+}
+
+// ASTC that the GPU decodes and re-compresses to BC1/BC3 itself (TranscodeAstcToBcn): no CPU
+// decode, so streamed textures are ready the frame they are first used instead of seconds later.
+[[nodiscard]] bool WillUseGpuBcnTranscode(const Device& device, const ImageInfo& info) {
+ if (!IsPixelFormatASTC(info.format) || device.IsOptimalAstcSupported() ||
+ IsHugeAstcArray(device, info) || info.type != ImageType::e2D || info.size.depth != 1) {
+ return false;
+ }
+ return Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu &&
+ Settings::values.astc_recompression.GetValue() != Settings::AstcRecompression::Uncompressed;
+}
+
[[nodiscard]] bool WillUseAcceleratedAstcDecode(const Device& device, const ImageInfo& info) {
if (!IsPixelFormatASTC(info.format) || device.IsOptimalAstcSupported()) {
return false;
}
+ if (IsHugeAstcArray(device, info)) {
+ return true;
+ }
if (Settings::values.accelerate_astc.GetValue() != Settings::AstcDecodeMode::Gpu) {
return false;
}
@@ -160,6 +219,40 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
info.size.depth == 1;
}
+// The host shape of an image whose decoded copy would not fit in VRAM: a GPU-decoded ASTC array
+// over 1 GiB as RGBA8 (title 1's 8192x8192x68 ASTC 8x8 virtual textures are 24 GB each) keeps only
+// its mip levels from `level_offset` on, so host mip 0 is guest mip `level_offset`. Normalized
+// coordinates and derivative LOD still address it correctly; explicit LODs sample one level lower.
+[[nodiscard]] ImageInfo TrimmedHostInfo(const Device& device, const ImageInfo& info, u32& level_offset) {
+ level_offset = 0;
+ if (!WillUseAcceleratedAstcDecode(device, info) || info.type != ImageType::e2D) {
+ return info;
+ }
+ const auto decoded_bytes = [&](u32 skip) {
+ u64 bytes = 0;
+ for (s32 level = static_cast<s32>(skip); level < info.resources.levels; ++level) {
+ bytes += u64{(std::max)(info.size.width >> level, 1U)} *
+ (std::max)(info.size.height >> level, 1U) * 4;
+ }
+ return bytes * info.resources.layers;
+ };
+ if (decoded_bytes(0) <= (u64{1} << 30)) {
+ return info;
+ }
+ while (level_offset + 1 < static_cast<u32>(info.resources.levels) &&
+ decoded_bytes(level_offset) > (u64{512} << 20)) {
+ ++level_offset;
+ }
+ ImageInfo host = info;
+ host.size.width = (std::max)(info.size.width >> level_offset, 1U);
+ host.size.height = (std::max)(info.size.height >> level_offset, 1U);
+ host.resources.levels -= static_cast<s32>(level_offset);
+ LOG_WARNING(Render_Vulkan, "{}x{}x{} ASTC image would be {} MiB decoded; keeping mip levels {}+ ({}x{})",
+ info.size.width, info.size.height, info.resources.layers, decoded_bytes(0) >> 20,
+ level_offset, host.size.width, host.size.height);
+ return host;
+}
+
[[nodiscard]] VkImageCreateInfo MakeImageCreateInfo(const Device& device, const ImageInfo& info,
std::optional<VkFormat> format_override = {}) {
auto format_info =
@@ -527,6 +620,20 @@ TransformBufferCopies(std::span<const VideoCommon::BufferCopy> copies, size_t bu
};
}
+// A guest view's levels on a host image that keeps only levels >= host_level_offset.
+[[nodiscard]] SubresourceRange HostRange(const Image& image, SubresourceRange range) {
+ const s32 skip = static_cast<s32>(image.host_level_offset);
+ if (skip == 0) {
+ return range;
+ }
+ const s32 host_levels = image.info.resources.levels - skip;
+ const s32 base = std::clamp(range.base.level - skip, 0, host_levels - 1);
+ const s32 end = std::clamp(range.base.level + range.extent.levels - skip, base + 1, host_levels);
+ range.base.level = base;
+ range.extent.levels = end - base;
+ return range;
+}
+
[[nodiscard]] VkImageSubresourceRange MakeSubresourceRange(const ImageView* image_view) {
SubresourceRange range = image_view->range;
if (True(image_view->flags & VideoCommon::ImageViewFlagBits::Slice)) {
@@ -942,9 +1049,16 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched
: device{device_}, scheduler{scheduler_}, memory_allocator{memory_allocator_},
staging_buffer_pool{staging_buffer_pool_}, blit_image_helper{blit_image_helper_},
render_pass_cache{render_pass_cache_}, resolution{Settings::values.resolution_info} {
- if (Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu) {
+ // Huge ASTC arrays always decode on the GPU (IsHugeAstcArray), whatever the decode setting.
+ if (Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu ||
+ !device.IsOptimalAstcSupported()) {
astc_decoder_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool,
compute_pass_descriptor_queue, memory_allocator);
+ if (Settings::values.astc_recompression.GetValue() !=
+ Settings::AstcRecompression::Uncompressed) {
+ bc_encoder_pass.emplace(device, scheduler, descriptor_pool,
+ compute_pass_descriptor_queue);
+ }
}
if (!device.IsKhrImageFormatListSupported()) {
return;
@@ -1796,6 +1910,10 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const {
return device.GetDeviceMemoryUsage();
}
+u64 TextureCacheRuntime::GetDeviceMemoryBudget() const {
+ return device.GetDeviceMemoryBudget();
+}
+
bool TextureCacheRuntime::CanDownloadMsaa(const VideoCommon::ImageInfo& info) const {
const VkImageAspectFlags aspect_mask = ImageAspectMask(info.format);
if ((aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0) {
@@ -1814,6 +1932,20 @@ bool TextureCacheRuntime::CanReportMemoryUsage() const {
return device.CanReportMemoryUsage();
}
+void TextureCacheRuntime::LogMemoryStatistics() const {
+ Vulkan::LogMemoryStatistics(device);
+}
+
+bool TextureCacheRuntime::UseOpaqueAstcFormat() const {
+ // EDEN_NX2_ASTC_BC1=0 keeps every re-compressed ASTC image in BC3.
+ static const bool enabled = [] {
+ const char* value = std::getenv("EDEN_NX2_ASTC_BC1");
+ return value == nullptr || value[0] != '0';
+ }();
+ return enabled && !device.IsOptimalAstcSupported() &&
+ Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Bc3;
+}
+
void TextureCacheRuntime::FlushDeferredClear() {
scheduler.FlushDeferredClear();
}
@@ -1859,6 +1991,9 @@ void TextureCacheRuntime::ReleaseMsaaScratchImage(VkImage image) {
}
void TextureCacheRuntime::TickFrame() {
+ while (!pending_transcodes.empty() && scheduler.IsFree(pending_transcodes.front().tick)) {
+ pending_transcodes.pop_front();
+ }
static constexpr u32 MAX_UNUSED_SCRATCH_FRAMES = 60;
std::erase_if(msaa_scratch_images, [this](MsaaScratchImage& scratch) {
if (!scheduler.IsFree(scratch.tick)) {
@@ -1876,13 +2011,30 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu
VAddr cpu_addr_)
: VideoCommon::ImageBase(info_, gpu_addr_, cpu_addr_), scheduler{&runtime_.scheduler},
runtime{&runtime_},
- original_image(MakeImage(runtime_.device, runtime_.memory_allocator, info,
- runtime->ViewFormats(info.format))),
+ original_image(MakeImage(runtime_.device, runtime_.memory_allocator,
+ TrimmedHostInfo(runtime_.device, info, host_level_offset),
+ IsHugeAstcArray(runtime_.device, info)
+ ? std::span<const VkFormat>{HugeAstcViewFormats}
+ : IsOpaqueBc1Astc(runtime_.device, info)
+ ? std::span<const VkFormat>{OpaqueAstcViewFormats}
+ : runtime->ViewFormats(info.format),
+ AstcHostFormat(runtime_.device, info))),
aspect_mask(ImageAspectMask(info.format)) {
- if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) {
+ if (original_image.IsUnbacked()) {
+ flags |= VideoCommon::ImageFlagBits::Unbacked;
+ }
+ if (host_level_offset != 0) {
+ flags |= VideoCommon::ImageFlagBits::TrimmedLevels;
+ }
+ if (IsHugeAstcArray(runtime->device, info)) {
+ flags |= VideoCommon::ImageFlagBits::AcceleratedUpload;
+ flags |= VideoCommon::ImageFlagBits::Converted;
+ flags |= VideoCommon::ImageFlagBits::CostlyLoad;
+ } else if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) {
switch (Settings::values.accelerate_astc.GetValue()) {
case Settings::AstcDecodeMode::Gpu:
- if (WillUseAcceleratedAstcDecode(runtime->device, info)) {
+ if (WillUseAcceleratedAstcDecode(runtime->device, info) ||
+ WillUseGpuBcnTranscode(runtime->device, info)) {
flags |= VideoCommon::ImageFlagBits::AcceleratedUpload;
}
break;
@@ -1903,11 +2055,12 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu
original_image.SetObjectNameEXT(VideoCommon::Name(*this).c_str());
}
current_image = &Image::original_image;
- storage_image_views.resize(info.resources.levels);
+ const s32 host_levels = info.resources.levels - static_cast<s32>(host_level_offset);
+ storage_image_views.resize(host_levels);
if (WillUseAcceleratedAstcDecode(runtime->device, info)) {
const auto& device = runtime->device.GetLogical();
const VkFormat storage_format = VK_FORMAT_A8B8G8R8_UNORM_PACK32;
- for (s32 level = 0; level < info.resources.levels; ++level) {
+ for (s32 level = 0; level < host_levels; ++level) {
storage_image_views[level] =
MakeStorageView(device, level, *original_image, storage_format);
}
@@ -2085,6 +2238,11 @@ void Image::DownloadMemory(VkBuffer buffer, size_t offset,
void Image::DownloadMemory(std::span<VkBuffer> buffers_span, std::span<size_t> offsets_span,
std::span<const VideoCommon::BufferImageCopy> copies) {
+ if (!original_image) {
+ LOG_CRITICAL(Render_Vulkan, "Missing image at download: gpu={:#x}, cpu={:#x}, type={}, format={}, flags={:#x}, size={}, dimensions={}x{}",
+ gpu_addr, cpu_addr, static_cast<u32>(info.type), static_cast<u32>(info.format),
+ static_cast<u32>(flags), unswizzled_size_bytes, info.size.width, info.size.height);
+ }
const bool is_rescaled = True(flags & ImageFlagBits::Rescaled);
if (is_rescaled) {
ScaleDown();
@@ -2519,6 +2677,15 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI
}
}
auto format_info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format);
+ if (IsHugeAstcArray(*device, image.info)) {
+ format_info.format = VideoCore::Surface::IsPixelFormatSRGB(format)
+ ? VK_FORMAT_A8B8G8R8_SRGB_PACK32
+ : VK_FORMAT_A8B8G8R8_UNORM_PACK32;
+ } else if (IsOpaqueBc1Astc(*device, image.info) && IsPixelFormatASTC(format)) {
+ format_info.format = VideoCore::Surface::IsPixelFormatSRGB(format)
+ ? VK_FORMAT_BC1_RGBA_SRGB_BLOCK
+ : VK_FORMAT_BC1_RGBA_UNORM_BLOCK;
+ }
if (device->ApiVersion() >= VK_API_VERSION_1_3) {
const VkFormatProperties3 properties3 =
device->GetPhysical().GetFormatProperties3(format_info.format);
@@ -2567,7 +2734,7 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI
.viewType = VkImageViewType{},
.format = format_info.format,
.components = swizzle_mapping,
- .subresourceRange = MakeSubresourceRange(aspect_mask, info.range),
+ .subresourceRange = MakeSubresourceRange(aspect_mask, HostRange(image, info.range)),
};
const auto create = [&](TextureType tex_type, std::optional<u32> num_layers) {
VkImageViewCreateInfo ci{create_info};
@@ -2821,6 +2988,11 @@ Sampler::Sampler(TextureCacheRuntime& runtime, const Tegra::Texture::TSCEntry& t
if (tsc.mipmap_filter != TextureMipmapFilter::None) {
min_lod = tsc.MinLod();
max_lod = tsc.MaxLod();
+ // EDEN_NX2_FORCE_LOD0: sample only the top mip level (debug aid for mip layout problems).
+ if (static const bool force_lod0 = std::getenv("EDEN_NX2_FORCE_LOD0") != nullptr; force_lod0) {
+ min_lod = 0.0f;
+ max_lod = 0.25f;
+ }
}
base_ci = VkSamplerCreateInfo{
.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
@@ -3133,6 +3305,9 @@ void TextureCacheRuntime::AccelerateImageUpload(
u32 z_start, u32 z_count) {
if (IsPixelFormatASTC(image.info.format)) {
+ if (bc_encoder_pass && WillUseGpuBcnTranscode(device, image.info)) {
+ return TranscodeAstcToBcn(image, map, swizzles);
+ }
return astc_decoder_pass->Assemble(image, map, swizzles);
}
@@ -3151,6 +3326,100 @@ void TextureCacheRuntime::AccelerateImageUpload(
ASSERT(false);
}
+void TextureCacheRuntime::TranscodeAstcToBcn(
+ Image& image, const StagingBufferRef& map,
+ std::span<const VideoCommon::SwizzleParameters> swizzles) {
+ const ImageInfo& info = image.info;
+ const u32 width = info.size.width;
+ const u32 height = info.size.height;
+ const u32 levels = static_cast<u32>(info.resources.levels);
+ const u32 layers = static_cast<u32>(info.resources.layers);
+ constexpr VkFormat DecodeFormat = VK_FORMAT_A8B8G8R8_UNORM_PACK32;
+ PendingTranscode pending{
+ .tick = 0,
+ .image = memory_allocator.CreateImage(VkImageCreateInfo{
+ .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .imageType = VK_IMAGE_TYPE_2D,
+ .format = DecodeFormat,
+ .extent = {width, height, 1},
+ .mipLevels = levels,
+ .arrayLayers = layers,
+ .samples = VK_SAMPLE_COUNT_1_BIT,
+ .tiling = VK_IMAGE_TILING_OPTIMAL,
+ .usage = VK_IMAGE_USAGE_STORAGE_BIT,
+ .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
+ .queueFamilyIndexCount = 0,
+ .pQueueFamilyIndices = nullptr,
+ .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
+ }, true),
+ .views = {},
+ };
+ for (u32 level = 0; level < levels; ++level) {
+ pending.views.push_back(
+ MakeStorageView(device.GetLogical(), level, *pending.image, DecodeFormat));
+ }
+ astc_decoder_pass->AssembleInto(
+ info, 0, image.guest_size_bytes,
+ ASTCDecoderPass::Target{
+ .image = *pending.image,
+ .aspect_mask = VK_IMAGE_ASPECT_COLOR_BIT,
+ .initialized = false,
+ .level_view = [&pending](s32 level) { return *pending.views[level]; },
+ .wait = false,
+ },
+ map, swizzles);
+
+ // Encode every level into one buffer, then copy it into the image.
+ const bool bc3 = !IsOpaqueBc1Astc(device, info) &&
+ Settings::values.astc_recompression.GetValue() !=
+ Settings::AstcRecompression::Bc1;
+ const u64 block_bytes = bc3 ? 16 : 8;
+ boost::container::small_vector<VideoCommon::BufferImageCopy, 16> copies;
+ u64 total = 0;
+ for (u32 level = 0; level < levels; ++level) {
+ const u32 level_width = (std::max)(width >> level, 1U);
+ const u32 level_height = (std::max)(height >> level, 1U);
+ const u32 blocks_x = Common::DivCeil(level_width, 4U);
+ const u32 blocks_y = Common::DivCeil(level_height, 4U);
+ const u64 size = u64{blocks_x} * blocks_y * layers * block_bytes;
+ copies.push_back({
+ .buffer_offset = total,
+ .buffer_size = size,
+ .buffer_row_length = blocks_x * 4,
+ .buffer_image_height = blocks_y * 4,
+ .image_subresource{
+ .base_level = static_cast<s32>(level),
+ .base_layer = 0,
+ .num_layers = static_cast<s32>(layers),
+ },
+ .image_offset{0, 0, 0},
+ .image_extent{level_width, level_height, 1},
+ });
+ total += size;
+ }
+ const auto output = staging_buffer_pool.Request(total, MemoryUsage::DeviceLocal);
+ for (u32 level = 0; level < levels; ++level) {
+ bc_encoder_pass->Encode(*pending.views[level], copies[level].image_extent.width,
+ copies[level].image_extent.height, layers, bc3, output.buffer,
+ output.offset, total, copies[level].buffer_offset);
+ }
+ scheduler.Record([](vk::CommandBuffer cmdbuf) {
+ const VkMemoryBarrier barrier{
+ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+ .pNext = nullptr,
+ .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
+ .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
+ };
+ cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT,
+ 0, barrier);
+ });
+ image.UploadMemory(output.buffer, output.offset, copies);
+ pending.tick = scheduler.CurrentTick();
+ pending_transcodes.push_back(std::move(pending));
+}
+
void TextureCacheRuntime::TransitionImageLayout(Image& image) {
if (!image.ExchangeInitialization()) {
VkImageMemoryBarrier barrier{
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h
index 2e9736f707..1d92d439f4 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.h
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.h
@@ -6,6 +6,7 @@
#pragma once
+#include <deque>
#include <span>
#include "video_core/texture_cache/texture_cache_base.h"
@@ -66,8 +67,15 @@ public:
u64 GetDeviceMemoryUsage() const;
+ u64 GetDeviceMemoryBudget() const;
+
bool CanReportMemoryUsage() const;
+ void LogMemoryStatistics() const;
+
+ /// True when ASTC is re-compressed to BC3 and opaque images may use BC1 instead.
+ bool UseOpaqueAstcFormat() const;
+
bool CanDownloadMsaa(const VideoCommon::ImageInfo& info) const;
[[nodiscard]] VkImage AcquireMsaaScratchImage(const VkImageCreateInfo& image_ci);
@@ -157,6 +165,20 @@ public:
BlitImageHelper& blit_image_helper;
RenderPassCache& render_pass_cache;
std::optional<ASTCDecoderPass> astc_decoder_pass;
+ std::optional<BcEncoderPass> bc_encoder_pass;
+
+ /// ASTC decoded on the GPU and re-compressed to BC1/BC3 there (accelerate_astc=Gpu with
+ /// astc_recompression=Bc1/Bc3): decode into a temporary RGBA8 image, encode, copy.
+ void TranscodeAstcToBcn(Image& image, const StagingBufferRef& map,
+ std::span<const VideoCommon::SwizzleParameters> swizzles);
+
+ /// Temporary decode images, freed once the GPU is past `tick`.
+ struct PendingTranscode {
+ u64 tick;
+ vk::Image image;
+ std::vector<vk::ImageView> views;
+ };
+ std::deque<PendingTranscode> pending_transcodes;
std::optional<BlockLinearUnswizzle3DPass> bl3d_unswizzle_pass;
const Settings::ResolutionScalingInfo& resolution;
diff --git a/src/video_core/shader_cache.cpp b/src/video_core/shader_cache.cpp
index f1aff2810f..dfe80abac3 100644
--- a/src/video_core/shader_cache.cpp
+++ b/src/video_core/shader_cache.cpp
@@ -5,12 +5,17 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
+#include <cstring>
+#include <string>
+#include <unordered_set>
+#include <fmt/ranges.h>
#include <array>
#include <vector>
#include <boost/container/small_vector.hpp>
#include "common/assert.h"
#include "shader_recompiler/frontend/maxwell/control_flow.h"
+#include "shader_recompiler/exception.h"
#include "shader_recompiler/object_pool.h"
#include "video_core/control/channel_state.h"
#include "video_core/dirty_flags.h"
@@ -43,7 +48,7 @@ void ShaderCache::SyncGuestHost() {
ShaderCache::ShaderCache(Tegra::MaxwellDeviceMemoryManager& device_memory_)
: device_memory{device_memory_} {}
-bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) {
+bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try {
auto& dirty{maxwell3d->dirty.flags};
if (!dirty[VideoCommon::Dirty::Shaders]) {
return last_shaders_valid;
@@ -51,6 +56,7 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) {
dirty[VideoCommon::Dirty::Shaders] = false;
const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()};
+ const bool native = maxwell3d->GetNativeMacroEngine().native_turing;
for (size_t index = 0; index < Tegra::Engines::Maxwell3D::Regs::MaxShaderProgram; ++index) {
if (!maxwell3d->regs.IsShaderConfigEnabled(index)) {
unique_hashes[index] = 0;
@@ -63,7 +69,9 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) {
unique_hashes[index] = 0;
continue;
}
- const GPUVAddr shader_addr{base_addr + shader_config.offset};
+ const GPUVAddr shader_addr{native
+ ? (static_cast<u64>(shader_config.reserved[0] & 0xff) << 32) | shader_config.reserved[1]
+ : base_addr + shader_config.offset};
const std::optional<VAddr> cpu_shader_addr{gpu_memory->GpuToCpuAddress(shader_addr)};
if (!cpu_shader_addr) {
LOG_ERROR(HW_GPU, "Invalid GPU address for shader {:#016x}", shader_addr);
@@ -71,16 +79,68 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) {
return false;
}
const ShaderInfo* shader_info{TryGet(*cpu_shader_addr)};
+ if (!shader_info && native && IsRejected(*cpu_shader_addr)) {
+ // A rejected program fails the same way every time it is bound: skip its draws
+ // without rethrowing (two C++ throws each time cost ~1-2% of the GPU thread).
+ maxwell3d->GetNativeMacroEngine().native_execution_failed = true;
+ last_shaders_valid = false;
+ return false;
+ }
if (!shader_info) {
- const u32 start_address{shader_config.offset};
- GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, base_addr, start_address};
- shader_info = MakeShaderInfo(env, *cpu_shader_addr);
+ const u32 start_address{native ? 0 : shader_config.offset};
+ try {
+ // Native programs are lowered while the environment is built.
+ RethrowIfRejected(*cpu_shader_addr);
+ GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, native ? shader_addr : base_addr, start_address};
+ shader_info = MakeShaderInfo(env, *cpu_shader_addr);
+ } catch (const Shader::Exception&) {
+ RememberRejection(*cpu_shader_addr);
+ // Once per program: which stage slot it is bound to, its header and the slot's
+ // registers (to tell how a pipeline with task/mesh programs is set up).
+ static std::unordered_set<GPUVAddr> described;
+ if (native && described.insert(shader_addr).second) {
+ std::array<u32, 5> header{};
+ gpu_memory->ReadBlockUnsafe(shader_addr, header.data(), sizeof(header));
+ std::array<u32, sizeof(shader_config) / sizeof(u32)> config{};
+ std::memcpy(config.data(), &shader_config, sizeof(config));
+ LOG_ERROR(HW_GPU, "Rejected program {:#x} is bound to stage {}: header {:08x}, slot {:08x}",
+ shader_addr, index, fmt::join(header, " "), fmt::join(config, " "));
+ }
+ throw;
+ }
}
shader_infos[index] = shader_info;
unique_hashes[index] = shader_info->unique_hash;
}
last_shaders_valid = true;
return true;
+} catch (const Shader::Exception& error) {
+ auto& native = maxwell3d->GetNativeMacroEngine();
+ if (!native.native_turing) throw;
+ native.native_execution_failed = true;
+ last_shaders_valid = false;
+ // Each rejected program is logged once (the draws that use it keep failing quietly).
+ static std::unordered_set<std::string> logged;
+ if (logged.insert(error.what()).second) {
+ LOG_ERROR(HW_GPU, "Native graphics shader rejected: {}", error.what());
+ }
+ // Once: show what the vertex stage has bound, to find where an unbound bank's data lives.
+ static std::vector<std::string> dumped_errors;
+ if (const std::string what{error.what()};
+ what.find("constant buffer") != std::string::npos && dumped_errors.size() < 12 &&
+ std::count(dumped_errors.begin(), dumped_errors.end(), what) < 3) {
+ dumped_errors.push_back(what);
+ LOG_INFO(HW_GPU, "VS cbuf dump for: {}", what.substr(0, 40));
+ for (u32 slot = 3; slot < 8; ++slot) {
+ const auto& buffer = maxwell3d->state.shader_stages[0].const_buffers[slot];
+ if (!buffer.enabled) continue;
+ std::array<u32, 24> words{};
+ gpu_memory->ReadBlockUnsafe(buffer.address, words.data(), sizeof(words));
+ LOG_INFO(HW_GPU, "VS cbuf {} at {:#x} size {:#x}: {:08x}", slot, buffer.address, buffer.size,
+ fmt::join(words, " "));
+ }
+ }
+ return false;
}
const ShaderInfo* ShaderCache::ComputeShader() {
@@ -95,22 +155,54 @@ const ShaderInfo* ShaderCache::ComputeShader() {
if (const ShaderInfo* const shader = TryGet(*cpu_shader_addr)) {
return shader;
}
- ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start};
- return MakeShaderInfo(env, *cpu_shader_addr);
+ try {
+ RethrowIfRejected(*cpu_shader_addr);
+ ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start};
+ return MakeShaderInfo(env, *cpu_shader_addr);
+ } catch (const Shader::Exception&) {
+ RememberRejection(*cpu_shader_addr);
+ throw;
+ }
+}
+
+void ShaderCache::RethrowIfRejected(VAddr addr) {
+ std::scoped_lock lock{rejected_mutex};
+ if (const auto it = rejected_native.find(addr); it != rejected_native.end()) {
+ std::rethrow_exception(it->second);
+ }
+}
+
+bool ShaderCache::IsRejected(VAddr addr) {
+ std::scoped_lock lock{rejected_mutex};
+ return rejected_native.contains(addr);
+}
+
+void ShaderCache::RememberRejection(VAddr addr) {
+ std::scoped_lock lock{rejected_mutex};
+ rejected_native.emplace(addr, std::current_exception());
+ rejected_lowest.store((std::min)(rejected_lowest.load(std::memory_order_relaxed), addr),
+ std::memory_order_relaxed);
+ rejected_highest.store((std::max)(rejected_highest.load(std::memory_order_relaxed), addr),
+ std::memory_order_relaxed);
}
void ShaderCache::GetGraphicsEnvironments(GraphicsEnvironments& result,
const std::array<u64, NUM_PROGRAMS>& unique_hashes) {
size_t env_index{};
const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()};
+ const bool native = maxwell3d->GetNativeMacroEngine().native_turing;
for (size_t index = 0; index < NUM_PROGRAMS; ++index) {
if (unique_hashes[index] == 0) {
continue;
}
const auto program{static_cast<Tegra::Engines::Maxwell3D::Regs::ShaderType>(index)};
auto& env{result.envs[index]};
- const u32 start_address{maxwell3d->regs.pipelines[index].offset};
- env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, base_addr, start_address};
+ const auto& config = maxwell3d->regs.pipelines[index];
+ const u32 start_address{native ? 0 : config.offset};
+ const GPUVAddr address = native
+ ? (static_cast<u64>(config.reserved[0] & 0xff) << 32) | config.reserved[1]
+ : base_addr;
+ env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, address, start_address};
env.SetCachedSize(shader_infos[index]->size_bytes);
result.env_ptrs[env_index++] = &env;
}
@@ -144,6 +236,15 @@ void ShaderCache::Register(std::unique_ptr<ShaderInfo> data, VAddr addr, size_t
void ShaderCache::InvalidatePagesInRegion(VAddr addr, size_t size) {
const VAddr addr_end = addr + size;
+ // Rejected programs are few and in a few places: look at them only for writes that can
+ // reach them (this runs for every inline upload and pre-draw invalidation).
+ if (addr_end > rejected_lowest.load(std::memory_order_relaxed) &&
+ addr <= rejected_highest.load(std::memory_order_relaxed)) {
+ std::scoped_lock lock{rejected_mutex};
+ boost::unordered::erase_if(rejected_native, [&](const auto& rejected) {
+ return rejected.first >= addr && rejected.first < addr_end;
+ });
+ }
const u64 page_end = (addr_end + YUZU_PAGESIZE - 1) >> YUZU_PAGEBITS;
for (u64 page = addr >> YUZU_PAGEBITS; page < page_end; ++page) {
auto it = invalidation_cache.find(page);
@@ -241,7 +342,7 @@ const ShaderInfo* ShaderCache::MakeShaderInfo(GenericEnvironment& env, VAddr cpu
info->unique_hash = env.CalculateHash();
info->size_bytes = env.ReadSizeBytes();
}
- const size_t size_bytes{info->size_bytes};
+ const size_t size_bytes{env.SourceSizeBytes() ? env.SourceSizeBytes() : info->size_bytes};
const ShaderInfo* const result{info.get()};
Register(std::move(info), cpu_addr, size_bytes);
return result;
diff --git a/src/video_core/shader_cache.h b/src/video_core/shader_cache.h
index 997f2b0371..f4179dd435 100644
--- a/src/video_core/shader_cache.h
+++ b/src/video_core/shader_cache.h
@@ -7,6 +7,8 @@
#pragma once
#include <array>
+#include <atomic>
+#include <exception>
#include <memory>
#include <mutex>
#include <span>
@@ -143,6 +145,13 @@ private:
/// @brief Create a new shader entry and register it
const ShaderInfo* MakeShaderInfo(GenericEnvironment& env, VAddr cpu_addr);
+ /// Rethrows the error a native program at this address was rejected with, if any.
+ void RethrowIfRejected(VAddr addr);
+ /// Whether a native program at this address was rejected.
+ bool IsRejected(VAddr addr);
+ /// Remembers the exception being handled as this address's rejection.
+ void RememberRejection(VAddr addr);
+
Tegra::MaxwellDeviceMemoryManager& device_memory;
mutable std::mutex lookup_mutex;
@@ -152,6 +161,14 @@ private:
::Common::unordered_map<u64, std::vector<Entry*>> invalidation_cache;
std::vector<std::unique_ptr<ShaderInfo>> storage;
std::vector<Entry*> marked_for_removal;
+
+ /// Native programs the lowering rejected, by address, so each draw or dispatch rethrows the
+ /// first error instead of lowering again; dropped when guest memory there is written.
+ std::mutex rejected_mutex;
+ ::Common::unordered_map<u64, std::exception_ptr> rejected_native;
+ /// Bounds of every address ever rejected (never shrinks): a quick test for invalidations.
+ std::atomic<VAddr> rejected_lowest{~VAddr{}};
+ std::atomic<VAddr> rejected_highest{};
};
} // namespace VideoCommon
diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp
index 4d5db7e511..96306d9a27 100644
--- a/src/video_core/shader_environment.cpp
+++ b/src/video_core/shader_environment.cpp
@@ -5,7 +5,12 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
+#include <chrono>
+#include <cstdlib>
+#include <thread>
#include <filesystem>
+#include <cstring>
+#include <string>
#include <fstream>
#include <memory>
#include <optional>
@@ -21,6 +26,11 @@
#include "common/settings.h"
#include <ranges>
#include "shader_recompiler/environment.h"
+#include "shader_recompiler/exception.h"
+#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h"
+#include "shader_recompiler/frontend/ampere/program_override.h"
+#include "shader_recompiler/frontend/ampere/program_capture.h"
+#include "shader_recompiler/frontend/ampere/program_address_map.h"
#include "video_core/engines/kepler_compute.h"
#include "video_core/memory_manager.h"
#include "video_core/shader_environment.h"
@@ -149,11 +159,26 @@ u64 GenericEnvironment::ReadInstruction(u32 address) {
if (address >= cached_lowest && address < cached_highest) {
return code[(address - cached_lowest) / INST_SIZE];
}
+ if (native_source_size) {
+ throw Shader::RuntimeError("Lowered native instruction address out of range: {:#x}", address);
+ }
has_unbound_instructions = true;
return gpu_memory->Read<u64>(program_base + address);
}
+u32 GenericEnvironment::TranslateProgramAddress(u32 address) const {
+ if (const auto translated = Shader::Ampere::NativeProgramAddress(code, address)) return *translated;
+ if (native_source_size || Shader::Ampere::HasNativeAddressMap(code))
+ throw Shader::RuntimeError("Invalid native indirect branch address {:#x}", address);
+ return address;
+}
+
std::optional<u64> GenericEnvironment::Analyze() {
+ if (native_source_size) {
+ // Include the converted header and a version tag to isolate this experimental ABI.
+ return Common::CityHash64(reinterpret_cast<const char*>(code.data()),
+ code.size() * sizeof(u64)) ^ 0x4f554e4345000002ULL;
+ }
const std::optional<u64> size{TryFindSize()};
if (!size) {
return std::nullopt;
@@ -164,6 +189,12 @@ std::optional<u64> GenericEnvironment::Analyze() {
}
void GenericEnvironment::SetCachedSize(size_t size_bytes) {
+ if (native_source_size) {
+ if (size_bytes != CachedSizeBytes()) {
+ throw Shader::RuntimeError("Native shader changed while constructing pipeline");
+ }
+ return;
+ }
cached_lowest = start_address;
cached_highest = start_address + static_cast<u32>(size_bytes);
code.resize(CachedSizeWords());
@@ -291,6 +322,146 @@ Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr,
return entry;
}
+void GenericEnvironment::InitializeNativeCode(u32 source_header_size) {
+ using Shader::Ampere::Instruction;
+ const u32 native_header_size = source_header_size;
+ constexpr u32 maximum_instructions = Shader::Ampere::ProgramCaptureState::MaximumInstructions;
+ std::vector<Instruction> native;
+ Shader::Ampere::ProgramCaptureState capture;
+ for (u32 index = 0; index < maximum_instructions; ++index) {
+ const GPUVAddr address = program_base + native_header_size + index * sizeof(Instruction);
+ if (!gpu_memory->IsFullyMappedRange(address, sizeof(Instruction))) {
+ // The program may end before a branch target past its last EXIT (the capture can't
+ // tell); stop at the mapping's end and let the lowering judge what was captured.
+ if (!native.empty()) break;
+ throw Shader::RuntimeError("Unmapped native shader instruction at {:#x}", address);
+ }
+ auto& instruction = native.emplace_back();
+ gpu_memory->ReadBlock(address, &instruction, sizeof(instruction));
+ const auto status = capture.Observe(instruction, index);
+ if (status == Shader::Ampere::ProgramCaptureState::Result::InvalidTarget) {
+ if (const char* dir = std::getenv("EDEN_NX2_SHADER_DUMP")) {
+ const auto path = fmt::format("{}/rejected-{:x}.bin", dir, program_base + native_header_size);
+ if (std::FILE* file = std::fopen(path.c_str(), "wb")) {
+ std::fwrite(native.data(), sizeof(Instruction), std::min<size_t>(native.size(), 4096), file);
+ std::fclose(file);
+ }
+ }
+ throw Shader::RuntimeError("Native shader branch outside capture bounds at {:#x} +{:#x} [{:08x} {:08x} {:08x} {:08x}]",
+ program_base + native_header_size, index * 16,
+ instruction.words[0], instruction.words[1],
+ instruction.words[2], instruction.words[3]);
+ }
+ if (status == Shader::Ampere::ProgramCaptureState::Result::Complete) break;
+ }
+ auto lowered = Shader::Ampere::LowerToMaxwell(native);
+ // EDEN_NX2_REJECT=<hex address>,...: treat these programs as rejected (to find which newly
+ // lowered program changes a game's behaviour without rebuilding).
+ if (static const std::string reject_list = [] {
+ const char* text = std::getenv("EDEN_NX2_REJECT");
+ return text ? "," + std::string{text} + "," : std::string{};
+ }();
+ !reject_list.empty() &&
+ reject_list.find(fmt::format(",{:x},", program_base)) != std::string::npos) {
+ throw Shader::RuntimeError("Native shader at {:#x} rejected by EDEN_NX2_REJECT", program_base);
+ }
+ // EDEN_NX2_SHADER_DUMP=<dir>: keep every captured native program (named by its key) so the
+ // lowering can be tested offline against all of a game's shaders.
+ if (static const char* const dump_directory = std::getenv("EDEN_NX2_SHADER_DUMP");
+ dump_directory != nullptr) {
+ const auto folder = std::filesystem::path{dump_directory};
+ const auto path = folder / fmt::format("{:016x}-{}-{:x}.bin", Shader::Ampere::NativeProgramKey(native),
+ source_header_size == 0 ? "cs" : "gfx", program_base);
+ if (!std::filesystem::exists(path)) {
+ std::filesystem::create_directories(folder);
+ std::ofstream dump(path, std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native.data()),
+ native.size() * sizeof(Shader::Ampere::Instruction));
+ }
+ }
+ if (!lowered) {
+ if (const char* directory = std::getenv("EDEN_NATIVE_SHADER_OVERRIDE_DIR")) {
+ const auto key = fmt::format("{:016x}", Shader::Ampere::NativeProgramKey(native));
+ const auto folder = std::filesystem::path{directory};
+ std::filesystem::create_directories(folder);
+ const auto source_path = folder / (key + ".source.bin");
+ const auto override_path = folder / (key + ".override.bin");
+ {
+ std::ofstream dump(source_path, std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native.data()), native.size() * sizeof(Shader::Ampere::Instruction));
+ }
+ {
+ // Retain mapped bytes beyond the first EXIT for investigating
+ // native CALL targets in compiler library subroutines.
+ std::ofstream mapped(folder / (key + ".mapped.bin"), std::ios::binary);
+ std::array<u8, 0x1000> block{};
+ for (u32 offset = 0; offset < 0x10000; offset += 0x1000) {
+ if (!gpu_memory->IsFullyMappedRange(program_base + offset, block.size())) break;
+ gpu_memory->ReadBlock(program_base + offset, block.data(), block.size());
+ mapped.write(reinterpret_cast<const char*>(block.data()), block.size());
+ }
+ }
+ LOG_WARNING(Shader, "Native shader paused: key={}, offset={:#x}, reason={}, source={}",
+ key, lowered.error_offset, lowered.error, source_path.string());
+ const auto deadline = std::chrono::steady_clock::now() + std::chrono::minutes{30};
+ while (!gpu_memory->IsShuttingDown() && std::chrono::steady_clock::now() < deadline) {
+ if (std::filesystem::exists(override_path)) {
+ std::ifstream file(override_path, std::ios::binary | std::ios::ate);
+ const auto length = file.tellg();
+ if (length <= 0 || length > 16 * 1024 * 1024) {
+ throw Shader::RuntimeError("Invalid native shader override size");
+ }
+ std::vector<u8> bytes(static_cast<size_t>(length));
+ file.seekg(0);
+ file.read(reinterpret_cast<char*>(bytes.data()), bytes.size());
+ if (!file) throw Shader::RuntimeError("Native shader override read failed");
+ const u32 source_bytes = Shader::Ampere::ProgramOverrideSourceSize(bytes);
+ if (source_bytes > native.size() * sizeof(Shader::Ampere::Instruction)) {
+ if (!gpu_memory->IsFullyMappedRange(program_base + native_header_size, source_bytes))
+ throw Shader::RuntimeError("Unmapped extended native shader source");
+ native.resize(source_bytes / sizeof(Shader::Ampere::Instruction));
+ gpu_memory->ReadBlock(program_base + native_header_size, native.data(), source_bytes);
+ }
+ lowered = Shader::Ampere::DecodeProgramOverride(bytes, native);
+ if (!lowered) throw Shader::RuntimeError("Native shader override rejected: {}", lowered.error);
+ LOG_WARNING(Shader, "Native shader resumed: key={}, instructions={}", key, lowered.instructions.size());
+ break;
+ }
+ std::this_thread::sleep_for(std::chrono::milliseconds{100});
+ }
+ }
+ }
+ if (!lowered) {
+ const size_t failed = lowered.error_offset / sizeof(Shader::Ampere::Instruction);
+ const auto words = failed < native.size() ? native[failed].words : std::array<u32, 4>{};
+ throw Shader::RuntimeError(
+ "Native shader at {:#x}, instruction {:#x}: {} [op {:03x} {:08x} {:08x} {:08x} {:08x}]",
+ program_base, lowered.error_offset, lowered.error, words[0] & 0xfff, words[0],
+ words[1], words[2], words[3]);
+ }
+ if (lowered.required_texture_buffer != ~0U) {
+ // Native TEX carries its cbuf index explicitly; TEXS obtains it from the environment.
+ texture_bound = lowered.required_texture_buffer;
+ }
+ native_source_size = native_header_size + lowered.source_size;
+ code.resize(sizeof(sph) / sizeof(u64));
+ std::memcpy(code.data(), &sph, sizeof(sph));
+ for (const u64 instruction : lowered.instructions) {
+ if ((code.size() * sizeof(u64)) % 32 == 0) {
+ code.push_back(0); // Maxwell scheduling word, absent from native instructions.
+ }
+ code.push_back(instruction);
+ }
+ // Keep the original padding ABI, then retain the native address map as
+ // unreachable data. Existing disk-cache serialization carries it with code.
+ code.push_back(0);
+ Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets);
+ cached_lowest = 0;
+ cached_highest = static_cast<u32>((code.size() - 1) * sizeof(u64));
+ LOG_INFO(Shader, "Lowered native shader at {:#x}: {} source bytes, {} instructions",
+ program_base, native_source_size, lowered.instructions.size());
+}
+
GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_,
Tegra::MemoryManager& gpu_memory_,
Maxwell::ShaderType program, GPUVAddr program_base_,
@@ -298,6 +469,23 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_,
: GenericEnvironment{gpu_memory_, program_base_, start_address_}, maxwell3d{&maxwell3d_} {
gpu_memory->ReadBlock(program_base + start_address, &sph, sizeof(sph));
initial_offset = sizeof(sph);
+ texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot;
+ if (maxwell3d->GetNativeMacroEngine().native_turing) {
+ if (program != Maxwell::ShaderType::VertexB && program != Maxwell::ShaderType::Pixel) {
+ throw Shader::RuntimeError("Unsupported native graphics shader stage {}", static_cast<u32>(program));
+ }
+ if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) {
+ const u32 native_stage = program == Maxwell::ShaderType::Pixel ? 4 : 0;
+ const auto& table = maxwell3d->state.shader_stages[native_stage].const_buffers[1];
+ if (table.enabled && table.size && table.size <= Tegra::Engines::Maxwell3D::Regs::MaxConstBufferSize) {
+ std::vector<u8> bytes(table.size);
+ gpu_memory->ReadBlock(table.address, bytes.data(), bytes.size());
+ std::ofstream dump(fmt::format("native-graphics-cbuf-{}-1.bin", static_cast<u32>(program)), std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+ }
+ }
+ InitializeNativeCode();
+ }
gp_passthrough_mask = maxwell3d->regs.post_vtg_shader_attrib_skip_mask;
switch (program) {
case Maxwell::ShaderType::VertexA:
@@ -331,8 +519,7 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_,
const u64 local_size{sph.LocalMemorySize()};
ASSERT(local_size <= (std::numeric_limits<u32>::max)());
local_memory_size = static_cast<u32>(local_size) + sph.common3.shader_local_memory_crs_size;
- texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot;
- is_proprietary_driver = texture_bound == 2;
+ is_proprietary_driver = maxwell3d->GetNativeMacroEngine().native_turing || texture_bound == 2;
has_hle_engine_state =
maxwell3d->engine_state == Tegra::Engines::Maxwell3D::EngineHint::OnHLEMacro;
}
@@ -417,6 +604,12 @@ ComputeEnvironment::ComputeEnvironment(Tegra::Engines::KeplerCompute& kepler_com
is_proprietary_driver = texture_bound == 2;
shared_memory_size = qmd.shared_alloc;
workgroup_size = {qmd.block_dim_x, qmd.block_dim_y, qmd.block_dim_z};
+ if (kepler_compute->IsNativeMode()) {
+ initial_offset = sizeof(sph);
+ is_proprietary_driver = true;
+ InitializeNativeCode(0);
+ start_address = sizeof(sph);
+ }
}
u32 ComputeEnvironment::ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) {
@@ -511,7 +704,7 @@ void FileEnvironment::Deserialize(std::ifstream& file) {
if (stage == Shader::Stage::Compute) {
file.read(reinterpret_cast<char*>(&workgroup_size), sizeof(workgroup_size))
.read(reinterpret_cast<char*>(&shared_memory_size), sizeof(shared_memory_size));
- initial_offset = 0;
+ initial_offset = Shader::Ampere::HasNativeAddressMap(code) ? sizeof(sph) : 0;
} else {
file.read(reinterpret_cast<char*>(&sph), sizeof(sph));
initial_offset = sizeof(sph);
@@ -526,6 +719,13 @@ void FileEnvironment::Dump(u64 pipeline_hash, u64 shader_hash) {
DumpImpl(pipeline_hash, shader_hash, code, read_highest, read_lowest, initial_offset, stage);
}
+u32 FileEnvironment::TranslateProgramAddress(u32 address) const {
+ if (const auto translated = Shader::Ampere::NativeProgramAddress(code, address)) return *translated;
+ if (Shader::Ampere::HasNativeAddressMap(code))
+ throw Shader::RuntimeError("Invalid cached native indirect branch address {:#x}", address);
+ return address;
+}
+
u64 FileEnvironment::ReadInstruction(u32 address) {
if (address < read_lowest || address > read_highest) {
throw Shader::LogicError("Out of bounds address {}", address);
diff --git a/src/video_core/shader_environment.h b/src/video_core/shader_environment.h
index 924e61ef46..e49f01d95f 100644
--- a/src/video_core/shader_environment.h
+++ b/src/video_core/shader_environment.h
@@ -47,6 +47,8 @@ public:
[[nodiscard]] u64 ReadInstruction(u32 address) final;
+ [[nodiscard]] u32 TranslateProgramAddress(u32 address) const override;
+
[[nodiscard]] std::optional<u64> Analyze();
void SetCachedSize(size_t size_bytes);
@@ -57,6 +59,10 @@ public:
[[nodiscard]] size_t ReadSizeBytes() const noexcept;
+ [[nodiscard]] size_t SourceSizeBytes() const noexcept {
+ return native_source_size;
+ }
+
[[nodiscard]] bool CanBeSerialized() const noexcept;
[[nodiscard]] u64 CalculateHash() const;
@@ -72,6 +78,9 @@ public:
protected:
std::optional<u64> TryFindSize();
+ void InitializeNativeCode(u32 source_header_size = 128);
+ size_t native_source_size{};
+
Tegra::Texture::TICEntry ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit,
bool via_header_index, u32 raw);
@@ -172,6 +181,8 @@ public:
[[nodiscard]] u64 ReadInstruction(u32 address) override;
+ [[nodiscard]] u32 TranslateProgramAddress(u32 address) const override;
+
[[nodiscard]] u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) override;
[[nodiscard]] Shader::TextureType ReadTextureType(u32 handle) override;
diff --git a/src/video_core/texture_cache/accelerated_swizzle.cpp b/src/video_core/texture_cache/accelerated_swizzle.cpp
index 4c3f724d79..8ae0ed218d 100644
--- a/src/video_core/texture_cache/accelerated_swizzle.cpp
+++ b/src/video_core/texture_cache/accelerated_swizzle.cpp
@@ -37,6 +37,7 @@ BlockLinearSwizzle2DParams MakeBlockLinearSwizzle2DParams(const SwizzleParameter
.x_shift = GOB_SIZE_SHIFT + block.height + block.depth,
.block_height = block.height,
.block_height_mask = (1U << block.height) - 1,
+ .native_tiling = info.native_tiling,
};
}
@@ -63,6 +64,7 @@ BlockLinearSwizzle3DParams MakeBlockLinearSwizzle3DParams(const SwizzleParameter
.block_height_mask = (1U << block.height) - 1,
.block_depth = block.depth,
.block_depth_mask = (1U << block.depth) - 1,
+ .native_tiling = info.native_tiling,
};
}
diff --git a/src/video_core/texture_cache/accelerated_swizzle.h b/src/video_core/texture_cache/accelerated_swizzle.h
index d4250da689..8296950c38 100644
--- a/src/video_core/texture_cache/accelerated_swizzle.h
+++ b/src/video_core/texture_cache/accelerated_swizzle.h
@@ -20,6 +20,7 @@ struct BlockLinearSwizzle2DParams {
u32 x_shift;
u32 block_height;
u32 block_height_mask;
+ u32 native_tiling;
};
struct BlockLinearSwizzle3DParams {
@@ -33,6 +34,7 @@ struct BlockLinearSwizzle3DParams {
u32 block_height_mask;
u32 block_depth;
u32 block_depth_mask;
+ u32 native_tiling;
};
[[nodiscard]] BlockLinearSwizzle2DParams MakeBlockLinearSwizzle2DParams(
diff --git a/src/video_core/texture_cache/format_lookup_table.cpp b/src/video_core/texture_cache/format_lookup_table.cpp
index d6e019cddf..20bb96aae3 100644
--- a/src/video_core/texture_cache/format_lookup_table.cpp
+++ b/src/video_core/texture_cache/format_lookup_table.cpp
@@ -157,6 +157,7 @@ PixelFormat PixelFormatFromTextureInfo(TextureFormat format, ComponentType red,
return PixelFormat::S8_UINT_D24_UNORM;
case Hash(TextureFormat::G24R8, UINT, UNORM, UNORM, UNORM, LINEAR):
return PixelFormat::S8_UINT_D24_UNORM;
+ case Hash(TextureFormat::S8Z24, UNORM, UINT, UNORM, UNORM, LINEAR):
case Hash(TextureFormat::S8Z24, UNORM, UINT, UINT, UINT, LINEAR):
return PixelFormat::D24_UNORM_S8_UINT;
case Hash(TextureFormat::Z32_X24S8, FLOAT, UINT, UNORM, UNORM, LINEAR):
diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index 79fc8505d6..47bab9617a 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -45,6 +45,11 @@ enum class ImageFlagBits : u32 {
AsynchronousDecode = 1 << 16,
IsDecoding = 1 << 17, ///< Is currently being decoded asynchronously.
+ Unbacked = 1 << 18, ///< Host image has no memory (too big to back); never upload/download.
+ /// The host image keeps only mip levels >= host_level_offset (its decoded copy would not fit
+ /// in VRAM); uploads skip the rest, downloads are never done.
+ TrimmedLevels = 1 << 19,
+ ZeroFilled = 1 << 20, ///< Cleared to zero blocks before its first asynchronous decode.
};
DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits)
@@ -93,6 +98,7 @@ struct ImageBase {
u32 unswizzled_size_bytes = 0;
u32 converted_size_bytes = 0;
u32 scale_rating = 0;
+ u32 host_level_offset = 0; ///< Guest mip level held by host mip 0 (see TrimmedLevels).
u64 scale_tick = 0;
bool has_scaled = false;
diff --git a/src/video_core/texture_cache/image_info.cpp b/src/video_core/texture_cache/image_info.cpp
index 8b59eb6841..7e86bf00cf 100644
--- a/src/video_core/texture_cache/image_info.cpp
+++ b/src/video_core/texture_cache/image_info.cpp
@@ -4,9 +4,13 @@
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <algorithm>
+#include <array>
+#include <vector>
#include <fmt/ranges.h>
#include "common/assert.h"
+#include "common/logging.h"
#include "common/settings.h"
#include "video_core/surface.h"
#include "video_core/texture_cache/format_lookup_table.h"
@@ -109,14 +113,23 @@ ImageInfo::ImageInfo(const TICEntry& config) noexcept {
size.height = config.Height();
resources.layers = config.BaseLayer() + config.Depth() * 6;
break;
- case TextureType::Texture3D:
+ case TextureType::Texture3D: {
ASSERT(config.BaseLayer() == 0);
type = ImageType::e3D;
size.width = config.Width();
size.height = config.Height();
size.depth = config.Depth();
resources.layers = 1;
+ // A few distinct 3D descriptors, raw (to check Switch 2 volume fields).
+ static std::vector<std::array<u64, 4>> logged;
+ if (logged.size() < 16 && std::find(logged.begin(), logged.end(), config.raw) == logged.end()) {
+ logged.push_back(config.raw);
+ LOG_INFO(HW_GPU, "3D TIC {:016x} {:016x} {:016x} {:016x}: {}x{}x{} block_depth {} levels {}",
+ config.raw[0], config.raw[1], config.raw[2], config.raw[3], size.width, size.height,
+ size.depth, config.block_depth.Value(), config.max_mip_level.Value() + 1);
+ }
break;
+ }
case TextureType::Texture1DBuffer:
type = ImageType::Buffer;
size.width = config.Width();
diff --git a/src/video_core/texture_cache/image_info.h b/src/video_core/texture_cache/image_info.h
index eb490a6424..79237aa482 100644
--- a/src/video_core/texture_cache/image_info.h
+++ b/src/video_core/texture_cache/image_info.h
@@ -14,6 +14,17 @@ namespace VideoCommon {
using Tegra::Texture::TICEntry;
using VideoCore::Surface::PixelFormat;
+// Native descriptors address the selected view directly. The legacy NVN
+// layer-base bookkeeping is not an additional array prefix for this ABI.
+inline TICEntry TextureViewHeader(TICEntry entry, bool native) {
+ if (native) {
+ entry.layer_base_0_2.Assign(0);
+ entry.layer_base_3_7.Assign(0);
+ entry.layer_base_8_10.Assign(0);
+ }
+ return entry;
+}
+
struct ImageInfo {
ImageInfo() = default;
explicit ImageInfo(const TICEntry& config) noexcept;
@@ -37,11 +48,14 @@ struct ImageInfo {
u32 maybe_unaligned_layer_stride = 0;
u32 num_samples = 1;
u32 tile_width_spacing = 0;
+ bool native_tiling = false;
bool rescaleable = false;
bool downscaleable = false;
bool forced_flushed = false;
bool dma_downloaded = false;
bool is_sparse = false;
+ // ASTC data whose blocks carry no alpha: re-compressed to BC1 instead of BC3 (half the size).
+ bool astc_opaque = false;
};
} // namespace VideoCommon
diff --git a/src/video_core/texture_cache/pending_download.h b/src/video_core/texture_cache/pending_download.h
new file mode 100644
index 0000000000..52a48e82d3
--- /dev/null
+++ b/src/video_core/texture_cache/pending_download.h
@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <algorithm>
+#include <memory>
+#include <unordered_set>
+#include <cstddef>
+#include <utility>
+#include "common/slot_vector.h"
+
+namespace VideoCommon {
+
+template <class Image>
+struct PendingImageDownload {
+ bool is_swizzle;
+ size_t async_buffer_id;
+ Common::SlotId object_id;
+ std::shared_ptr<Image> retired_image{};
+
+ Image& GetImage(Common::SlotVector<Image>& images) const {
+ return retired_image ? *retired_image : images[object_id];
+ }
+};
+
+// Whole-image copies are recorded together at commit time. Repeated requests
+// for the same live image therefore observe the same contents. Keep the first
+// occurrence so reverse CPU writeback order remains unchanged for aliases.
+template <class Image, class Downloads>
+size_t DeduplicatePendingImageDownloads(Downloads& downloads, Common::SlotVector<Image>& images) {
+ std::unordered_set<const Image*> seen;
+ const size_t previous_size = downloads.size();
+ const auto end = std::remove_if(downloads.begin(), downloads.end(), [&](const auto& download) {
+ return download.is_swizzle && !seen.insert(&download.GetImage(images)).second;
+ });
+ downloads.erase(end, downloads.end());
+ return previous_size - downloads.size();
+}
+
+// Cache slots may be recycled before queued GPU downloads are submitted or
+// consumed. Keep the actual image, including its metadata, until those finish.
+template <class Image, class Downloads, class Batches>
+bool RetainPendingImageDownloads(Common::SlotId id, Image& image, Downloads& pending,
+ Batches& submitted) {
+ std::shared_ptr<Image> retained;
+ const auto retain = [&](auto& downloads) {
+ for (auto& download : downloads) {
+ if (!download.is_swizzle || download.object_id != id || download.retired_image) continue;
+ if (!retained) retained = std::make_shared<Image>(std::move(image));
+ download.retired_image = retained;
+ }
+ };
+ retain(pending);
+ for (auto& downloads : submitted) retain(downloads);
+ return retained != nullptr;
+}
+
+} // namespace VideoCommon
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index e92e54f8ce..fb8adef161 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -6,7 +6,11 @@
#pragma once
+#include <algorithm>
+#include <cstdlib>
#include <limits>
+#include <map>
+#include <vector>
#include <optional>
#include <bit>
#include "common/container/unordered_map.h"
@@ -24,6 +28,7 @@
#include "video_core/texture_cache/samples_helper.h"
#include "video_core/texture_cache/texture_cache_base.h"
#include "video_core/texture_cache/util.h"
+#include "video_core/nx2_profile.h"
#include "video_core/textures/decoders.h"
namespace VideoCommon {
@@ -76,6 +81,7 @@ TextureCache<P>::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag
critical_memory = DEFAULT_CRITICAL_MEMORY + 1_GiB;
minimum_memory = 0;
}
+ ApplyThresholdOverrides();
const bool gpu_unswizzle_enabled = Settings::values.gpu_unswizzle_enabled.GetValue();
@@ -148,6 +154,7 @@ void TextureCache<P>::RunGarbageCollector() {
if (True(image.flags & ImageFlagBits::Tracked)) {
UntrackImage(image, image_id);
}
+ ++(True(image.flags & ImageFlagBits::CostlyLoad) ? gc_costly_deletes : gc_deletes);
UnregisterImage(image_id);
DeleteImage(image_id, image.scale_tick > frame_tick + 5);
if (aggressive_mode && total_used_memory < critical_memory) {
@@ -168,11 +175,118 @@ void TextureCache<P>::RunGarbageCollector() {
}
template <class P>
+void TextureCache<P>::ApplyThresholdOverrides(bool log) {
+ // EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB override the collector's thresholds.
+ for (const auto& [name, target] : {std::pair{"EDEN_NX2_TC_EXPECTED_MIB", &expected_memory},
+ std::pair{"EDEN_NX2_TC_CRITICAL_MIB", &critical_memory}}) {
+ if (const char* value = std::getenv(name)) {
+ *target = std::strtoull(value, nullptr, 10) << 20;
+ }
+ }
+ if (log) {
+ LOG_INFO(HW_GPU, "Texture cache thresholds: expected {} MiB, critical {} MiB, minimum {} MiB",
+ expected_memory >> 20, critical_memory >> 20, minimum_memory >> 20);
+ }
+}
+
+template <class P>
void TextureCache<P>::TickFrame() {
+ if constexpr (HAS_DEVICE_MEMORY_INFO) {
+ // Switch 2 titles (native tiling) stream far more texture data than Eden's thresholds
+ // were tuned for: title 1's attract loop alone keeps ~5.3 GB resident, so the collector
+ // sat in aggressive mode evicting decoded ASTC (and decoding it again). Total use is
+ // measured from the driver, so collect only near the device budget instead.
+ if (!native_thresholds_applied && gpu_memory != nullptr && gpu_memory->UsesNativeTiling()) {
+ native_thresholds_applied = true;
+ const u64 device_local = runtime.GetDeviceLocalMemory();
+ critical_memory = (std::max)(critical_memory, device_local - 256_MiB);
+ expected_memory = (std::max)(expected_memory, device_local - 1280_MiB);
+ ApplyThresholdOverrides();
+ native_critical_memory = critical_memory;
+ native_expected_memory = expected_memory;
+ }
+ // Follow the driver's budget (re-read now and then): it shrinks when other programs use
+ // VRAM (another game: 7.0 -> 6.4 GB), and collecting only past it means the driver pages
+ // textures out. It is also the room to use: Eden's device memory figure is capped near
+ // 7 GB, which on a 12 GB card (11.2 GB budget) kept title 1 evicting textures with 3 GB
+ // unused. EDEN_NX2_TC_*_MIB overrides still win.
+ if constexpr (requires { runtime.GetDeviceMemoryBudget(); }) {
+ if (native_thresholds_applied && budget_check_frames++ % 60 == 0) {
+ const u64 budget = runtime.GetDeviceMemoryBudget();
+ if (budget > 2_GiB) {
+ const u64 old_critical = critical_memory;
+ critical_memory = budget - 768_MiB;
+ expected_memory = budget - 1536_MiB;
+ // Log only real changes (the budget moves by a few MiB all the time).
+ const u64 change = critical_memory > old_critical
+ ? critical_memory - old_critical
+ : old_critical - critical_memory;
+ ApplyThresholdOverrides(change >= 256_MiB);
+ }
+ }
+ }
+ }
// If we can obtain the memory info, use it instead of the estimate.
if (runtime.CanReportMemoryUsage()) {
total_used_memory = runtime.GetDeviceMemoryUsage();
}
+ // Every 600 frames: device memory in use against the collector's thresholds, and what the
+ // images themselves account for (VRAM pressure makes the collector thrash).
+ if (static u64 frames{}; ++frames % 600 == 0) {
+ u64 image_bytes{};
+ size_t images{};
+ std::vector<std::pair<u64, const Image*>> largest;
+ std::map<u32, std::pair<u64, u32>> by_format;
+ for (const auto [image_id, image] : slot_images) {
+ const u64 bytes = GetScaledImageSizeBytes(*image);
+ image_bytes += bytes;
+ ++images;
+ largest.emplace_back(bytes, image);
+ auto& format = by_format[static_cast<u32>(image->info.format)];
+ format.first += bytes;
+ ++format.second;
+ }
+ LOG_INFO(HW_GPU, "Texture cache: device memory {} MiB (expected {} MiB, critical {} MiB), {} images {} MiB",
+ total_used_memory >> 20, expected_memory >> 20, critical_memory >> 20, images,
+ image_bytes >> 20);
+ std::partial_sort(largest.begin(), largest.begin() + std::min<size_t>(12, largest.size()),
+ largest.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
+ for (size_t i = 0; i < std::min<size_t>(12, largest.size()); ++i) {
+ const Image& image = *largest[i].second;
+ LOG_INFO(HW_GPU, " {} MiB: {:#x} {}x{}x{} layers {} levels {} format {} type {} flags {:#x}",
+ largest[i].first >> 20, image.gpu_addr, image.info.size.width, image.info.size.height,
+ image.info.size.depth, image.info.resources.layers, image.info.resources.levels,
+ static_cast<u32>(image.info.format), static_cast<u32>(image.info.type),
+ static_cast<u32>(image.flags));
+ }
+ std::vector<std::pair<u64, u32>> formats;
+ for (const auto& [format, total] : by_format) formats.emplace_back(total.first, format);
+ std::sort(formats.rbegin(), formats.rend());
+ for (size_t i = 0; i < std::min<size_t>(8, formats.size()); ++i) {
+ LOG_INFO(HW_GPU, " format {}: {} MiB in {} images", formats[i].second, formats[i].first >> 20,
+ by_format[formats[i].second].second);
+ }
+ if constexpr (requires { runtime.LogMemoryStatistics(); }) {
+ runtime.LogMemoryStatistics();
+ }
+ LOG_INFO(HW_GPU, "Texture cache last 600 frames: created {} images, deleted {} (collector {}, {} costly), {} async decodes, ASTC {} opaque / {} alpha",
+ images_created, images_deleted, gc_deletes + gc_costly_deletes, gc_costly_deletes,
+ async_decodes_queued, astc_opaque_images, astc_alpha_images);
+ astc_opaque_images = astc_alpha_images = 0;
+ std::vector<std::pair<u32, u64>> shapes;
+ for (const auto& [key, value] : created_shapes) shapes.emplace_back(value.first, key);
+ std::sort(shapes.rbegin(), shapes.rend());
+ for (size_t i = 0; i < std::min<size_t>(10, shapes.size()); ++i) {
+ const u64 key = shapes[i].second;
+ LOG_INFO(HW_GPU, " created {}x: format {} type {} levels {} layers {} {}x{} (last at {:#x})",
+ shapes[i].first, key >> 56, (key >> 52) & 0xF, (key >> 48) & 0xF,
+ (key >> 36) & 0xFFF, (key >> 18) & 0x3FFFF, key & 0x3FFFF,
+ created_shapes[key].second);
+ }
+ created_shapes.clear();
+ gc_deletes = gc_costly_deletes = async_decodes_queued = 0;
+ images_created = images_deleted = 0;
+ }
if (total_used_memory > minimum_memory) {
RunGarbageCollector();
}
@@ -597,7 +711,7 @@ void TextureCache<P>::WriteMemory(DAddr cpu_addr, size_t size) {
template <class P>
bool TextureCache<P>::IsDownloadable(const ImageBase& image) const noexcept {
- if (!image.IsSafeGpuCopy()) {
+ if (!image.IsSafeGpuCopy() || True(image.flags & (ImageFlagBits::Unbacked | ImageFlagBits::TrimmedLevels))) {
return false;
}
if (image.info.num_samples == 1) {
@@ -714,6 +828,14 @@ bool TextureCache<P>::BlitImage(const Tegra::Engines::Fermi2D::Surface& dst,
Image& dst_image = slot_images[dst_id];
Image& src_image = slot_images[src_id];
+ if (True(dst_image.flags & ImageFlagBits::Unbacked)) {
+ static u32 logged{};
+ if (logged++ < 64) {
+ LOG_INFO(HW_GPU, "2D blit into unbacked image {:#x}: dst {:#x} {}x{} from {:#x} {}x{}",
+ dst_image.gpu_addr, dst.Address(), dst.width, dst.height, src.Address(),
+ src.width, src.height);
+ }
+ }
bool is_src_rescaled = True(src_image.flags & ImageFlagBits::Rescaled);
bool is_dst_rescaled = True(dst_image.flags & ImageFlagBits::Rescaled);
@@ -803,6 +925,17 @@ std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImage
if (it == page_table.end()) {
return {};
}
+ const auto view_format = [&]() {
+ switch (config.pixel_format) {
+ case Service::android::PixelFormat::Rgb565:
+ return PixelFormat::R5G6B5_UNORM;
+ case Service::android::PixelFormat::Bgra8888:
+ return PixelFormat::B8G8R8A8_UNORM;
+ default:
+ return PixelFormat::A8B8G8R8_UNORM;
+ }
+ }();
+
const auto& image_map_ids = it->second;
boost::container::small_vector<ImageId, 4> valid_image_ids;
for (const ImageMapId map_id : image_map_ids) {
@@ -814,20 +947,17 @@ std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImage
if (image.image_view_ids.empty()) {
continue;
}
+ if (!VideoCore::Surface::IsViewCompatible(image.info.format, view_format, false, true)) {
+ static u32 rejected_alias_count{};
+ if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && rejected_alias_count++ < 8) {
+ LOG_WARNING(HW_GPU, "Ignoring incompatible framebuffer alias: address={:#x}, image={}, view={}",
+ cpu_addr, image.info.format, view_format);
+ }
+ continue;
+ }
valid_image_ids.push_back(map.image_id);
}
- const auto view_format = [&]() {
- switch (config.pixel_format) {
- case Service::android::PixelFormat::Rgb565:
- return PixelFormat::R5G6B5_UNORM;
- case Service::android::PixelFormat::Bgra8888:
- return PixelFormat::B8G8R8A8_UNORM;
- default:
- return PixelFormat::A8B8G8R8_UNORM;
- }
- }();
-
const auto GetImageViewForFramebuffer = [&](ImageId image_id) {
ImageViewInfo info{ImageViewType::e2D, view_format};
if (config.blending == Tegra::BlendMode::Opaque) {
@@ -860,6 +990,11 @@ bool TextureCache<P>::HasUncommittedFlushes() const noexcept {
}
template <class P>
+bool TextureCache<P>::HasAsyncFlushWork() const noexcept {
+ return !uncommitted_downloads.empty() || !uncommitted_async_buffers.empty();
+}
+
+template <class P>
bool TextureCache<P>::ShouldWaitAsyncFlushes() const noexcept {
return !committed_downloads.empty() && !committed_downloads.front().empty();
}
@@ -869,6 +1004,7 @@ void TextureCache<P>::CommitAsyncFlushes() {
// This is intentionally passing the value by copy
if constexpr (IMPLEMENTS_ASYNC_DOWNLOADS) {
auto& download_ids = uncommitted_downloads;
+ const size_t redundant_images = DeduplicatePendingImageDownloads(download_ids, slot_images);
if (download_ids.empty()) {
committed_downloads.emplace_back(std::move(uncommitted_downloads));
uncommitted_downloads.clear();
@@ -882,17 +1018,32 @@ void TextureCache<P>::CommitAsyncFlushes() {
for (PendingDownload& download_info : download_ids) {
if (download_info.is_swizzle) {
total_size_bytes +=
- Common::AlignUp(slot_images[download_info.object_id].unswizzled_size_bytes, 64);
+ Common::AlignUp(download_info.GetImage(slot_images).unswizzled_size_bytes, 64);
any_none_dma = true;
download_info.async_buffer_id = last_async_buffer_id;
}
}
if (any_none_dma) {
+ if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && total_size_bytes >= 64_MiB) {
+ LOG_WARNING(HW_GPU, "Large image readback batch: bytes={}, entries={}, duplicates_removed={}, submitted_batches={}, retained_buffers={}",
+ total_size_bytes, download_ids.size(), redundant_images,
+ committed_downloads.size(), async_buffers_death_ring.size());
+ size_t traced_images = 0;
+ for (const auto& download : download_ids) {
+ if (!download.is_swizzle || traced_images++ >= 8) continue;
+ const auto& image = download.GetImage(slot_images);
+ LOG_WARNING(HW_GPU, "Readback image: gpu={:#x}, cpu={:#x}, format={}, size={}x{}x{}, levels={}, layers={}, layer_stride={}, guest_bytes={}, download_bytes={}",
+ image.gpu_addr, image.cpu_addr, image.info.format,
+ image.info.size.width, image.info.size.height, image.info.size.depth,
+ image.info.resources.levels, image.info.resources.layers,
+ image.info.layer_stride, image.guest_size_bytes, image.unswizzled_size_bytes);
+ }
+ }
auto download_map = runtime.DownloadStagingBuffer(total_size_bytes, true);
for (const PendingDownload& download_info : download_ids) {
if (download_info.is_swizzle) {
- Image& image = slot_images[download_info.object_id];
+ Image& image = download_info.GetImage(slot_images);
const auto copies = FullDownloadCopies(image.info);
image.DownloadMemory(download_map, copies);
download_map.offset += Common::AlignUp(image.unswizzled_size_bytes, 64);
@@ -925,7 +1076,7 @@ void TextureCache<P>::PopAsyncFlushes() {
auto& download_info = download_ids[i - 1];
auto& download_buffer = download_map[download_info.async_buffer_id];
if (download_info.is_swizzle) {
- const ImageBase& image = slot_images[download_info.object_id];
+ const ImageBase& image = download_info.GetImage(slot_images);
const auto copies = FullDownloadCopies(image.info);
download_buffer.offset -= Common::AlignUp(image.unswizzled_size_bytes, 64);
std::span<u8> download_span =
@@ -954,7 +1105,7 @@ void TextureCache<P>::PopAsyncFlushes() {
size_t total_size_bytes = 0;
for (const PendingDownload& download_info : download_ids) {
if (download_info.is_swizzle) {
- total_size_bytes += slot_images[download_info.object_id].unswizzled_size_bytes;
+ total_size_bytes += download_info.GetImage(slot_images).unswizzled_size_bytes;
}
}
auto download_map = runtime.DownloadStagingBuffer(total_size_bytes);
@@ -963,7 +1114,7 @@ void TextureCache<P>::PopAsyncFlushes() {
if (!download_info.is_swizzle) {
continue;
}
- Image& image = slot_images[download_info.object_id];
+ Image& image = download_info.GetImage(slot_images);
const auto copies = FullDownloadCopies(image.info);
image.DownloadMemory(download_map, copies);
download_map.offset += image.unswizzled_size_bytes;
@@ -976,7 +1127,7 @@ void TextureCache<P>::PopAsyncFlushes() {
if (!download_info.is_swizzle) {
continue;
}
- const ImageBase& image = slot_images[download_info.object_id];
+ const ImageBase& image = download_info.GetImage(slot_images);
const auto copies = FullDownloadCopies(image.info);
SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, download_span, swizzle_data_buffer);
download_map.offset += image.unswizzled_size_bytes;
@@ -1117,11 +1268,18 @@ void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) {
// Only upload modified images
return;
}
+ VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::TextureUpload};
image.flags &= ~ImageFlagBits::CpuModified;
TrackImage(image, image_id);
+ if (True(image.flags & ImageFlagBits::Unbacked)) {
+ // Nothing to upload into (and staging it could take tens of GB).
+ runtime.TransitionImageLayout(image);
+ return;
+ }
+
if (image.info.num_samples > 1 && !runtime.CanUploadMSAA()) {
LOG_WARNING(HW_GPU, "MSAA image uploads are not implemented");
runtime.TransitionImageLayout(image);
@@ -1156,6 +1314,24 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging)
const std::span<u8> mapped_span = staging.mapped_span;
const GPUVAddr gpu_addr = image.gpu_addr;
+ if (True(image.flags & ImageFlagBits::AcceleratedUpload) && image.host_level_offset != 0) {
+ // Only levels >= host_level_offset exist on the host: read each layer's tail of the mip
+ // chain into a compact buffer (layer stride = that tail's size) and decode those levels.
+ auto uploads = FullUploadSwizzles(image.info);
+ const size_t level_offset = uploads[image.host_level_offset].buffer_offset;
+ const size_t tail = image.info.layer_stride - level_offset;
+ for (s32 layer = 0; layer < image.info.resources.layers; ++layer) {
+ gpu_memory->ReadBlock(gpu_addr + layer * image.info.layer_stride + level_offset,
+ mapped_span.data() + layer * tail, tail,
+ VideoCommon::CacheType::NoTextureCache);
+ }
+ uploads.erase(uploads.begin(), uploads.begin() + image.host_level_offset);
+ for (auto& upload : uploads) {
+ upload.buffer_offset -= level_offset;
+ }
+ runtime.AccelerateImageUpload(image, staging, uploads, 0, 0);
+ return;
+ }
if (True(image.flags & ImageFlagBits::AcceleratedUpload)) {
gpu_memory->ReadBlock(gpu_addr, mapped_span.data(), mapped_span.size_bytes(),
VideoCommon::CacheType::NoTextureCache);
@@ -1178,7 +1354,18 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging)
}
template <class P>
-ImageViewId TextureCache<P>::CreateImageView(const TICEntry& config) {
+ImageViewId TextureCache<P>::CreateImageView(const TICEntry& raw_config) {
+ const bool native = gpu_memory->UsesNativeTiling();
+ if (native && raw_config.BaseLayer() && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) {
+ static u32 texture_view_traces{};
+ if (texture_view_traces++ < 8) {
+ LOG_WARNING(HW_GPU, "Native texture view header: address={:#x}, dimensions={}x{}x{}, type={}, legacy_layer={}, raw={:016x}/{:016x}/{:016x}/{:016x}",
+ raw_config.Address(), raw_config.Width(), raw_config.Height(), raw_config.Depth(),
+ static_cast<u32>(raw_config.texture_type.Value()), raw_config.BaseLayer(),
+ raw_config.raw[0], raw_config.raw[1], raw_config.raw[2], raw_config.raw[3]);
+ }
+ }
+ const TICEntry config = TextureViewHeader(raw_config, native);
const ImageInfo info(config);
if (info.type == ImageType::Buffer) {
const ImageViewInfo view_info(config, 0);
@@ -1213,12 +1400,9 @@ ImageId TextureCache<P>::FindOrInsertImage(const ImageInfo& info, GPUVAddr gpu_a
template <class P>
ImageId TextureCache<P>::FindImage(const ImageInfo& info, GPUVAddr gpu_addr,
RelaxedOptions options) {
- std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr);
+ const std::optional<DAddr> cpu_addr = ImageCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info));
if (!cpu_addr) {
- cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info));
- if (!cpu_addr) {
- return ImageId{};
- }
+ return ImageId{};
}
const bool broken_views =
runtime.HasBrokenTextureViewFormats() || True(options & RelaxedOptions::ForceBrokenViews);
@@ -1344,9 +1528,10 @@ u64 TextureCache<P>::GetScaledImageSizeBytes(const ImageBase& image) {
template <class P>
void TextureCache<P>::QueueAsyncDecode(Image& image, ImageId image_id) {
UNIMPLEMENTED_IF(False(image.flags & ImageFlagBits::Converted));
- LOG_INFO(HW_GPU, "Queuing async texture decode");
+ LOG_DEBUG(HW_GPU, "Queuing async texture decode {:#x}", image.gpu_addr);
image.flags |= ImageFlagBits::IsDecoding;
+ ++async_decodes_queued;
auto decode = std::make_unique<AsyncDecodeContext>();
auto* decode_ptr = decode.get();
decode->image_id = image_id;
@@ -1356,6 +1541,18 @@ void TextureCache<P>::QueueAsyncDecode(Image& image, ImageId image_id) {
Tegra::Memory::GpuGuestMemory<u8, Tegra::Memory::GuestMemoryFlags::UnsafeRead> swizzle_data(*gpu_memory, image.gpu_addr, image.guest_size_bytes, &swizzle_data_buffer);
auto copies = UnswizzleImage(*gpu_memory, image.gpu_addr, image.info, swizzle_data, local_unswizzle_data_buffer);
const size_t out_size = MapSizeBytes(image);
+ if (False(image.flags & ImageFlagBits::ZeroFilled)) {
+ // Until the decode lands (a second or more while title 1 streams ~40 textures a second)
+ // a new image showed whatever its memory last held, often other textures. Zero blocks
+ // read as opaque black (BC1) or transparent (BC3) instead.
+ image.flags |= ImageFlagBits::ZeroFilled;
+ auto zero_copies = copies;
+ ConvertedLayout(image.info, zero_copies);
+ auto staging = runtime.UploadStagingBuffer(out_size);
+ std::memset(staging.mapped_span.data(), 0, out_size);
+ image.UploadMemory(staging, zero_copies);
+ runtime.InsertUploadMemoryBarrier();
+ }
auto func = [out_size, copies, info = image.info,
input = std::move(local_unswizzle_data_buffer),
@@ -1392,12 +1589,26 @@ void TextureCache<P>::TickAsyncDecode() {
auto i = async_decodes.begin();
while (i != async_decodes.end()) {
auto* async_decode = i->get();
- std::unique_lock lock{async_decode->mutex};
- if (!async_decode->complete) {
- ++i;
+ {
+ // Release the lock before the context is erased below: unlocking it afterwards wrote
+ // into freed memory (heap corruption, crashes minutes into title 1).
+ std::unique_lock lock{async_decode->mutex};
+ if (!async_decode->complete) {
+ ++i;
+ continue;
+ }
+ }
+ if (async_decode->cancelled) {
+ i = async_decodes.erase(i);
continue;
}
Image& image = slot_images[async_decode->image_id];
+ if (async_decode->decoded_data.size() > MapSizeBytes(image)) {
+ LOG_ERROR(HW_GPU, "Async decode of {} bytes for a {}-byte image at {:#x}, dropped",
+ async_decode->decoded_data.size(), MapSizeBytes(image), image.gpu_addr);
+ i = async_decodes.erase(i);
+ continue;
+ }
auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image));
std::memcpy(staging.mapped_span.data(), async_decode->decoded_data.data(),
async_decode->decoded_data.size());
@@ -1513,20 +1724,42 @@ bool TextureCache<P>::ScaleDown(Image& image) {
}
template <class P>
+std::optional<DAddr> TextureCache<P>::ImageCpuAddress(GPUVAddr gpu_addr, size_t size) const {
+ if (const std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr)) {
+ return cpu_addr;
+ }
+ if (const std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, size)) {
+ return cpu_addr;
+ }
+ // No CPU backing (title 1 renders a bloom chain at GPU 0xf0000-0x15b600): a stand-in address
+ // fixed by the GPU address, at or above ~(1 << 40) so TrackImage skips it. A fresh address per
+ // lookup made every use create another image (thousands a minute, never found again).
+ constexpr GPUVAddr StandInRange = 1ULL << 40;
+ if (gpu_addr >= StandInRange || size > StandInRange - gpu_addr - 1) {
+ return std::nullopt;
+ }
+ return ~(1ULL << 40ULL) + gpu_addr;
+}
+
+template <class P>
ImageId TextureCache<P>::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr,
RelaxedOptions options) {
- std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr);
+ std::optional<DAddr> cpu_addr = ImageCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info));
if (!cpu_addr) {
- const auto size = CalculateGuestSizeInBytes(info);
- cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, size);
- if (!cpu_addr) {
- const DAddr fake_addr = ~(1ULL << 40ULL) + virtual_invalid_space;
- virtual_invalid_space += Common::AlignUp(size, 32);
- cpu_addr = std::optional<DAddr>(fake_addr);
- }
+ const DAddr fake_addr = ~(1ULL << 40ULL) + virtual_invalid_space;
+ virtual_invalid_space += Common::AlignUp(CalculateGuestSizeInBytes(info), 32);
+ cpu_addr = std::optional<DAddr>(fake_addr);
}
ASSERT_MSG(cpu_addr, "Tried to insert an image to an invalid gpu_addr={:#x}", gpu_addr);
- const ImageId image_id = JoinImages(info, gpu_addr, *cpu_addr);
+ ImageInfo storage_info = info;
+ storage_info.native_tiling = gpu_memory->UsesNativeTiling();
+ if (storage_info.native_tiling && info.type == ImageType::e3D) {
+ // Switch 2 3D images (lighting/fog volumes) are rare: log each one's shape.
+ LOG_INFO(HW_GPU, "3D image at {:#x}: {}x{}x{} levels {} format {} block {}x{}x{}",
+ gpu_addr, info.size.width, info.size.height, info.size.depth, info.resources.levels,
+ static_cast<u32>(info.format), info.block.width, info.block.height, info.block.depth);
+ }
+ const ImageId image_id = JoinImages(storage_info, gpu_addr, *cpu_addr);
const Image& image = slot_images[image_id];
// Using "image.gpu_addr" instead of "gpu_addr" is important because it might be different
const auto [it, is_new] = image_allocs_table.try_emplace(image.gpu_addr);
@@ -1634,10 +1867,36 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA
}
}
+ if constexpr (requires { runtime.UseOpaqueAstcFormat(); }) {
+ // Opaque ASTC goes to BC1 rather than BC3 when it is re-compressed (the Vulkan runtime
+ // decides). The guest data is already there when an image is first used.
+ constexpr size_t MaxScanBytes = 64_MiB;
+ if (IsPixelFormatASTC(new_info.format) && runtime.UseOpaqueAstcFormat()) {
+ const size_t scan_bytes = CalculateGuestSizeInBytes(new_info);
+ if (scan_bytes <= MaxScanBytes) {
+ astc_scan_buffer.resize(scan_bytes);
+ gpu_memory->ReadBlockUnsafe(gpu_addr, astc_scan_buffer.data(), scan_bytes);
+ new_info.astc_opaque = AstcDataLooksOpaque(astc_scan_buffer);
+ ++(new_info.astc_opaque ? astc_opaque_images : astc_alpha_images);
+ }
+ }
+ }
const ImageId new_image_id = slot_images.insert(runtime, new_info, gpu_addr, cpu_addr);
Image& new_image = slot_images[new_image_id];
new_image.allocation_tick = frame_tick;
+ {
+ // Which shapes are being created (periodic log): format, type, levels, layers, size.
+ const u64 key = (u64{static_cast<u32>(new_info.format)} << 56) |
+ (u64{static_cast<u32>(new_info.type)} << 52) |
+ (u64{static_cast<u32>(new_info.resources.levels) & 0xF} << 48) |
+ (u64{static_cast<u32>(new_info.resources.layers) & 0xFFF} << 36) |
+ (u64{new_info.size.width & 0x3FFFF} << 18) | (new_info.size.height & 0x3FFFF);
+ auto& shape = created_shapes[key];
+ ++shape.first;
+ shape.second = gpu_addr;
+ ++images_created;
+ }
if (!gpu_memory->IsContinuousRange(new_image.gpu_addr, new_image.guest_size_bytes) &&
new_info.is_sparse) {
@@ -1833,12 +2092,9 @@ std::optional<typename TextureCache<P>::BlitImages> TextureCache<P>::GetBlitImag
template <class P>
ImageId TextureCache<P>::FindDMAImage(const ImageInfo& info, GPUVAddr gpu_addr) {
- std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr);
+ const std::optional<DAddr> cpu_addr = ImageCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info));
if (!cpu_addr) {
- cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info));
- if (!cpu_addr) {
- return ImageId{};
- }
+ return ImageId{};
}
ImageId image_id{};
boost::container::small_vector<ImageId, 8> image_ids;
@@ -1859,6 +2115,13 @@ ImageId TextureCache<P>::FindDMAImage(const ImageInfo& info, GPUVAddr gpu_addr)
return true;
}
} else if (IsSubCopy(info, existing_image, gpu_addr)) {
+ if (True(existing_image.flags & ImageFlagBits::Unbacked)) {
+ static u32 logged{};
+ if (logged++ < 64) {
+ LOG_INFO(HW_GPU, "DMA copy at {:#x} ({}x{}) into unbacked image {:#x}", gpu_addr,
+ info.size.width, info.size.height, existing_image.gpu_addr);
+ }
+ }
image_id = existing_image_id;
image_ids.push_back(existing_image_id);
return true;
@@ -1954,6 +2217,13 @@ ImageViewId TextureCache<P>::FindRenderTargetView(const ImageInfo& info, GPUVAdd
base = image.TryFindBase(gpu_addr).value();
}
const s32 layers = image.info.type == ImageType::e3D ? info.size.depth : info.resources.layers;
+ if (True(image.flags & ImageFlagBits::Unbacked)) {
+ static u32 logged{};
+ if (logged++ < 64) {
+ LOG_INFO(HW_GPU, "Render target in unbacked image {:#x} at {:#x}: {}x{} level {} layer {}",
+ image.gpu_addr, gpu_addr, info.size.width, info.size.height, base.level, base.layer);
+ }
+ }
const SubresourceRange range{
.base = base,
.extent = {.levels = 1, .layers = layers},
@@ -2136,6 +2406,13 @@ ImageViewId TextureCache<P>::FindOrEmplaceImageView(ImageId image_id, const Imag
if (const ImageViewId image_view_id = image.FindView(info); image_view_id) {
return image_view_id;
}
+ if (True(image.flags & ImageFlagBits::Unbacked)) {
+ // How the game uses an image too large to back (e.g. a sparse virtual texture).
+ LOG_INFO(HW_GPU, "View of unbacked image {:#x}: type {} format {} levels {}+{} layers {}+{}",
+ image.gpu_addr, static_cast<u32>(info.type), static_cast<u32>(info.format),
+ info.range.base.level, info.range.extent.levels, info.range.base.layer,
+ info.range.extent.layers);
+ }
const ImageViewId image_view_id = slot_image_views.insert(runtime, info, image_id, image, slot_images);
image.InsertView(info, image_view_id);
return image_view_id;
@@ -2316,6 +2593,16 @@ void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) {
template <class P>
void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) {
ImageBase& image = slot_images[image_id];
+ ++images_deleted;
+ // Drop every decode still queued for it, whatever IsDecoding says: an image re-decoded while
+ // a decode was in flight has two, and the first to finish clears the flag. A decode left
+ // behind was uploaded into the next image to take this slot (another texture's data: garbage
+ // textures, and a staging overrun when that image was smaller).
+ for (auto& async_decode : async_decodes) {
+ if (async_decode->image_id == image_id) {
+ async_decode->cancelled = true;
+ }
+ }
if (image.HasScaled()) {
total_used_memory -= GetScaledImageSizeBytes(image);
}
@@ -2385,7 +2672,11 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) {
}
slot_image_views.erase(image_view_id);
}
- if (!immediate_delete) {
+ const bool retained = RetainPendingImageDownloads(image_id, slot_images[image_id],
+ uncommitted_downloads, committed_downloads);
+ if (retained) {
+ LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value());
+ } else if (!immediate_delete) {
sentenced_images.Push(std::move(slot_images[image_id]));
}
slot_images.erase(image_id);
@@ -2537,6 +2828,11 @@ template <class P>
void TextureCache<P>::CopyImage(ImageId dst_id, ImageId src_id, std::vector<ImageCopy> copies) {
Image& dst = slot_images[dst_id];
Image& src = slot_images[src_id];
+ if (dst.info.astc_opaque != src.info.astc_opaque) {
+ // BC1 and BC3 blocks differ in size, so the host can't copy between them; both hold the
+ // same guest data, which is uploaded from guest memory instead.
+ return;
+ }
const bool is_rescaled = True(src.flags & ImageFlagBits::Rescaled);
if (is_rescaled) {
ASSERT(True(dst.flags & ImageFlagBits::Rescaled));
@@ -2628,7 +2924,8 @@ void TextureCache<P>::BindRenderTarget(ImageViewId* old_id, ImageViewId new_id)
}
if (new_id) {
const ImageViewBase& old_view = slot_image_views[new_id];
- if (True(old_view.flags & ImageViewFlagBits::PreemtiveDownload)) {
+ if (True(old_view.flags & ImageViewFlagBits::PreemtiveDownload) &&
+ False(slot_images[old_view.image_id].flags & (ImageFlagBits::Unbacked | ImageFlagBits::TrimmedLevels))) {
const PendingDownload new_download{true, 0, old_view.image_id};
uncommitted_downloads.emplace_back(new_download);
}
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index 8a2318cc72..11ac8a2421 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -9,6 +9,7 @@
#include <atomic>
#include <deque>
#include <limits>
+#include <map>
#include <mutex>
#include <span>
#include <type_traits>
@@ -22,12 +23,14 @@
#include <queue>
#include "common/common_types.h"
+#include "common/spin_recursive_mutex.h"
#include "common/hash.h"
#include "common/literals.h"
#include "common/lru_cache.h"
#include <ranges>
#include "common/scratch_buffer.h"
#include "common/slot_vector.h"
+#include "video_core/texture_cache/pending_download.h"
#include "common/thread_worker.h"
#include "video_core/compatible_formats.h"
#include "video_core/control/channel_state_cache.h"
@@ -67,6 +70,7 @@ struct AsyncDecodeContext {
boost::container::small_vector<BufferImageCopy, 16> copies;
std::mutex mutex;
std::atomic_bool complete;
+ bool cancelled{}; ///< The image was deleted meanwhile: drop the result (GPU thread only).
};
using TextureCacheGPUMap = ::Common::unordered_map<u64, std::vector<ImageId>, Common::IdentityHash<u64>>;
@@ -118,6 +122,21 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI
static constexpr s64 TARGET_THRESHOLD = 4_GiB;
#endif
+ // Periodic statistics (TickFrame).
+ u64 gc_deletes{};
+ u64 gc_costly_deletes{};
+ u64 async_decodes_queued{};
+ u64 images_created{};
+ bool native_thresholds_applied{};
+ u64 native_critical_memory{};
+ u64 native_expected_memory{};
+ u64 budget_check_frames{};
+ u64 astc_opaque_images{};
+ u64 astc_alpha_images{};
+ std::vector<u8> astc_scan_buffer;
+ u64 images_deleted{};
+ std::map<u64, std::pair<u32, GPUVAddr>> created_shapes;
+
static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB;
static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB;
static constexpr size_t GC_EMERGENCY_COUNTS = 2;
@@ -226,6 +245,9 @@ public:
/// Return true when there are uncommitted images to be downloaded
[[nodiscard]] bool HasUncommittedFlushes() const noexcept;
+ /// Whether CommitAsyncFlushes would commit anything (downloads or their buffers).
+ [[nodiscard]] bool HasAsyncFlushWork() const noexcept;
+
/// Return true when the caller should wait for async downloads
[[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept;
@@ -258,7 +280,7 @@ public:
/// Prepare an image to be used
void PrepareImage(ImageId image_id, bool is_modification, bool invalidate);
- std::recursive_mutex mutex;
+ Common::SpinRecursiveMutex mutex;
private:
/// Iterate over all page indices in a range
@@ -448,6 +470,12 @@ private:
DAddr virtual_invalid_space{};
+ /// Applies EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB and logs the thresholds.
+ void ApplyThresholdOverrides(bool log = true);
+
+ /// The CPU address an image at gpu_addr is keyed by: its backing, or a fixed stand-in.
+ [[nodiscard]] std::optional<DAddr> ImageCpuAddress(GPUVAddr gpu_addr, size_t size) const;
+
bool has_deleted_images = false;
bool is_rescaling = false;
u64 total_used_memory = 0;
@@ -463,11 +491,7 @@ private:
size_t size;
};
- struct PendingDownload {
- bool is_swizzle;
- size_t async_buffer_id;
- Common::SlotId object_id;
- };
+ using PendingDownload = PendingImageDownload<Image>;
Common::SlotVector<Image> slot_images;
Common::SlotVector<ImageMapView> slot_map_views;
diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp
index e55d0752ec..ffc1d59ec3 100644
--- a/src/video_core/texture_cache/util.cpp
+++ b/src/video_core/texture_cache/util.cpp
@@ -11,6 +11,7 @@
#include <algorithm>
#include <array>
+#include <cstring>
#include <numeric>
#include <optional>
#include <span>
@@ -439,7 +440,7 @@ template <u32 GOB_EXTENT>
}
layers = 1;
} else {
- layers = (std::max)(resources.layers, info.resources.layers + base->layer);
+ layers = (std::max)(info.resources.layers, resources.layers + base->layer);
}
return OverlapResult{
.gpu_addr = overlap.gpu_addr,
@@ -566,7 +567,7 @@ void SwizzleBlockLinearImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr
dst(gpu_memory, gpu_addr + guest_offset, subresource_size, &tmp_buffer);
SwizzleTexture(dst, src, bytes_per_block, num_tiles.width, num_tiles.height,
- num_tiles.depth, block.height, block.depth);
+ num_tiles.depth, block.height, block.depth, 1, gpu_memory.UsesNativeTiling());
}
host_offset += host_bytes_per_layer;
@@ -603,6 +604,10 @@ u32 CalculateUnswizzledSizeBytes(const ImageInfo& info) noexcept {
return NumBlocksPerLayer(info, tile_size) * info.resources.layers * BytesPerBlock(info.format);
}
+bool AstcDataLooksOpaque(std::span<const u8> data) noexcept {
+ return Tegra::Texture::ASTC::IsOpaque(data);
+}
+
u32 CalculateConvertedSizeBytes(const ImageInfo& info) noexcept {
if (info.type == ImageType::Buffer) {
return info.size.width * BytesPerBlock(info.format);
@@ -611,8 +616,10 @@ u32 CalculateConvertedSizeBytes(const ImageInfo& info) noexcept {
if (IsPixelFormatASTC(info.format) && Settings::values.astc_recompression.GetValue() !=
Settings::AstcRecompression::Uncompressed) {
const u32 bpp_div =
- Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Bc1 ? 2
- : 1;
+ Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Bc1 ||
+ info.astc_opaque
+ ? 2
+ : 1;
// NumBlocksPerLayer doesn't account for this correctly, so we have to do it manually.
u32 output_size = 0;
for (s32 i = 0; i < info.resources.levels; i++) {
@@ -913,7 +920,7 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory
const std::span<u8> dst = output.subspan(host_offset);
const std::span<const u8> src = input.subspan(guest_offset + guest_layer_offset);
UnswizzleTexture(dst, src, 1U << bpp_log2, num_tiles.width, num_tiles.height,
- num_tiles.depth, block.height, block.depth, stride_alignment);
+ num_tiles.depth, block.height, block.depth, stride_alignment, gpu_memory.UsesNativeTiling());
guest_layer_offset += layer_stride;
host_offset += host_bytes_per_layer;
}
@@ -922,6 +929,35 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory
return copies;
}
+void ConvertedLayout(const ImageInfo& info, std::span<BufferImageCopy> copies) {
+ // Same offsets and sizes as ConvertImage below.
+ const auto recompression = Settings::values.astc_recompression.GetValue();
+ const bool astc = IsPixelFormatASTC(info.format);
+ u64 output_offset = 0;
+ for (BufferImageCopy& copy : copies) {
+ const Extent3D mip_size = AdjustMipSize(info.size, copy.image_subresource.base_level);
+ const u64 layers = static_cast<u64>(copy.image_subresource.num_layers) * copy.image_extent.depth;
+ u64 size;
+ if (astc && recompression == Settings::AstcRecompression::Uncompressed) {
+ size = u64{copy.image_extent.width} * copy.image_extent.height * layers *
+ BytesPerBlock(PixelFormat::A8B8G8R8_UNORM);
+ } else if (astc) {
+ const u64 bpp_div =
+ recompression == Settings::AstcRecompression::Bc1 || info.astc_opaque ? 2 : 1;
+ size = u64{Common::AlignUp(copy.image_extent.width, 4U)} *
+ Common::AlignUp(copy.image_extent.height, 4U) * layers / bpp_div;
+ } else {
+ size = u64{copy.image_extent.width} * copy.image_extent.height * layers *
+ ConvertedBytesPerBlock(info.format);
+ }
+ copy.buffer_offset = output_offset;
+ copy.buffer_size = size;
+ copy.buffer_row_length = mip_size.width;
+ copy.buffer_image_height = mip_size.height;
+ output_offset += size;
+ }
+}
+
void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8> output,
std::span<BufferImageCopy> copies) {
u32 output_offset = 0;
@@ -955,10 +991,11 @@ void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8
} else if (astc) {
// BC1 uses 0.5 bytes per texel
// BC3 uses 1 byte per texel
- const auto compress = recompression_setting == Settings::AstcRecompression::Bc1
- ? Tegra::Texture::BCN::CompressBC1
- : Tegra::Texture::BCN::CompressBC3;
- const auto bpp_div = recompression_setting == Settings::AstcRecompression::Bc1 ? 2 : 1;
+ const bool bc1 =
+ recompression_setting == Settings::AstcRecompression::Bc1 || info.astc_opaque;
+ const auto compress =
+ bc1 ? Tegra::Texture::BCN::CompressBC1 : Tegra::Texture::BCN::CompressBC3;
+ const auto bpp_div = bc1 ? 2 : 1;
const u32 plane_dim = copy.image_extent.width * copy.image_extent.height;
const u32 level_size = plane_dim * copy.image_extent.depth *
@@ -1298,6 +1335,13 @@ void DeduceBlitImages(ImageInfo& dst_info, ImageInfo& src_info, const ImageBase*
}
u32 MapSizeBytes(const ImageBase& image) {
+ if (True(image.flags & ImageFlagBits::AcceleratedUpload) && image.host_level_offset != 0) {
+ // Each layer's mip chain from host_level_offset on (see UploadImageContents).
+ const auto uploads = FullUploadSwizzles(image.info);
+ const u32 tail = image.info.layer_stride -
+ static_cast<u32>(uploads[image.host_level_offset].buffer_offset);
+ return tail * image.info.resources.layers;
+ }
if (True(image.flags & ImageFlagBits::AcceleratedUpload)) {
return image.guest_size_bytes;
} else if (True(image.flags & ImageFlagBits::Converted)) {
diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h
index 590d85b042..0381d2d24b 100644
--- a/src/video_core/texture_cache/util.h
+++ b/src/video_core/texture_cache/util.h
@@ -36,6 +36,9 @@ struct OverlapResult {
[[nodiscard]] u32 CalculateConvertedSizeBytes(const ImageInfo& info) noexcept;
+/// True if no 16-byte ASTC block in `data` decodes to alpha below 1.
+[[nodiscard]] bool AstcDataLooksOpaque(std::span<const u8> data) noexcept;
+
[[nodiscard]] u32 CalculateLayerStride(const ImageInfo& info) noexcept;
[[nodiscard]] u32 CalculateLayerSize(const ImageInfo& info) noexcept;
@@ -69,6 +72,9 @@ struct OverlapResult {
Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info,
std::span<const u8> input, std::span<u8> output);
+/// Rewrites `copies` (from UnswizzleImage) to the layout ConvertImage produces, without converting.
+void ConvertedLayout(const ImageInfo& info, std::span<BufferImageCopy> copies);
+
void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8> output,
std::span<BufferImageCopy> copies);
diff --git a/src/video_core/textures/astc.cpp b/src/video_core/textures/astc.cpp
index 0b6d06c49a..950ce28d6b 100644
--- a/src/video_core/textures/astc.cpp
+++ b/src/video_core/textures/astc.cpp
@@ -2038,4 +2038,69 @@ void Decompress(std::span<const uint8_t> data, uint32_t width, uint32_t height,
}
}
+bool IsOpaque(std::span<const uint8_t> data) {
+ // Endpoint modes with an alpha channel: LDR luminance+alpha (4, 5), RGB scale + two alpha
+ // (10), LDR RGBA (12, 13) and HDR RGBA (14, 15).
+ constexpr u32 AlphaModes = (1U << 4) | (1U << 5) | (1U << 10) | (1U << 12) | (1U << 13) |
+ (1U << 14) | (1U << 15);
+ for (size_t offset = 0; offset + 16 <= data.size(); offset += 16) {
+ const std::span<const u8> block = data.subspan(offset, 16);
+ InputBitStream strm(block);
+ const TexelWeightParams params = DecodeBlockInfo(strm);
+ if (params.m_bError) {
+ continue; // decodes to the opaque error colour
+ }
+ if (params.m_bVoidExtentLDR || params.m_bVoidExtentHDR) {
+ // One RGBA16 colour in bits 64-127: UNORM16, or FP16 for HDR.
+ const u32 alpha = block[14] | (u32{block[15]} << 8);
+ if (alpha != (params.m_bVoidExtentHDR ? 0x3C00U : 0xFFFFU)) {
+ return false;
+ }
+ continue;
+ }
+ const u32 partitions = strm.ReadBits<2>() + 1;
+ if (partitions == 4 && params.m_bDualPlane) {
+ continue;
+ }
+ if (partitions == 1) {
+ if ((AlphaModes >> strm.ReadBits<4>()) & 1) {
+ return false;
+ }
+ continue;
+ }
+ strm.ReadBits<10>(); // partition index
+ const u32 base_cem = strm.ReadBits<6>();
+ const u32 base_mode = base_cem & 3;
+ if (base_mode == 0) {
+ if ((AlphaModes >> (base_cem >> 2)) & 1) {
+ return false;
+ }
+ continue;
+ }
+ // The rest of the per-partition modes sit just below the weights (DecompressBlock).
+ const u32 weight_bits = params.GetPackedBitSize();
+ if (weight_bits < 24 || weight_bits > 96) {
+ continue;
+ }
+ const u32 extra_bits = partitions == 2 ? 2 : partitions == 3 ? 5 : 8;
+ const u32 extra_offset = 128 - weight_bits - extra_bits;
+ u32 extra = 0;
+ for (u32 bit = 0; bit < extra_bits; ++bit) {
+ const u32 position = extra_offset + bit;
+ extra |= ((block[position / 8] >> (position % 8)) & 1U) << bit;
+ }
+ u32 cem = ((extra << 6) | base_cem) >> 2;
+ const u32 classes = cem;
+ cem >>= partitions;
+ for (u32 i = 0; i < partitions; ++i) {
+ const u32 mode = ((base_mode - (((classes >> i) & 1) != 0 ? 0 : 1)) << 2) |
+ ((cem >> (2 * i)) & 3);
+ if ((AlphaModes >> mode) & 1) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
} // namespace Tegra::Texture::ASTC
diff --git a/src/video_core/textures/astc.h b/src/video_core/textures/astc.h
index afd3933c3e..95c2c7ffea 100644
--- a/src/video_core/textures/astc.h
+++ b/src/video_core/textures/astc.h
@@ -8,4 +8,7 @@ namespace Tegra::Texture::ASTC {
void Decompress(std::span<const uint8_t> data, uint32_t width, uint32_t height, uint32_t depth,
uint32_t block_width, uint32_t block_height, std::span<uint8_t> output);
+/// True if no 16-byte block in `data` decodes to alpha below 1.
+bool IsOpaque(std::span<const uint8_t> data);
+
} // namespace Tegra::Texture::ASTC
diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp
index 25b0860b0f..ef50be156b 100644
--- a/src/video_core/textures/decoders.cpp
+++ b/src/video_core/textures/decoders.cpp
@@ -18,6 +18,11 @@
namespace Tegra::Texture {
namespace {
+constexpr u32 NativeGobOffset(u32 offset, bool native_tiling) {
+ if (!native_tiling) return offset;
+ const u32 difference = ((offset >> 5) ^ (offset >> 6)) & 1;
+ return offset ^ (difference << 5) ^ (difference << 6);
+}
template <u32 mask>
constexpr u32 pdep(u32 value) {
u32 result = 0;
@@ -38,7 +43,7 @@ void incrpdep(u32& value) {
template <bool TO_LINEAR, u32 BYTES_PER_PIXEL>
void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 height, u32 depth,
- u32 block_height, u32 block_depth, u32 stride) {
+ u32 block_height, u32 block_depth, u32 stride, bool native_tiling) {
// The origin of the transformation can be configured here, leave it as zero as the current API
// doesn't expose it.
static constexpr u32 origin_x = 0;
@@ -77,7 +82,7 @@ void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32
const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift;
const u32 base_swizzled_offset = offset_z + offset_y + offset_x;
- const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y);
+ const u32 swizzled_offset = NativeGobOffset(base_swizzled_offset + (swizzled_x | swizzled_y), native_tiling);
const u32 unswizzled_offset =
slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL;
@@ -94,7 +99,7 @@ void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32
template <bool TO_LINEAR, u32 BYTES_PER_PIXEL>
void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 height,
u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 num_lines,
- u32 block_height, u32 block_depth, u32 pitch_linear) {
+ u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) {
// The origin of the transformation can be configured here, leave it as zero as the current API
// doesn't expose it.
static constexpr u32 origin_z = 0;
@@ -136,7 +141,7 @@ void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 wid
const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift;
const u32 base_swizzled_offset = offset_z + offset_y + offset_x;
- const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y);
+ const u32 swizzled_offset = NativeGobOffset(base_swizzled_offset + (swizzled_x | swizzled_y), native_tiling);
const u32 unswizzled_offset =
slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL;
@@ -156,12 +161,12 @@ void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 wid
template <bool TO_LINEAR>
void Swizzle(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
- u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment) {
+ u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment, bool native_tiling) {
switch (bytes_per_pixel) {
#define BPP_CASE(x) \
case x: \
return SwizzleImpl<TO_LINEAR, x>(output, input, width, height, depth, block_height, \
- block_depth, stride_alignment);
+ block_depth, stride_alignment, native_tiling);
BPP_CASE(1)
BPP_CASE(2)
BPP_CASE(3)
@@ -181,35 +186,35 @@ void Swizzle(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixe
void UnswizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel,
u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth,
- u32 stride_alignment) {
+ u32 stride_alignment, bool native_tiling) {
const u32 stride = Common::AlignUpLog2(width, stride_alignment) * bytes_per_pixel;
const u32 new_bpp = (std::min)(4U, static_cast<u32>(std::countr_zero(width * bytes_per_pixel)));
width = (width * bytes_per_pixel) >> new_bpp;
bytes_per_pixel = 1U << new_bpp;
Swizzle<false>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth,
- stride);
+ stride, native_tiling);
}
void SwizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
u32 height, u32 depth, u32 block_height, u32 block_depth,
- u32 stride_alignment) {
+ u32 stride_alignment, bool native_tiling) {
const u32 stride = Common::AlignUpLog2(width, stride_alignment) * bytes_per_pixel;
const u32 new_bpp = (std::min)(4U, static_cast<u32>(std::countr_zero(width * bytes_per_pixel)));
width = (width * bytes_per_pixel) >> new_bpp;
bytes_per_pixel = 1U << new_bpp;
Swizzle<true>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth,
- stride);
+ stride, native_tiling);
}
void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 extent_y,
- u32 block_height, u32 block_depth, u32 pitch_linear) {
+ u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) {
switch (bytes_per_pixel) {
#define BPP_CASE(x) \
case x: \
return SwizzleSubrectImpl<true, x>(output, input, width, height, depth, origin_x, \
origin_y, extent_x, extent_y, block_height, \
- block_depth, pitch_linear);
+ block_depth, pitch_linear, native_tiling);
BPP_CASE(1)
BPP_CASE(2)
BPP_CASE(3)
@@ -227,13 +232,13 @@ void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_p
void UnswizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel,
u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x,
- u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear) {
+ u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) {
switch (bytes_per_pixel) {
#define BPP_CASE(x) \
case x: \
return SwizzleSubrectImpl<false, x>(output, input, width, height, depth, origin_x, \
origin_y, extent_x, extent_y, block_height, \
- block_depth, pitch_linear);
+ block_depth, pitch_linear, native_tiling);
BPP_CASE(1)
BPP_CASE(2)
BPP_CASE(3)
diff --git a/src/video_core/textures/decoders.h b/src/video_core/textures/decoders.h
index 1e14a38753..6151967869 100644
--- a/src/video_core/textures/decoders.h
+++ b/src/video_core/textures/decoders.h
@@ -29,12 +29,12 @@ constexpr u32 SWIZZLE_Y_BITS = 0b011010000;
/// Unswizzles a block linear texture into linear memory.
void UnswizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel,
u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth,
- u32 stride_alignment = 1);
+ u32 stride_alignment = 1, bool native_tiling = false);
/// Swizzles linear memory into a block linear texture.
void SwizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
u32 height, u32 depth, u32 block_height, u32 block_depth,
- u32 stride_alignment = 1);
+ u32 stride_alignment = 1, bool native_tiling = false);
/// This function calculates the correct size of a texture depending if it's tiled or not.
std::size_t CalculateSize(bool tiled, u32 bytes_per_pixel, u32 width, u32 height, u32 depth,
@@ -43,12 +43,12 @@ std::size_t CalculateSize(bool tiled, u32 bytes_per_pixel, u32 width, u32 height
/// Copies an untiled subrectangle into a tiled surface.
void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 extent_y,
- u32 block_height, u32 block_depth, u32 pitch_linear);
+ u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling = false);
/// Copies a tiled subrectangle into a linear surface.
void UnswizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel,
u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x,
- u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear);
+ u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling = false);
/// Obtains the offset of the gob for positions 'dst_x' & 'dst_y'
u64 GetGOBOffset(u32 width, u32 height, u32 dst_x, u32 dst_y, u32 block_height,
diff --git a/src/video_core/textures/workers.cpp b/src/video_core/textures/workers.cpp
index a1953c4bf4..e3a0f26cf2 100644
--- a/src/video_core/textures/workers.cpp
+++ b/src/video_core/textures/workers.cpp
@@ -4,13 +4,23 @@
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <cstdlib>
+
#include "video_core/textures/workers.h"
namespace Tegra::Texture {
Common::ThreadWorker& GetThreadWorkers() {
- static Common::ThreadWorker workers{(std::max)(std::thread::hardware_concurrency(), 2U) / 2,
- "ImageTranscode"};
+ // A quarter of the hardware threads (at least 2). Half took ~3.5 cores from the emulated
+ // CPU cores and the GPU thread on an 8-core laptop when textures were thrashing; with an
+ // eighth (2) both threads stayed saturated and streamed textures arrived seconds late (shown
+ // as placeholders), while 4 cost no frame rate. EDEN_NX2_TRANSCODE_THREADS=<n> overrides.
+ static Common::ThreadWorker workers{[] {
+ if (const char* text = std::getenv("EDEN_NX2_TRANSCODE_THREADS")) {
+ return (std::max)(static_cast<u32>(std::strtoul(text, nullptr, 10)), 1U);
+ }
+ return (std::max)(std::thread::hardware_concurrency() / 4, 2U);
+ }(), "ImageTranscode"};
return workers;
}
diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp
index be09bca8d3..69c3ba14fa 100644
--- a/src/video_core/vulkan_common/vulkan_device.cpp
+++ b/src/video_core/vulkan_common/vulkan_device.cpp
@@ -5,6 +5,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
+#include <array>
#include <bitset>
#include <chrono>
#include <filesystem>
@@ -752,9 +753,10 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
.flags = flags,
.physicalDevice = physical,
.device = *logical,
- .preferredLargeHeapBlockSize = is_integrated
- ? (64u * 1024u * 1024u)
- : (256u * 1024u * 1024u),
+ // 64 MiB blocks on discrete GPUs too: with 256 MiB ones, Switch 2 titles' mix of
+ // texture sizes left ~38% of the image blocks empty (2 GB of an 8 GB card in title
+ // 1), which the texture cache counted as used and kept evicting textures for.
+ .preferredLargeHeapBlockSize = 64u * 1024u * 1024u,
.pAllocationCallbacks = nullptr,
.pDeviceMemoryCallbacks = nullptr,
.pHeapSizeLimit = nullptr,
@@ -1609,6 +1611,33 @@ u64 Device::GetDeviceMemoryUsage() const {
for (const size_t heap : valid_heap_memory) {
result += budget.heapUsage[heap];
}
+ // Space in VMA's blocks that no allocation uses counts as used, but freeing an image gives
+ // its share back only once its whole block is empty: the caches, collecting until this drops,
+ // evicted thousands of textures in a row without lowering it (title 1). Count only what
+ // allocations use.
+ if (allocator != nullptr) {
+ std::array<VmaBudget, VK_MAX_MEMORY_HEAPS> heap_budgets{};
+ vmaGetHeapBudgets(allocator, heap_budgets.data());
+ for (const size_t heap : valid_heap_memory) {
+ const VmaStatistics& statistics = heap_budgets[heap].statistics;
+ const u64 unused = statistics.blockBytes - statistics.allocationBytes;
+ result -= (std::min)(result, unused);
+ }
+ }
+ return result;
+}
+
+u64 Device::GetDeviceMemoryBudget() const {
+ if (!extensions.memory_budget) {
+ return 0;
+ }
+ VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{};
+ budget.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT;
+ physical.GetMemoryProperties(&budget);
+ u64 result{};
+ for (const size_t heap : valid_heap_memory) {
+ result += budget.heapBudget[heap];
+ }
return result;
}
diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h
index bbf3754e39..b1fa8db183 100644
--- a/src/video_core/vulkan_common/vulkan_device.h
+++ b/src/video_core/vulkan_common/vulkan_device.h
@@ -963,6 +963,9 @@ FN_MAX_LIMIT_LIST
u64 GetDeviceMemoryUsage() const;
+ /// The driver's current memory budget for this process (VK_EXT_memory_budget), 0 without it.
+ u64 GetDeviceMemoryBudget() const;
+
u32 GetSetsPerPool() const {
return sets_per_pool;
}
@@ -1145,7 +1148,7 @@ private:
private:
VkInstance instance; ///< Vulkan instance.
- VmaAllocator allocator; ///< VMA allocator.
+ VmaAllocator allocator{}; ///< VMA allocator.
vk::DeviceDispatch dld; ///< Device function pointers.
vk::PhysicalDevice physical; ///< Physical device.
vk::Device logical; ///< Logical device.
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
index f4721543ae..27f8fa6b4c 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
@@ -5,6 +5,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
+#include <array>
#include <bit>
#include <limits>
#include <optional>
@@ -205,10 +206,12 @@ MemoryAllocator::MemoryAllocator(const Device &device_)
MemoryAllocator::~MemoryAllocator() = default;
-vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const
+vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicated) const
{
const VmaAllocationCreateInfo alloc_ci = {
- .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT,
+ .flags = static_cast<VmaAllocationCreateFlags>(VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) |
+ (dedicated ? VmaAllocationCreateFlags{VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT}
+ : VmaAllocationCreateFlags{0}),
.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
.requiredFlags = 0,
.preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
@@ -221,7 +224,53 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const
VkImage handle{};
VmaAllocation allocation{};
VmaAllocationInfo alloc_info{};
- vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info));
+ // Images this large (title 1's sparse texture arrays reach 4.5G texels, ~23 GB) are never
+ // backed: VMA would quietly place them in system memory once VRAM is full. They take the
+ // unbacked sparse fallback below.
+ const u64 texels = u64{ci.extent.width} * ci.extent.height * ci.extent.depth * ci.arrayLayers;
+ const bool huge = texels > (u64{1} << 28);
+ VkResult result = huge ? VK_ERROR_OUT_OF_DEVICE_MEMORY
+ : vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info);
+ if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && !huge) {
+ // Over the device memory budget (Switch 2 titles fill 8 GB cards): let the driver page
+ // it out rather than failing the draw and aborting.
+ static u32 logged{};
+ if (logged++ < 32) {
+ LOG_WARNING(Render_Vulkan,
+ "Image {}x{}x{} ({} levels, {} layers, format {}) is over the memory "
+ "budget; allocating it anyway",
+ ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels,
+ ci.arrayLayers, static_cast<u32>(ci.format));
+ }
+ VmaAllocationCreateInfo relaxed = alloc_ci;
+ relaxed.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT;
+ result = vmaCreateImage(allocator, &ci, &relaxed, &handle, &allocation, &alloc_info);
+ }
+ if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && ci.imageType != VK_IMAGE_TYPE_1D) {
+ // Too big to back at all, e.g. title 1's sparse virtual texture arrays (8192x8192,
+ // 68 layers, ~23 GB): on the console only resident tiles have memory. Create a sparse
+ // resident image with nothing bound, so views and copies still work; reads of it return
+ // zero and writes are dropped, instead of aborting.
+ VkImageCreateInfo sparse_ci = ci;
+ sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
+ VkImage sparse{};
+ if (device.GetDispatchLoader().vkCreateImage(*device.GetLogical(), &sparse_ci, nullptr,
+ &sparse) == VK_SUCCESS) {
+ LOG_WARNING(Render_Vulkan,
+ "Image {}x{}x{} ({} levels, {} layers, format {}) does not fit; using an "
+ "unbacked sparse image",
+ ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels,
+ ci.arrayLayers, static_cast<u32>(ci.format));
+ return vk::Image(sparse, ci.usage, *device.GetLogical(), allocator, VK_NULL_HANDLE,
+ device.GetDispatchLoader());
+ }
+ }
+ if (result != VK_SUCCESS) {
+ LOG_CRITICAL(Render_Vulkan, "Image {}x{}x{} ({} levels, {} layers, format {}) failed: {}",
+ ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels,
+ ci.arrayLayers, static_cast<u32>(ci.format), static_cast<int>(result));
+ }
+ vk::Check(result);
// Log GPU memory allocation for images
if (GPU::Logging::IsActive() &&
@@ -405,4 +454,29 @@ MemoryCommit MemoryAllocator::Commit(const vk::Buffer &buffer, MemoryUsage usage
return MemoryCommit(allocator, a, info);
}
+void LogMemoryStatistics(const Device& device) {
+ const VmaAllocator allocator = device.GetAllocator();
+ const VkPhysicalDeviceMemoryProperties* properties{};
+ vmaGetMemoryProperties(allocator, &properties);
+ VmaTotalStatistics stats{};
+ vmaCalculateStatistics(allocator, &stats);
+ for (u32 type = 0; type < properties->memoryTypeCount; ++type) {
+ const VmaStatistics& s = stats.memoryType[type].statistics;
+ if (s.blockCount == 0) {
+ continue;
+ }
+ LOG_INFO(Render_Vulkan, "VMA type {} (heap {}, flags {:#x}): {} blocks {} MiB, {} allocations {} MiB",
+ type, properties->memoryTypes[type].heapIndex,
+ static_cast<u32>(properties->memoryTypes[type].propertyFlags), s.blockCount,
+ s.blockBytes >> 20, s.allocationCount, s.allocationBytes >> 20);
+ }
+ std::array<VmaBudget, VK_MAX_MEMORY_HEAPS> budgets{};
+ vmaGetHeapBudgets(allocator, budgets.data());
+ for (u32 heap = 0; heap < properties->memoryHeapCount; ++heap) {
+ LOG_INFO(Render_Vulkan, "VMA heap {}: usage {} MiB of budget {} MiB, VMA blocks {} MiB",
+ heap, budgets[heap].usage >> 20, budgets[heap].budget >> 20,
+ budgets[heap].statistics.blockBytes >> 20);
+ }
+}
+
} // namespace Vulkan
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h
index a0f9081fc0..fa7d4b41c3 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.h
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h
@@ -103,7 +103,9 @@ namespace Vulkan {
MemoryAllocator(const MemoryAllocator &) = delete;
- vk::Image CreateImage(const VkImageCreateInfo &ci) const;
+ /// `dedicated`: own device memory, returned to the driver on destruction (short-lived
+ /// images would otherwise leave holes in the shared blocks).
+ vk::Image CreateImage(const VkImageCreateInfo &ci, bool dedicated = false) const;
vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const;
@@ -142,4 +144,7 @@ namespace Vulkan {
u32 valid_memory_types{~0u};
};
+ /// Logs VMA's blocks and allocations per memory type, and each heap's budget.
+ void LogMemoryStatistics(const Device& device);
+
} // namespace Vulkan
diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h
index 97d2ba3700..947150648d 100644
--- a/src/video_core/vulkan_common/vulkan_wrapper.h
+++ b/src/video_core/vulkan_common/vulkan_wrapper.h
@@ -689,6 +689,11 @@ public:
allocation{allocation_}, dld{&dld_} {}
Image() = default;
+ /// True for an image created without memory (an unbacked sparse image).
+ bool IsUnbacked() const noexcept {
+ return handle != VK_NULL_HANDLE && allocation == VK_NULL_HANDLE;
+ }
+
Image(const Image&) = delete;
Image& operator=(const Image&) = delete;
diff --git a/src/yuzu_cmd/CMakeLists.txt b/src/yuzu_cmd/CMakeLists.txt
index 17a1545167..a87b4e5288 100644
--- a/src/yuzu_cmd/CMakeLists.txt
+++ b/src/yuzu_cmd/CMakeLists.txt
@@ -31,6 +31,8 @@ add_executable(yuzu-cmd
emu_window/emu_window_sdl3_null.h
emu_window/emu_window_sdl3_vk.cpp
emu_window/emu_window_sdl3_vk.h
+ nx2_sampler.cpp
+ nx2_sampler.h
sdl_config.cpp
sdl_config.h
yuzu.cpp
diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp
index 950e001f81..174f8b5328 100644
--- a/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp
+++ b/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp
@@ -4,6 +4,10 @@
// SPDX-FileCopyrightText: 2016 Citra Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <cstdlib>
+#include <cstdio>
+#include <chrono>
+#include <array>
#include <SDL3/SDL.h>
#include <SDL3/SDL_timer.h>
#ifdef __EMSCRIPTEN__
@@ -39,7 +43,10 @@ EmuWindow_SDL3::EmuWindow_SDL3(InputCommon::InputSubsystem* input_subsystem_, Co
}, this);
mouse_timer = SDL_AddTimer(100, [](void *userdata, SDL_TimerID, Uint32) -> Uint32 {
auto* this_ = (EmuWindow_SDL3*)userdata;
- this_->input_subsystem->GetMouse()->NotifyChanged();
+ // The timer can fire before the input subsystem has created its mouse driver.
+ if (auto* const mouse = this_->input_subsystem->GetMouse()) {
+ mouse->NotifyChanged();
+ }
return 100;
}, this);
}
@@ -115,7 +122,26 @@ void EmuWindow_SDL3::OnFingerUp() {
input_subsystem->GetTouchScreen()->ReleaseAllTouch();
}
+// Seconds since the first frame; shared by the key script (yuzu.cpp) and key recording.
+double Nx2KeyClockSeconds() {
+ static const auto begin = std::chrono::steady_clock::now();
+ return std::chrono::duration<double>(std::chrono::steady_clock::now() - begin).count();
+}
+
void EmuWindow_SDL3::OnKeyEvent(int key, u8 state) {
+ // EDEN_NX2_KEY_RECORD=<file>: append each press/release ("seconds:code", "seconds:-code").
+ static std::FILE* const record = [] {
+ const char* path = std::getenv("EDEN_NX2_KEY_RECORD");
+ return path ? std::fopen(path, "w") : nullptr;
+ }();
+ static std::array<bool, 512> held{};
+ if (record && key >= 0 && key < static_cast<int>(held.size()) && held[key] != (state != 0)) {
+ held[key] = state != 0;
+ std::fprintf(record, "f%llu:%s%d,",
+ static_cast<unsigned long long>(Core::g_nx2_present_count.load()),
+ state != 0 ? "" : "-", key);
+ std::fflush(record);
+ }
if (state != 0) {
input_subsystem->GetKeyboard()->PressKey(static_cast<std::size_t>(key));
} else {
@@ -201,6 +227,7 @@ void EmuWindow_SDL3::OnEvent(SDL_Event& event) {
is_shown = true;
return OnResize();
case SDL_EVENT_WINDOW_CLOSE_REQUESTED:
+ LOG_INFO(Frontend, "Window close requested");
is_open = false;
return;
case SDL_EVENT_KEY_DOWN:
@@ -224,6 +251,7 @@ void EmuWindow_SDL3::OnEvent(SDL_Event& event) {
case SDL_EVENT_FINGER_UP:
return OnFingerUp();
case SDL_EVENT_QUIT:
+ LOG_INFO(Frontend, "Quit requested");
is_open = false;
return;
default:
diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3.h b/src/yuzu_cmd/emu_window/emu_window_sdl3.h
index ffabaeb143..0dc2a4983a 100644
--- a/src/yuzu_cmd/emu_window/emu_window_sdl3.h
+++ b/src/yuzu_cmd/emu_window/emu_window_sdl3.h
@@ -6,6 +6,7 @@
#pragma once
+#include <cstdlib>
#include <tuple>
#include <utility>
@@ -24,6 +25,14 @@ class InputSubsystem;
enum class MouseButton;
} // namespace InputCommon
+// EDEN_NX2_HIDE_WINDOW=1 (anything but 0) creates the render window hidden and without focus, for unattended
+// test runs that must not cover or interrupt the desktop.
+inline SDL_WindowFlags Nx2HiddenWindowFlags() {
+ const char* hide = std::getenv("EDEN_NX2_HIDE_WINDOW");
+ return hide != nullptr && hide[0] != '0' ? SDL_WINDOW_HIDDEN | SDL_WINDOW_NOT_FOCUSABLE
+ : SDL_WindowFlags{0};
+}
+
class EmuWindow_SDL3 : public Core::Frontend::EmuWindow {
public:
explicit EmuWindow_SDL3(InputCommon::InputSubsystem* input_subsystem_, Core::System& system_);
diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp
index e86858a232..38cc340426 100644
--- a/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp
+++ b/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp
@@ -101,7 +101,7 @@ EmuWindow_SDL3_GL::EmuWindow_SDL3_GL(InputCommon::InputSubsystem* input_subsyste
SDL_CreateWindow(window_title.c_str(), Layout::ScreenUndocked::Width,
Layout::ScreenUndocked::Height,
SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE |
- SDL_WINDOW_HIGH_PIXEL_DENSITY);
+ SDL_WINDOW_HIGH_PIXEL_DENSITY | Nx2HiddenWindowFlags());
if (render_window == nullptr) {
LOG_CRITICAL(Frontend, "Failed to create SDL3 window! {}", SDL_GetError());
diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp
index f5e4e19a30..45ff657f13 100644
--- a/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp
+++ b/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp
@@ -31,7 +31,7 @@ EmuWindow_SDL3_Null::EmuWindow_SDL3_Null(InputCommon::InputSubsystem* input_subs
render_window =
SDL_CreateWindow(window_title.c_str(), Layout::ScreenUndocked::Width,
Layout::ScreenUndocked::Height,
- SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY);
+ SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY | Nx2HiddenWindowFlags());
SetWindowIcon();
diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp
index 9019cbbba0..afbe67716f 100644
--- a/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp
+++ b/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp
@@ -29,7 +29,7 @@ EmuWindow_SDL3_VK::EmuWindow_SDL3_VK(InputCommon::InputSubsystem* input_subsyste
render_window =
SDL_CreateWindow(window_title.c_str(), Layout::ScreenUndocked::Width,
Layout::ScreenUndocked::Height,
- SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY);
+ SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY | Nx2HiddenWindowFlags());
const SDL_PropertiesID window_props = SDL_GetWindowProperties(render_window);
diff --git a/src/yuzu_cmd/nx2_sampler.cpp b/src/yuzu_cmd/nx2_sampler.cpp
new file mode 100644
index 0000000000..c01b812650
--- /dev/null
+++ b/src/yuzu_cmd/nx2_sampler.cpp
@@ -0,0 +1,333 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "yuzu_cmd/nx2_sampler.h"
+
+#ifdef _WIN32
+
+#include <algorithm>
+#include <array>
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <map>
+#include <set>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <vector>
+
+#include <windows.h>
+#include <tlhelp32.h>
+#include <dbghelp.h>
+
+namespace Nx2Sampler {
+namespace {
+
+constexpr size_t MaxDepth = 40;
+
+struct Sample {
+ unsigned thread;
+ unsigned depth;
+ std::array<DWORD64, MaxDepth> frames;
+};
+
+struct Target {
+ HANDLE handle;
+ std::string name;
+};
+
+double EnvSeconds(const char* name, double fallback) {
+ const char* text = std::getenv(name);
+ return text != nullptr ? std::strtod(text, nullptr) : fallback;
+}
+
+std::vector<std::string> SplitNames(const char* text) {
+ std::vector<std::string> names;
+ std::string current;
+ for (const char* c = text; *c != '\0'; ++c) {
+ if (*c == ',') {
+ names.push_back(current);
+ current.clear();
+ } else {
+ current += *c;
+ }
+ }
+ if (!current.empty()) {
+ names.push_back(current);
+ }
+ return names;
+}
+
+bool Matches(const std::string& thread_name, const std::vector<std::string>& patterns) {
+ for (const auto& pattern : patterns) {
+ if (!pattern.empty() && pattern.back() == '*') {
+ if (thread_name.compare(0, pattern.size() - 1, pattern, 0, pattern.size() - 1) == 0) {
+ return true;
+ }
+ } else if (thread_name == pattern) {
+ return true;
+ }
+ }
+ return false;
+}
+
+std::vector<Target> FindThreads(const std::vector<std::string>& patterns) {
+ using GetThreadDescriptionFn = HRESULT(WINAPI*)(HANDLE, PWSTR*);
+ const auto get_description = reinterpret_cast<GetThreadDescriptionFn>(
+ GetProcAddress(GetModuleHandleA("KernelBase.dll"), "GetThreadDescription"));
+ std::vector<Target> targets;
+ if (get_description == nullptr) {
+ return targets;
+ }
+ const HANDLE snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
+ if (snapshot == INVALID_HANDLE_VALUE) {
+ return targets;
+ }
+ THREADENTRY32 entry{};
+ entry.dwSize = sizeof(entry);
+ const DWORD self = GetCurrentThreadId();
+ for (BOOL ok = Thread32First(snapshot, &entry); ok; ok = Thread32Next(snapshot, &entry)) {
+ if (entry.th32OwnerProcessID != GetCurrentProcessId() || entry.th32ThreadID == self) {
+ continue;
+ }
+ const HANDLE handle = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT |
+ THREAD_QUERY_LIMITED_INFORMATION,
+ FALSE, entry.th32ThreadID);
+ if (handle == nullptr) {
+ continue;
+ }
+ PWSTR wide{};
+ std::string name;
+ if (SUCCEEDED(get_description(handle, &wide)) && wide != nullptr) {
+ char buffer[256]{};
+ WideCharToMultiByte(CP_UTF8, 0, wide, -1, buffer, sizeof(buffer) - 1, nullptr,
+ nullptr);
+ name = buffer;
+ LocalFree(wide);
+ }
+ if (Matches(name, patterns)) {
+ targets.push_back({handle, name});
+ } else {
+ CloseHandle(handle);
+ }
+ }
+ CloseHandle(snapshot);
+ return targets;
+}
+
+// Walks a suspended thread's stack from its context. Nothing here may allocate: the thread
+// could be paused holding the heap lock.
+unsigned Walk(CONTEXT context, std::array<DWORD64, MaxDepth>& frames) {
+ unsigned depth = 0;
+ while (depth < MaxDepth && context.Rip != 0) {
+ frames[depth++] = context.Rip;
+ DWORD64 image_base{};
+ const PRUNTIME_FUNCTION function = RtlLookupFunctionEntry(context.Rip, &image_base, nullptr);
+ if (function == nullptr) {
+ // A leaf function (no unwind data): its return address is on top of the stack. JIT
+ // code has none either; past the first frame, stop there.
+ if (depth != 1) {
+ break;
+ }
+ context.Rip = *reinterpret_cast<const DWORD64*>(context.Rsp);
+ context.Rsp += 8;
+ continue;
+ }
+ PVOID handler_data{};
+ DWORD64 establisher_frame{};
+ RtlVirtualUnwind(UNW_FLAG_NHANDLER, image_base, context.Rip, function, &context,
+ &handler_data, &establisher_frame, nullptr);
+ }
+ return depth;
+}
+
+struct Symbolizer {
+ using SymInitializeFn = BOOL(WINAPI*)(HANDLE, PCSTR, BOOL);
+ using SymSetOptionsFn = DWORD(WINAPI*)(DWORD);
+ using SymFromAddrFn = BOOL(WINAPI*)(HANDLE, DWORD64, PDWORD64, PSYMBOL_INFO);
+
+ Symbolizer() {
+ const HMODULE dbghelp = LoadLibraryA("dbghelp.dll");
+ if (dbghelp == nullptr) {
+ return;
+ }
+ const auto set_options =
+ reinterpret_cast<SymSetOptionsFn>(GetProcAddress(dbghelp, "SymSetOptions"));
+ const auto initialize =
+ reinterpret_cast<SymInitializeFn>(GetProcAddress(dbghelp, "SymInitialize"));
+ from_address = reinterpret_cast<SymFromAddrFn>(GetProcAddress(dbghelp, "SymFromAddr"));
+ if (set_options == nullptr || initialize == nullptr || from_address == nullptr) {
+ from_address = nullptr;
+ return;
+ }
+ set_options(SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS);
+ if (!initialize(GetCurrentProcess(), nullptr, TRUE)) {
+ from_address = nullptr;
+ }
+ }
+
+ const std::string& Name(DWORD64 address) {
+ const auto [it, is_new] = cache.try_emplace(address);
+ if (!is_new) {
+ return it->second;
+ }
+ if (from_address != nullptr) {
+ alignas(SYMBOL_INFO) char buffer[sizeof(SYMBOL_INFO) + 1024]{};
+ auto* const symbol = reinterpret_cast<SYMBOL_INFO*>(buffer);
+ symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+ symbol->MaxNameLen = 1024;
+ DWORD64 displacement{};
+ if (from_address(GetCurrentProcess(), address, &displacement, symbol)) {
+ it->second.assign(symbol->Name, std::min<size_t>(symbol->NameLen, 160));
+ return it->second;
+ }
+ }
+ HMODULE module{};
+ if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
+ GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+ reinterpret_cast<LPCSTR>(address), &module)) {
+ char path[MAX_PATH] = "?";
+ GetModuleFileNameA(module, path, sizeof(path));
+ const char* base = std::strrchr(path, '\\');
+ it->second = std::string{base != nullptr ? base + 1 : path} + "!?";
+ } else {
+ it->second = "(jit)";
+ }
+ return it->second;
+ }
+
+ SymFromAddrFn from_address{};
+ std::unordered_map<DWORD64, std::string> cache;
+};
+
+void Report(const std::vector<Target>& targets, const std::vector<Sample>& samples,
+ const char* path) {
+ Symbolizer symbolizer;
+ std::FILE* const file = std::fopen(path, "w");
+ if (file == nullptr) {
+ return;
+ }
+ for (unsigned thread = 0; thread < targets.size(); ++thread) {
+ std::map<std::string, unsigned> self;
+ std::map<std::string, unsigned> inclusive;
+ std::map<std::string, unsigned> self_caller;
+ unsigned total = 0;
+ for (const Sample& sample : samples) {
+ if (sample.thread != thread || sample.depth == 0) {
+ continue;
+ }
+ ++total;
+ const std::string& top = symbolizer.Name(sample.frames[0]);
+ ++self[top];
+ if (sample.depth > 1) {
+ ++self_caller[top + " <- " + symbolizer.Name(sample.frames[1])];
+ }
+ std::set<std::string> seen;
+ for (unsigned i = 0; i < sample.depth; ++i) {
+ const std::string& name = symbolizer.Name(sample.frames[i]);
+ if (seen.insert(name).second) {
+ ++inclusive[name];
+ }
+ }
+ }
+ const auto print = [&](const char* title, const std::map<std::string, unsigned>& counts,
+ size_t limit) {
+ std::vector<std::pair<unsigned, std::string>> sorted;
+ for (const auto& [name, count] : counts) {
+ sorted.emplace_back(count, name);
+ }
+ std::sort(sorted.rbegin(), sorted.rend());
+ std::fprintf(file, "-- %s\n", title);
+ for (size_t i = 0; i < std::min(limit, sorted.size()); ++i) {
+ std::fprintf(file, "%6u %5.1f%% %s\n", sorted[i].first,
+ 100.0 * sorted[i].first / std::max(total, 1U),
+ sorted[i].second.c_str());
+ }
+ };
+ std::fprintf(file, "=== thread \"%s\": %u samples\n", targets[thread].name.c_str(), total);
+ print("self (where the thread is)", self, 60);
+ print("self <- caller", self_caller, 50);
+ print("inclusive (on the stack)", inclusive, 90);
+ std::fprintf(file, "\n");
+ }
+ std::fclose(file);
+
+ // <out>.folded: every distinct stack with its count, root first ("thread;a;b;c count").
+ std::map<std::string, unsigned> folded;
+ for (const Sample& sample : samples) {
+ std::string line = targets[sample.thread].name;
+ for (unsigned i = sample.depth; i-- > 0;) {
+ line += ';';
+ line += symbolizer.Name(sample.frames[i]);
+ }
+ ++folded[line];
+ }
+ if (std::FILE* const out = std::fopen((std::string{path} + ".folded").c_str(), "w")) {
+ for (const auto& [line, count] : folded) {
+ std::fprintf(out, "%s %u\n", line.c_str(), count);
+ }
+ std::fclose(out);
+ }
+}
+
+void Run(std::vector<std::string> patterns, double start, double seconds, std::string out) {
+ std::this_thread::sleep_for(std::chrono::duration<double>(start));
+ const std::vector<Target> targets = FindThreads(patterns);
+ std::fprintf(stderr, "nx2 sampler: %zu threads match, sampling %.0f s\n", targets.size(),
+ seconds);
+ if (targets.empty()) {
+ return;
+ }
+ std::vector<Sample> samples;
+ samples.reserve(static_cast<size_t>(seconds * 1100.0) * targets.size() + 16);
+ const auto deadline = std::chrono::steady_clock::now() + std::chrono::duration<double>(seconds);
+ while (std::chrono::steady_clock::now() < deadline &&
+ samples.size() + targets.size() <= samples.capacity()) {
+ for (unsigned i = 0; i < targets.size(); ++i) {
+ const HANDLE handle = targets[i].handle;
+ if (SuspendThread(handle) == static_cast<DWORD>(-1)) {
+ continue;
+ }
+ CONTEXT context{};
+ context.ContextFlags = CONTEXT_FULL;
+ if (GetThreadContext(handle, &context)) {
+ Sample& sample = samples.emplace_back();
+ sample.thread = i;
+ sample.depth = Walk(context, sample.frames);
+ }
+ ResumeThread(handle);
+ }
+ Sleep(1);
+ }
+ Report(targets, samples, out.c_str());
+ std::fprintf(stderr, "nx2 sampler: %zu samples written to %s\n", samples.size(), out.c_str());
+ for (const auto& target : targets) {
+ CloseHandle(target.handle);
+ }
+}
+
+} // Anonymous namespace
+
+void StartFromEnvironment() {
+ const char* names = std::getenv("EDEN_NX2_SAMPLE");
+ if (names == nullptr || *names == '\0') {
+ return;
+ }
+ const char* out = std::getenv("EDEN_NX2_SAMPLE_OUT");
+ std::thread{Run, SplitNames(names), EnvSeconds("EDEN_NX2_SAMPLE_START", 60.0),
+ EnvSeconds("EDEN_NX2_SAMPLE_SECONDS", 30.0),
+ std::string{out != nullptr ? out : "nx2_samples.txt"}}
+ .detach();
+}
+
+} // namespace Nx2Sampler
+
+#else
+
+namespace Nx2Sampler {
+void StartFromEnvironment() {}
+} // namespace Nx2Sampler
+
+#endif
diff --git a/src/yuzu_cmd/nx2_sampler.h b/src/yuzu_cmd/nx2_sampler.h
new file mode 100644
index 0000000000..a73c8fa302
--- /dev/null
+++ b/src/yuzu_cmd/nx2_sampler.h
@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+namespace Nx2Sampler {
+
+// EDEN_NX2_SAMPLE=<thread names>: a sampling profiler for named host threads (Windows only).
+// Names are comma separated; a name ending in '*' matches by prefix (CPUCore_*). From
+// EDEN_NX2_SAMPLE_START seconds (default 60) for EDEN_NX2_SAMPLE_SECONDS (default 30), each
+// thread is paused about 1000 times a second and its call stack recorded; the counts per
+// function (where the thread is, and everything on its stack) go to EDEN_NX2_SAMPLE_OUT
+// (default nx2_samples.txt).
+void StartFromEnvironment();
+
+} // namespace Nx2Sampler
diff --git a/src/yuzu_cmd/yuzu.cpp b/src/yuzu_cmd/yuzu.cpp
index 473a5b1014..ffa1876c4f 100644
--- a/src/yuzu_cmd/yuzu.cpp
+++ b/src/yuzu_cmd/yuzu.cpp
@@ -4,6 +4,14 @@
// SPDX-FileCopyrightText: 2014 Citra Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <atomic>
+#include <iterator>
+#include <fstream>
+#include <vector>
+#include <cstdlib>
+#include <chrono>
+#include <cstdio>
+#include <cstring>
#include <iostream>
#include <memory>
#include <mutex>
@@ -33,6 +41,7 @@
#include "core/hle/service/filesystem/filesystem.h"
#include "core/loader/loader.h"
#include "frontend_common/config.h"
+#include "input_common/drivers/keyboard.h"
#include "input_common/main.h"
#include "network/network.h"
#include "sdl_config.h"
@@ -45,6 +54,7 @@
#endif
#include "yuzu_cmd/emu_window/emu_window_sdl3_null.h"
#include "yuzu_cmd/emu_window/emu_window_sdl3_vk.h"
+#include "yuzu_cmd/nx2_sampler.h"
#ifdef _WIN32
// windows.h needs to be included before shellapi.h
@@ -60,6 +70,75 @@ extern "C" {
__declspec(dllexport) unsigned long NvOptimusEnablement = 0x00000001;
__declspec(dllexport) int AmdPowerXpressRequestHighPerformance = 1;
}
+
+// EDEN_NX2_CRASH_DUMP=<dir>: on an unhandled exception (a real crash; fastmem faults are handled
+// before this), write <dir>/eden-crash-<pid>.dmp. A debugger changes the timing enough to hide
+// some crashes, so this catches them at full speed. dbghelp.dll is loaded only then.
+// Prints a code address as module+offset (resolve with llvm-symbolizer and the module's PDB).
+static void PrintCodeAddress(const char* label, const void* address) {
+ HMODULE module{};
+ char name[MAX_PATH] = "?";
+ if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
+ GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+ static_cast<LPCSTR>(address), &module)) {
+ GetModuleFileNameA(module, name, sizeof(name));
+ }
+ const char* base_name = std::strrchr(name, '\\');
+ std::fprintf(stderr, " %s %p %s+%#llx\n", label, address, base_name ? base_name + 1 : name,
+ static_cast<unsigned long long>(reinterpret_cast<uintptr_t>(address) -
+ reinterpret_cast<uintptr_t>(module)));
+}
+
+static LONG WINAPI WriteCrashDump(EXCEPTION_POINTERS* exception) {
+ static std::atomic<bool> written{};
+ const char* directory = std::getenv("EDEN_NX2_CRASH_DUMP");
+ if (directory == nullptr || written.exchange(true)) {
+ return EXCEPTION_CONTINUE_SEARCH;
+ }
+ // The faulting address and stack first: they survive even if the dump fails.
+ const EXCEPTION_RECORD& record = *exception->ExceptionRecord;
+ std::fprintf(stderr, "Unhandled exception %#lx on thread %lu, info %#llx %#llx\n",
+ record.ExceptionCode, GetCurrentThreadId(),
+ static_cast<unsigned long long>(record.NumberParameters > 0 ? record.ExceptionInformation[0] : 0),
+ static_cast<unsigned long long>(record.NumberParameters > 1 ? record.ExceptionInformation[1] : 0));
+ PrintCodeAddress("at", record.ExceptionAddress);
+ void* frames[48];
+ const USHORT count = RtlCaptureStackBackTrace(0, 48, frames, nullptr);
+ for (USHORT i = 0; i < count; ++i) {
+ PrintCodeAddress("frame", frames[i]);
+ }
+ std::fflush(stderr);
+ struct ExceptionInformation {
+ DWORD thread_id;
+ EXCEPTION_POINTERS* exception_pointers;
+ BOOL client_pointers;
+ };
+ using MiniDumpWriteDumpFn = BOOL(WINAPI*)(HANDLE, DWORD, HANDLE, int, const ExceptionInformation*,
+ const void*, const void*);
+ const HMODULE dbghelp = LoadLibraryA("dbghelp.dll");
+ const auto write = dbghelp ? reinterpret_cast<MiniDumpWriteDumpFn>(
+ GetProcAddress(dbghelp, "MiniDumpWriteDump"))
+ : nullptr;
+ if (write == nullptr) {
+ return EXCEPTION_CONTINUE_SEARCH;
+ }
+ char path[MAX_PATH];
+ std::snprintf(path, sizeof(path), "%s\\eden-crash-%lu.dmp", directory, GetCurrentProcessId());
+ const HANDLE file = CreateFileA(path, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS,
+ FILE_ATTRIBUTE_NORMAL, nullptr);
+ if (file != INVALID_HANDLE_VALUE) {
+ const ExceptionInformation information{GetCurrentThreadId(), exception, FALSE};
+ // MiniDumpWithThreadInfo; the larger MiniDumpWithIndirectlyReferencedMemory failed on
+ // Eden's address space and left an empty file.
+ const BOOL ok = write(GetCurrentProcess(), GetCurrentProcessId(), file, 0x1000,
+ &information, nullptr, nullptr);
+ const DWORD error = ok ? 0 : GetLastError();
+ CloseHandle(file);
+ std::fprintf(stderr, ok ? "Crash dump written to %s\n" : "Crash dump to %s failed (%#lx)\n",
+ path, error);
+ }
+ return EXCEPTION_CONTINUE_SEARCH;
+}
#endif
static void OnStateChanged(const Network::RoomMember::State& state) {
@@ -159,17 +238,30 @@ static void OnStatusMessageReceived(const Network::StatusMessageEntry& msg) {
struct SdlState {
Core::System system{};
+ // Must outlive emu_window, which keeps a pointer to it.
+ InputCommon::InputSubsystem input_subsystem{};
std::unique_ptr<EmuWindow_SDL3> emu_window;
};
extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) {
+#ifdef _WIN32
+ SetUnhandledExceptionFilter(WriteCrashDump);
+#endif
+ Nx2Sampler::StartFromEnvironment();
+ // SDL calls SDL_AppIterate back to back by default, which kept a host core at 100% polling
+ // input; 240 Hz is plenty for input and the key script.
+ SDL_SetHint(SDL_HINT_MAIN_CALLBACK_RATE, "240");
SdlState* state = new SdlState();
+ *appstate = state;
#ifdef _WIN32
if (AttachConsole(ATTACH_PARENT_PROCESS)) {
freopen("CONOUT$", "wb", stdout);
freopen("CONOUT$", "wb", stderr);
}
+ // Raise the CRT's open-file limit like the Qt frontend: loose RomFS folders keep thousands of
+ // files open through the VFS cache.
+ _setmaxstdio(8192);
#endif
Common::Log::Initialize();
@@ -202,7 +294,6 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) {
state->system.Initialize();
- InputCommon::InputSubsystem input_subsystem{};
// Apply the command line arguments
state->system.ApplySettings();
@@ -212,14 +303,14 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) {
case Settings::RendererBackend::OpenGL_GLSL:
case Settings::RendererBackend::OpenGL_GLASM:
case Settings::RendererBackend::OpenGL_SPIRV:
- state->emu_window = std::make_unique<EmuWindow_SDL3_GL>(&input_subsystem, state->system, lp.fullscreen);
+ state->emu_window = std::make_unique<EmuWindow_SDL3_GL>(&state->input_subsystem, state->system, lp.fullscreen);
break;
#endif
case Settings::RendererBackend::Vulkan:
- state->emu_window = std::make_unique<EmuWindow_SDL3_VK>(&input_subsystem, state->system, lp.fullscreen);
+ state->emu_window = std::make_unique<EmuWindow_SDL3_VK>(&state->input_subsystem, state->system, lp.fullscreen);
break;
case Settings::RendererBackend::Null:
- state->emu_window = std::make_unique<EmuWindow_SDL3_Null>(&input_subsystem, state->system, lp.fullscreen);
+ state->emu_window = std::make_unique<EmuWindow_SDL3_Null>(&state->input_subsystem, state->system, lp.fullscreen);
break;
default:
LOG_CRITICAL(Frontend, "Invalid renderer backend");
@@ -319,16 +410,73 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) {
void(state->system.Run());
if (state->system.DebuggerEnabled())
state->system.InitializeDebugger();
- return SDL_APP_SUCCESS;
+ return SDL_APP_CONTINUE;
}
+// EDEN_NX2_KEY_SCRIPT="seconds:code,seconds:-code,..." (or "@<file>"): press (code) or release (-code) keyboard
+// keys, by SDL scancode, that many seconds after start. Lets hidden test runs drive menus.
+double Nx2KeyClockSeconds(); // emu_window_sdl3.cpp
+
+static void RunKeyScript(InputCommon::InputSubsystem& input_subsystem) {
+ struct KeyEvent {
+ double seconds; // when frame is 0
+ u64 frame; // "f<frame>:code": after that many presents
+ int code;
+ };
+ static const std::vector<KeyEvent> script = [] {
+ std::vector<KeyEvent> events;
+ if (const char* text = std::getenv("EDEN_NX2_KEY_SCRIPT")) {
+ // "@<file>" reads the list from a file: long scripts in the environment slow down
+ // every getenv in the process.
+ std::string list{text};
+ if (!list.empty() && list[0] == '@') {
+ std::ifstream file{list.substr(1)};
+ list.assign(std::istreambuf_iterator<char>{file}, {});
+ }
+ size_t start = 0;
+ while (start < list.size()) {
+ const size_t end = std::min(list.find(',', start), list.size());
+ const std::string item = list.substr(start, end - start);
+ if (const size_t colon = item.find(':'); colon != std::string::npos) {
+ const std::string when = item.substr(0, colon);
+ const int code = std::stoi(item.substr(colon + 1));
+ if (!when.empty() && when[0] == 'f') {
+ events.push_back({0.0, std::stoull(when.substr(1)), code});
+ } else {
+ events.push_back({std::stod(when), 0, code});
+ }
+ }
+ start = end + 1;
+ }
+ }
+ return events;
+ }();
+ static size_t next = 0;
+ const double elapsed = Nx2KeyClockSeconds();
+ const u64 frame = Core::g_nx2_present_count.load();
+ while (next < script.size() && (script[next].frame != 0 ? script[next].frame <= frame
+ : script[next].seconds <= elapsed)) {
+ const int code = script[next].code;
+ LOG_INFO(Frontend, "Key script: {} key {} at {:.1f} s (frame {})",
+ code < 0 ? "release" : "press", code < 0 ? -code : code, elapsed, frame);
+ if (code < 0) {
+ input_subsystem.GetKeyboard()->ReleaseKey(static_cast<std::size_t>(-code));
+ } else {
+ input_subsystem.GetKeyboard()->PressKey(static_cast<std::size_t>(code));
+ }
+ ++next;
+ }
+}
+
extern "C" SDL_AppResult SDL_AppIterate(void *appstate) {
SdlState *state = (SdlState *)appstate;
- return state->emu_window->IsOpen() ? SDL_APP_CONTINUE : SDL_APP_SUCCESS;
+ RunKeyScript(state->input_subsystem);
+ return state->emu_window && state->emu_window->IsOpen() ? SDL_APP_CONTINUE : SDL_APP_SUCCESS;
}
extern "C" SDL_AppResult SDL_AppEvent(void *appstate, SDL_Event *event) {
SdlState *state = (SdlState *)appstate;
- state->emu_window->OnEvent(*event);
- return SDL_APP_SUCCESS;
+ if (state->emu_window)
+ state->emu_window->OnEvent(*event);
+ return SDL_APP_CONTINUE;
}
extern "C" void SDL_AppQuit(void *appstate, SDL_AppResult result) {
SdlState *state = (SdlState *)appstate;