diff options
| author | Takoroka <137927095+Takoroka@users.noreply.github.com> | 2026-10-29 03:39:52 -0400 |
|---|---|---|
| committer | Takoroka <137927095+Takoroka@users.noreply.github.com> | 2026-10-29 03:39:52 -0400 |
| commit | e17fd9cef116a088f080ca502951e500864a58b6 (patch) | |
| tree | e7ff65f614622515d6a2e4263521909eea89a5d8 /src | |
| parent | ea6c1193fbfa95e586bb6d129dc684163a12cb3e (diff) | |
| parent | 80c79333fe6fd28a31b65b997f1cbf2987711c91 (diff) | |
Merge Switch 2 work from local Eden (nx2 at b0dd3fe)
Diffstat (limited to 'src')
177 files changed, 11720 insertions, 297 deletions
diff --git a/src/audio_core/renderer/behavior/info_updater.cpp b/src/audio_core/renderer/behavior/info_updater.cpp index a88eae91ad..ca3eaece94 100644 --- a/src/audio_core/renderer/behavior/info_updater.cpp +++ b/src/audio_core/renderer/behavior/info_updater.cpp @@ -588,7 +588,11 @@ Result InfoUpdater::UpdateBehaviorInfo(BehaviorInfo& behaviour_) { return Service::Audio::ResultInvalidUpdateInfo; } - if (in_params->revision != behaviour_.GetUserRevision()) { + // Compare revision numbers: Switch 2 opens the renderer with 'REV?' but tags its updates + // 'REv?' (lowercase v) for the same revision. + if (GetRevisionNum(in_params->revision) != GetRevisionNum(behaviour_.GetUserRevision())) { + LOG_ERROR(Service_Audio, "BehaviorInfo revision {:08X} does not match the renderer's {:08X}", + in_params->revision, behaviour_.GetUserRevision()); return Service::Audio::ResultInvalidUpdateInfo; } diff --git a/src/audio_core/sink/sink_stream.cpp b/src/audio_core/sink/sink_stream.cpp index c70020e9da..9dfd3bf747 100644 --- a/src/audio_core/sink/sink_stream.cpp +++ b/src/audio_core/sink/sink_stream.cpp @@ -4,8 +4,10 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <algorithm> #include <array> #include <atomic> +#include <cstdlib> #include <memory> #include <span> #include <vector> @@ -213,6 +215,19 @@ void SinkStream::ProcessAudioOutAndRender(std::span<s16> output_buffer, std::siz std::memcpy(last_frame.data(), &output_buffer[(frames_written - 1) * frame_size], frame_size_bytes); + // EDEN_NX2_AUDIO_LEVEL: log the output peak about every two seconds (is a game audible?). + if (static const bool meter = std::getenv("EDEN_NX2_AUDIO_LEVEL") != nullptr; meter) { + for (size_t i = 0; i < num_frames * frame_size; ++i) { + level_peak = std::max<s32>(level_peak, std::abs(static_cast<s32>(output_buffer[i]))); + } + level_frames += num_frames; + if (level_frames >= TargetSampleRate * 2) { + LOG_INFO(Audio, "Audio output peak over {} frames: {} of 32767", level_frames, level_peak); + level_frames = 0; + level_peak = 0; + } + } + { std::scoped_lock lk{sample_count_lock}; last_sample_count_update_time = system.CoreTiming().GetGlobalTimeNs(); diff --git a/src/audio_core/sink/sink_stream.h b/src/audio_core/sink/sink_stream.h index acb7537dda..280b7747ec 100644 --- a/src/audio_core/sink/sink_stream.h +++ b/src/audio_core/sink/sink_stream.h @@ -245,6 +245,9 @@ private: SinkBuffer playing_buffer{}; /// The last played (or received) frame of audio, used when the callback underruns std::array<s16, MaxChannels> last_frame{}; + /// EDEN_NX2_AUDIO_LEVEL meter state. + s32 level_peak{}; + u64 level_frames{}; /// Number of buffers waiting to be played std::atomic<u32> queued_buffers{}; /// The ring size for audio out buffers (usually 4, rarely 2 or 8) diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 5d1b30a549..d3e7b03cd2 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -96,6 +96,7 @@ add_library( ${CMAKE_CURRENT_BINARY_DIR}/scm_rev.cpp scm_rev.h scope_exit.h + spin_recursive_mutex.h scratch_buffer.h settings.cpp settings.h @@ -151,7 +152,7 @@ add_library( set_source_files_properties(zbic_compression.cpp PROPERTIES INCLUDE_DIRECTORIES "${ZBIC_INCLUDE_DIR}" - COMPILE_OPTIONS "$<$<CXX_COMPILER_ID:Clang,GNU>:-Wno-unused-function;-Wno-missing-declarations;-Wno-shadow>") + COMPILE_OPTIONS "$<$<CXX_COMPILER_ID:Clang,GNU>:-Wno-unused-function;-Wno-missing-declarations;-Wno-shadow>;$<$<CXX_COMPILER_ID:MSVC>:/wd4505;/wd5245>") if(WIN32) target_sources(common PRIVATE windows/timer_resolution.cpp diff --git a/src/common/cpu_features.cpp b/src/common/cpu_features.cpp index 8ccb6847f7..52fd003c44 100644 --- a/src/common/cpu_features.cpp +++ b/src/common/cpu_features.cpp @@ -225,6 +225,13 @@ const CPUCaps g_cpu_caps = [] { } } + // AMD CPUs have no leaf 0x15 (Zen 4 stops at 0x10): measure their invariant TSC too, or every + // guest clock read (CNTPCT, which games read thousands of times a frame) goes through the + // host's much slower QueryPerformanceCounter. + if (caps.invariant_tsc && caps.tsc_frequency == 0) { + caps.tsc_frequency = X64::EstimateRDTSCFrequency(); + } + if (max_std_fn >= 0x16) { __cpuid(cpu_id, 0x16); caps.base_frequency = cpu_id[0]; diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp index eee8ff6d18..b96340999d 100644 --- a/src/common/sparse_large_vector.cpp +++ b/src/common/sparse_large_vector.cpp @@ -26,7 +26,7 @@ static std::mutex vector_regions_mutex {}; // Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { DWORD code = info->ExceptionRecord->ExceptionCode; - u64 exception_addr = reinterpret_cast<u64>(info->ExceptionRecord->ExceptionInformation[1]); + u64 exception_addr = static_cast<u64>(info->ExceptionRecord->ExceptionInformation[1]); if (code != EXCEPTION_ACCESS_VIOLATION || info->ExceptionRecord->ExceptionInformation[0] == 1) { // Not our problem diff --git a/src/common/spin_recursive_mutex.h b/src/common/spin_recursive_mutex.h new file mode 100644 index 0000000000..291dd4e41e --- /dev/null +++ b/src/common/spin_recursive_mutex.h @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <atomic> +#include <cstdint> +#include <thread> + +#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) +#include <immintrin.h> +#endif + +namespace Common { + +/// A recursive mutex that spins briefly before sleeping. The GPU caches' locks are taken tens of +/// thousands of times a second by the GPU thread, the fencing thread and CPU write traps, and +/// held for microseconds: std::recursive_mutex put a waiting thread to sleep at once, and its +/// wake-up took longer than the critical section (title 1's GPU thread spent ~8% asleep there). +class SpinRecursiveMutex { +public: + void lock() { + const std::thread::id self = std::this_thread::get_id(); + if (owner.load(std::memory_order_relaxed) == self) { + ++depth; + return; + } + std::uint32_t expected = 0; + if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + LockContended(); + } + owner.store(self, std::memory_order_relaxed); + depth = 1; + } + + bool try_lock() { + const std::thread::id self = std::this_thread::get_id(); + if (owner.load(std::memory_order_relaxed) == self) { + ++depth; + return true; + } + std::uint32_t expected = 0; + if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + return false; + } + owner.store(self, std::memory_order_relaxed); + depth = 1; + return true; + } + + void unlock() { + if (--depth != 0) { + return; + } + owner.store(std::thread::id{}, std::memory_order_relaxed); + if (state.exchange(0, std::memory_order_release) == 2) { + state.notify_one(); + } + } + +private: + // About 10 microseconds of spinning before sleeping. + static constexpr int SpinCount = 1024; + + static void Pause() { +#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) + _mm_pause(); +#elif defined(__aarch64__) + asm volatile("yield"); +#endif + } + + void LockContended() { + for (int spin = 0; spin < SpinCount; ++spin) { + Pause(); + if (state.load(std::memory_order_relaxed) == 0) { + std::uint32_t expected = 0; + if (state.compare_exchange_weak(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + return; + } + } + } + // 2: held with (possible) sleepers; taken this way it stays marked, so its unlock wakes + // the next sleeper. + while (state.exchange(2, std::memory_order_acquire) != 0) { + state.wait(2, std::memory_order_relaxed); + } + } + + std::atomic<std::uint32_t> state{0}; ///< 0 free, 1 held, 2 held with sleepers + std::atomic<std::thread::id> owner{}; + std::uint32_t depth{}; ///< Recursion depth, touched only by the owner +}; + +} // namespace Common diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 1b578f67ba..56fa653738 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -457,6 +457,7 @@ add_library(core STATIC hle/service/am/service/application_functions.cpp hle/service/am/service/application_functions.h hle/service/am/service/application_proxy.cpp + hle/service/am/service/graphics_scale_controller.h hle/service/am/service/application_proxy.h hle/service/am/service/application_proxy_service.cpp hle/service/am/service/application_proxy_service.h @@ -526,6 +527,7 @@ add_library(core STATIC hle/service/audio/audio_out.h hle/service/audio/audio_renderer_manager.cpp hle/service/audio/audio_renderer_manager.h + hle/service/audio/nx2_audio_user_client.h hle/service/audio/audio_renderer.cpp hle/service/audio/audio_renderer.h hle/service/audio/audio.cpp @@ -865,6 +867,7 @@ add_library(core STATIC hle/service/nvdrv/devices/nvhost_vic.h hle/service/nvdrv/devices/nvmap.cpp hle/service/nvdrv/devices/nvmap.h + hle/service/nvdrv/devices/nx2_stub_device.h hle/service/nvdrv/nvdata.h hle/service/nvdrv/nvdrv.cpp hle/service/nvdrv/nvdrv.h @@ -1033,6 +1036,8 @@ add_library(core STATIC hle/service/server_manager.h hle/service/service.cpp hle/service/service.h + hle/service/nx2_stub_interface.h + hle/service/nx2_gsv.h hle/service/services.cpp hle/service/services.h hle/service/set/factory_settings_server.cpp diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index 894857a86d..c2bb05afba 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -4,6 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <atomic> #include "common/settings.h" #include "core/arm/dynarmic/arm_dynarmic.h" #include "core/arm/dynarmic/arm_dynarmic_64.h" @@ -24,13 +25,24 @@ DynarmicCallbacks64::DynarmicCallbacks64(ArmDynarmic64& parent, Kernel::KProcess u64 DynarmicCallbacks64::MemoryRead(u64 vaddr, size_t size) { CheckMemoryAccess(vaddr, size, Kernel::DebugWatchpointType::Read); + u64 value{}; switch (size) { - case sizeof(u64): return m_memory.Read64(vaddr); - case sizeof(u32): return m_memory.Read32(vaddr); - case sizeof(u16): return m_memory.Read16(vaddr); - case sizeof(u8): return m_memory.Read8(vaddr); + case sizeof(u64): value = m_memory.Read64(vaddr); break; + case sizeof(u32): value = m_memory.Read32(vaddr); break; + case sizeof(u16): value = m_memory.Read16(vaddr); break; + case sizeof(u8): value = m_memory.Read8(vaddr); break; default: UNREACHABLE(); } + // With memory aborts ignored the guest keeps running on zeros: say where it read from. + if (auto& unmapped = Core::Memory::UnmappedReadFlag(); unmapped != 0) { + unmapped = 0; + static std::atomic<u32> logged{}; + if (logged.fetch_add(1) < 32) { + LOG_ERROR(Core_ARM, "Unmapped read {:#x} ({} bytes) at pc {:#x} lr {:#x} sp {:#x}", vaddr, size, + m_parent.m_jit->GetPC(), m_parent.m_jit->GetRegister(30), m_parent.m_jit->GetSP()); + } + } + return value; } Dynarmic::A64::Vector DynarmicCallbacks64::MemoryRead128(u64 vaddr) { CheckMemoryAccess(vaddr, 16, Kernel::DebugWatchpointType::Read); diff --git a/src/core/core.cpp b/src/core/core.cpp index 06543b0833..eee91ef4fc 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -68,6 +68,8 @@ namespace Core { +std::atomic<u64> g_nx2_present_count{}; + FileSys::VirtualFile GetGameFileFromPath(const FileSys::VirtualFilesystem& vfs, const std::string& path) { // To account for split 00+01+etc files. diff --git a/src/core/core.h b/src/core/core.h index d88ed64151..0276679d76 100644 --- a/src/core/core.h +++ b/src/core/core.h @@ -6,6 +6,7 @@ #pragma once +#include <atomic> #include <cstddef> #include <deque> #include <functional> @@ -115,6 +116,9 @@ class RenderdocAPI; namespace Core { +/// Frames the guest has presented (Switch 2 vi2 present calls); input replay is locked to it. +extern std::atomic<u64> g_nx2_present_count; + class CpuManager; class Debugger; class DeviceMemory; diff --git a/src/core/debugger/gdbstub.cpp b/src/core/debugger/gdbstub.cpp index 44ddf11c39..71af2acee3 100644 --- a/src/core/debugger/gdbstub.cpp +++ b/src/core/debugger/gdbstub.cpp @@ -25,7 +25,10 @@ #include "core/debugger/gdbstub_arch.h" #include "core/hle/kernel/k_page_table.h" #include "core/hle/kernel/k_process.h" +#include "core/hle/kernel/k_event.h" +#include "core/hle/kernel/k_readable_event.h" #include "core/hle/kernel/k_thread.h" +#include "core/hle/service/kernel_helpers.h" #include "core/loader/loader.h" #include "core/memory.h" @@ -731,8 +734,26 @@ void GDBStub::HandleRcmd(const std::vector<u8>& command) { break; cur_addr = next_address; } + } else if (command_str == "handles" || command_str == "get handles") { + // Every handle of the process: its object type and, for service events, their name. + reply = "Handles:\n"; + auto& handle_table = debug_process->GetHandleTable(); + for (size_t i = 0; i < handle_table.GetTableSize(); ++i) { + Kernel::Handle handle{}; + auto object = handle_table.GetObjectByIndex(system.Kernel(), std::addressof(handle), i); + if (object.IsNull()) { + continue; + } + std::string name; + if (auto* readable = object->DynamicCast<Kernel::KReadableEvent*>(); readable != nullptr) { + name = Service::KernelHelpers::ServiceContext::GetEventName(readable->GetParent()); + } else if (auto* event = object->DynamicCast<Kernel::KEvent*>(); event != nullptr) { + name = Service::KernelHelpers::ServiceContext::GetEventName(event); + } + reply += fmt::format(" {:#010x} {} {}\n", handle, object->GetTypeObj().GetName(), name); + } } else { - reply += "Commands: fastmem, info, mappings\n"; + reply += "Commands: fastmem, info, mappings, handles\n"; } std::span<const u8> reply_span{reinterpret_cast<u8*>(&reply.front()), reply.size()}; diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index b4b3b46088..9ed0b6f5e5 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -139,7 +139,7 @@ private: static constexpr size_t device_virtual_bits = Traits::device_virtual_bits; static constexpr size_t device_as_size = 1ULL << device_virtual_bits; static constexpr size_t physical_min_bits = 32; - static constexpr size_t physical_max_bits = 33; + static constexpr size_t physical_max_bits = 35; // covers the 12 GiB layout plus nx2 graphics server memory static constexpr size_t page_bits = 12; static constexpr size_t page_size = 1ULL << page_bits; static constexpr size_t page_mask = page_size - 1ULL; diff --git a/src/core/file_sys/program_metadata.cpp b/src/core/file_sys/program_metadata.cpp index 63ca9d3fe3..464e496ea3 100644 --- a/src/core/file_sys/program_metadata.cpp +++ b/src/core/file_sys/program_metadata.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <cstddef> +#include <cstring> #include <vector> #include "common/logging.h" @@ -33,13 +34,30 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) { return Loader::ResultStatus::ErrorBadACIDHeader; } + // Switch 2 ACID blocks have a 0x300-byte signature/key area instead of 0x200, so the + // header starts 0x100 bytes later. Offsets inside the header are unchanged. + if (std::memcmp(acid_header.magic.data(), "ACID", 4) != 0) { + constexpr std::size_t acid_sig_size_v1 = 0x200; + constexpr std::size_t acid_sig_size_v2 = 0x300; + constexpr std::size_t acid_body_size = sizeof(AcidHeader) - acid_sig_size_v1; + auto* const body = reinterpret_cast<u8*>(&acid_header) + acid_sig_size_v1; + if (file->ReadBytes(body, acid_body_size, npdm_header.acid_offset + acid_sig_size_v2) != + acid_body_size || + std::memcmp(acid_header.magic.data(), "ACID", 4) != 0) { + return Loader::ResultStatus::ErrorBadACIDHeader; + } + LOG_INFO(Loader, "NPDM uses the Switch 2 ACID layout"); + is_switch2 = true; + } + if (sizeof(AciHeader) != file->ReadObject(&aci_header, npdm_header.aci_offset)) { return Loader::ResultStatus::ErrorBadACIHeader; } // Load acid_file_access per-component instead of the entire struct, since this struct does not // reflect the layout of the real data. - std::size_t current_offset = acid_header.fac_offset; + // FAC/FAH offsets are relative to the start of the ACID/ACI block, like the KAC offsets. + std::size_t current_offset = npdm_header.acid_offset + acid_header.fac_offset; if (sizeof(FileAccessControl::version) != file->ReadBytes(&acid_file_access.version, sizeof(FileAccessControl::version), current_offset)) { @@ -57,7 +75,7 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) { } // Load aci_file_access per-component instead of the entire struct, same as acid_file_access - current_offset = aci_header.fah_offset; + current_offset = npdm_header.aci_offset + aci_header.fah_offset; if (sizeof(FileAccessHeader::version) != file->ReadBytes(&aci_file_access.version, sizeof(FileAccessHeader::version), current_offset)) { diff --git a/src/core/file_sys/program_metadata.h b/src/core/file_sys/program_metadata.h index 115e6d6cd1..5ce32af1a4 100644 --- a/src/core/file_sys/program_metadata.h +++ b/src/core/file_sys/program_metadata.h @@ -72,6 +72,10 @@ public: KernelCapabilityDescriptors capabilities); bool Is64BitProgram() const; + /// True when the NPDM uses the Switch 2 ACID layout. + bool IsSwitch2() const { + return is_switch2; + } ProgramAddressSpaceType GetAddressSpaceType() const; u8 GetMainThreadPriority() const; u8 GetMainThreadCore() const; @@ -179,6 +183,7 @@ private: Header npdm_header{}; AciHeader aci_header{}; AcidHeader acid_header{}; + bool is_switch2{}; FileAccessControl acid_file_access{}; FileAccessHeader aci_file_access{}; diff --git a/src/core/file_sys/vfs/vfs_real.cpp b/src/core/file_sys/vfs/vfs_real.cpp index 14e0caad3c..fd36752fee 100644 --- a/src/core/file_sys/vfs/vfs_real.cpp +++ b/src/core/file_sys/vfs/vfs_real.cpp @@ -19,6 +19,10 @@ // For FileTimeStampRaw #include <sys/stat.h> +#ifdef _WIN32 +#include <cstdio> // _getmaxstdio +#endif + #ifdef _MSC_VER #define stat _stat64 #endif @@ -33,7 +37,17 @@ namespace FS = Common::FS; namespace { -constexpr size_t MaxOpenFiles = 8192; +// Files the cache keeps open. On Windows it has to stay below the C runtime's open-file limit with +// room to spare: at the limit every other open fails (saves, shader caches, and the cache's own +// next file, which it only evicts for once it reaches its cap). +size_t MaxOpenFiles() { +#ifdef _WIN32 + static const size_t limit = static_cast<size_t>((std::max)(_getmaxstdio() - 512, 64)); + return (std::min)(size_t{8192}, limit); +#else + return 8192; +#endif +} bool IsWithinRoot(std::string_view root, std::string_view full_path) { if (root.empty()) @@ -251,7 +265,7 @@ void RealVfsFilesystem::DropReference(std::unique_ptr<FileReference>&& reference } void RealVfsFilesystem::EvictSingleReferenceLocked() { - if (num_open_files < MaxOpenFiles || open_references.empty()) { + if (num_open_files < MaxOpenFiles() || open_references.empty()) { return; } diff --git a/src/core/hardware_properties.h b/src/core/hardware_properties.h index 3f870becbf..aeea5f858f 100644 --- a/src/core/hardware_properties.h +++ b/src/core/hardware_properties.h @@ -15,11 +15,14 @@ namespace Hardware { constexpr u64 BASE_CLOCK_RATE = 1'020'000'000; // Default CPU Frequency = 1020 MHz constexpr u64 CNTFREQ = 19'200'000; // CNTPCT_EL0 Frequency = 19.2 MHz -constexpr u32 NUM_CPU_CORES = 4; // Number of CPU Cores - sync with dynarmic exclusive_monitor.h +// Switch 1 applications get cores 0..3. Switch 2 applications ask for 0..5 (title 4's npdm), and +// its job system starts a worker per core, so emulate six: with four, cores 0 and 1 each ran two +// of its workers. Switch 1 titles leave cores 4 and 5 idle. +constexpr u32 NUM_CPU_CORES = 6; // Number of CPU Cores - sync with dynarmic exclusive_monitor.h // Virtual to Physical core map. constexpr std::array<s32, Common::BitSize<u64>()> VirtualToPhysicalCoreMap{ - 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, }; diff --git a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp index 3343d1d282..ba4737e25f 100644 --- a/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp +++ b/src/core/hle/kernel/board/nintendo/nx/k_system_control.cpp @@ -94,7 +94,7 @@ size_t KSystemControl::Init::GetIntendedMemorySize() { case Smc::MemorySize_10GB: return 10_GiB; case Smc::MemorySize_12GB: - return 12_GiB; + return 12_GiB + KSystemControl::Nx2GraphicsServerMemorySize; } } diff --git a/src/core/hle/kernel/board/nintendo/nx/k_system_control.h b/src/core/hle/kernel/board/nintendo/nx/k_system_control.h index 41a25ba1c8..cc5572bde8 100644 --- a/src/core/hle/kernel/board/nintendo/nx/k_system_control.h +++ b/src/core/hle/kernel/board/nintendo/nx/k_system_control.h @@ -21,6 +21,10 @@ public: // This can be overridden as needed. static constexpr size_t SecureAppletMemorySize = 4 * 1024 * 1024; // 4_MB + // Extra DRAM on the 12 GB layout for memory the Switch 2 graphics server keeps itself (nx2:gpu + // 214 GPU-only regions). It sits in the non-secure system pool, outside the application pool. + static constexpr size_t Nx2GraphicsServerMemorySize = 8ULL * 1024 * 1024 * 1024; // 8_GiB + public: class Init { public: diff --git a/src/core/hle/kernel/global_scheduler_context.cpp b/src/core/hle/kernel/global_scheduler_context.cpp index 6d1cd030e0..98d14c8549 100644 --- a/src/core/hle/kernel/global_scheduler_context.cpp +++ b/src/core/hle/kernel/global_scheduler_context.cpp @@ -40,12 +40,15 @@ void GlobalSchedulerContext::RemoveThread(KThread* thread) noexcept { /// @note This operation happens every 10ms. void GlobalSchedulerContext::PreemptThreads(KernelCore& kernel) noexcept { // The priority levels at which the global scheduler preempts threads every 10 ms. They are - // ordered from Core 0 to Core 3. + // ordered from Core 0 to Core 5; core 3 is shared with the system, 4 and 5 are application + // cores like 0..2. static constexpr std::array<u32, Core::Hardware::NUM_CPU_CORES> per_core{ 59, 59, 59, 63, + 59, + 59, }; ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); for (u32 core_id = 0; core_id < per_core.size(); core_id++) diff --git a/src/core/hle/kernel/k_memory_block.h b/src/core/hle/kernel/k_memory_block.h index f6543bdb3a..2abca31d87 100644 --- a/src/core/hle/kernel/k_memory_block.h +++ b/src/core/hle/kernel/k_memory_block.h @@ -195,7 +195,9 @@ enum class KMemoryAttribute : u8 { Uncached = static_cast<u8>(Svc::MemoryAttribute::Uncached), PermissionLocked = static_cast<u8>(Svc::MemoryAttribute::PermissionLocked), - SetMask = Uncached | PermissionLocked, + GpuAccessible = static_cast<u8>(Svc::MemoryAttribute::GpuAccessible), + + SetMask = Uncached | PermissionLocked | GpuAccessible, }; DECLARE_ENUM_FLAG_OPERATORS(KMemoryAttribute); diff --git a/src/core/hle/kernel/k_page_table_base.cpp b/src/core/hle/kernel/k_page_table_base.cpp index 2a0a7d49d5..fd5d2c2dce 100644 --- a/src/core/hle/kernel/k_page_table_base.cpp +++ b/src/core/hle/kernel/k_page_table_base.cpp @@ -2102,7 +2102,8 @@ Result KPageTableBase::SetMemoryAttribute(KProcessAddress addr, size_t size, KMe constexpr KMemoryAttribute AttributeTestMask = ~(KMemoryAttribute::SetMask | KMemoryAttribute::DeviceShared); const KMemoryState state_test_mask = - (True(mask & KMemoryAttribute::Uncached) ? KMemoryState::FlagCanChangeAttribute + (True(mask & (KMemoryAttribute::Uncached | KMemoryAttribute::GpuAccessible)) + ? KMemoryState::FlagCanChangeAttribute : KMemoryState::None) | (True(mask & KMemoryAttribute::PermissionLocked) ? KMemoryState::FlagCanPermissionLock : KMemoryState::None); diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp index 27d1b1bfc4..4aac818b77 100644 --- a/src/core/hle/kernel/k_process.cpp +++ b/src/core/hle/kernel/k_process.cpp @@ -1261,6 +1261,13 @@ Result KProcess::LoadFromMetadata(KernelCore& kernel, const FileSys::ProgramMeta // Assign remaining properties. m_ideal_core_id = metadata.GetMainThreadCore(); + // Switch 2 applications get more memory than a 64-bit heap region holds (nninitStartup sizes + // its heap as total memory minus ~6.3 GiB), so count the alias region as well, bounded by the + // pool through the resource limit. + if (metadata.IsSwitch2()) { + m_max_process_memory = m_page_table.GetHeapRegionSize() + m_page_table.GetAliasRegionSize(); + } + // Set up emulation context. this->InitializeInterfaces(kernel); diff --git a/src/core/hle/kernel/k_spin_lock.cpp b/src/core/hle/kernel/k_spin_lock.cpp index 8525320374..107d6f1aa3 100644 --- a/src/core/hle/kernel/k_spin_lock.cpp +++ b/src/core/hle/kernel/k_spin_lock.cpp @@ -3,18 +3,60 @@ #include "core/hle/kernel/k_spin_lock.h" +#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) +#include <immintrin.h> +#endif + namespace Kernel { +namespace { + +// Spins of about 30-140 cycles each before a contended Lock sleeps (a few microseconds). +constexpr int SpinCount = 128; + +void Pause() { +#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) + _mm_pause(); +#elif defined(__aarch64__) + asm volatile("yield"); +#endif +} + +} // Anonymous namespace + void KSpinLock::Lock() { - m_lock.lock(); + u32 expected = 0; + if (m_state.compare_exchange_strong(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + return; + } + for (int spin = 0; spin < SpinCount; ++spin) { + Pause(); + if (m_state.load(std::memory_order_relaxed) == 0) { + expected = 0; + if (m_state.compare_exchange_weak(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + return; + } + } + } + // Mark the lock contended and sleep until it is released; taken this way it stays marked, + // so its Unlock wakes the next sleeper. + while (m_state.exchange(2, std::memory_order_acquire) != 0) { + m_state.wait(2, std::memory_order_relaxed); + } } void KSpinLock::Unlock() { - m_lock.unlock(); + if (m_state.exchange(0, std::memory_order_release) == 2) { + m_state.notify_one(); + } } bool KSpinLock::TryLock() { - return m_lock.try_lock(); + u32 expected = 0; + return m_state.compare_exchange_strong(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed); } } // namespace Kernel diff --git a/src/core/hle/kernel/k_spin_lock.h b/src/core/hle/kernel/k_spin_lock.h index 094a1e6be0..f7bc8b4d05 100644 --- a/src/core/hle/kernel/k_spin_lock.h +++ b/src/core/hle/kernel/k_spin_lock.h @@ -3,9 +3,11 @@ #pragma once -#include <mutex> +#include <atomic> +#include <mutex> // Kernel headers that use std::mutex get it through this one #include "common/common_funcs.h" +#include "common/common_types.h" #include "core/hle/kernel/k_scoped_lock.h" namespace Kernel { @@ -22,7 +24,11 @@ public: bool TryLock(); private: - std::mutex m_lock; + // 0 free, 1 held, 2 held with (possible) sleepers. The scheduler lock is taken by every + // core thousands of times a frame and held for microseconds: a contended Lock spins briefly + // before sleeping, instead of putting the host thread to sleep at once and paying its + // wake-up latency (std::mutex did). + std::atomic<u32> m_state{0}; }; // TODO(bunnei): Alias for now, in case we want to implement these accurately in the future. diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index 4f1d849209..b437dd4b4e 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -171,7 +171,7 @@ struct KernelCore::Impl { schedulers[core_id].reset(); } - // Next host thead ID to use, 0-3 IDs represent core threads, >3 represent others + // Next host thead ID to use, IDs below NUM_CPU_CORES represent core threads, the rest others next_host_thread_id = Core::Hardware::NUM_CPU_CORES; // Close kernel objects that were not freed on shutdown diff --git a/src/core/hle/kernel/physical_core.cpp b/src/core/hle/kernel/physical_core.cpp index c5236c64e3..c3ae7629aa 100644 --- a/src/core/hle/kernel/physical_core.cpp +++ b/src/core/hle/kernel/physical_core.cpp @@ -4,6 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <chrono> +#include <cstdlib> +#include <thread> #include "common/scope_exit.h" #include "common/settings.h" #include "core/core.h" @@ -222,8 +225,22 @@ void PhysicalCore::LogBacktrace(KernelCore& kernel) { } void PhysicalCore::Idle() { + // Guest job systems hand work between cores many times per frame; blocking at once adds the + // host's thread wake-up latency to every handoff. EDEN_NX2_IDLE_SPIN_US=<n>: poll for that + // many microseconds before blocking. + static const auto spin = std::chrono::microseconds{[] { + const char* text = std::getenv("EDEN_NX2_IDLE_SPIN_US"); + return text ? std::strtoll(text, nullptr, 10) : 0LL; + }()}; + if (spin.count() > 0) { + const auto deadline = std::chrono::steady_clock::now() + spin; + while (!m_is_interrupted.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + } std::unique_lock lk{m_guard}; - m_on_interrupt.wait(lk, [this] { return m_is_interrupted; }); + m_on_interrupt.wait(lk, [this] { return m_is_interrupted.load(); }); } bool PhysicalCore::IsInterrupted() const { diff --git a/src/core/hle/kernel/physical_core.h b/src/core/hle/kernel/physical_core.h index b92201204a..20371d6717 100644 --- a/src/core/hle/kernel/physical_core.h +++ b/src/core/hle/kernel/physical_core.h @@ -6,6 +6,7 @@ #pragma once +#include <atomic> #include <condition_variable> #include <cstddef> #include <memory> @@ -71,7 +72,7 @@ private: std::condition_variable m_on_interrupt; Core::ArmInterface* m_arm_interface{}; KThread* m_current_thread{}; - bool m_is_interrupted{}; + std::atomic<bool> m_is_interrupted{}; bool m_is_single_core{}; }; diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp index 3542d5a3b6..53d964ecbc 100644 --- a/src/core/hle/kernel/svc.cpp +++ b/src/core/hle/kernel/svc.cpp @@ -6,6 +6,8 @@ // This file is automatically generated using svc_generator.py. // DO NOT MODIFY IT MANUALLY +#include <cstdlib> +#include <cstring> #include <type_traits> #include "core/arm/arm_interface.h" @@ -2505,11 +2507,60 @@ void Call(Core::System& system, u32 imm) { LOG_TRACE(Kernel_SVC, "{} [0]={:#x} [1]={:#x} [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x} [6]={:#x}", imm, GetArg64(args, 0), GetArg64(args, 1), GetArg64(args, 2), GetArg64(args, 3), GetArg64(args, 4), GetArg64(args, 5), GetArg64(args, 6)); - if (process.Is64Bit()) { + // Set EDEN_NX2_SVC_TRACE=1 to log every SVC with its arguments and results, or + // EDEN_NX2_SVC_TRACE=<first>:<last> for presents first..last only (a frame-time profile). + static const std::pair<u64, u64> trace_range = [] { + const char* text = std::getenv("EDEN_NX2_SVC_TRACE"); + if (text == nullptr) return std::pair<u64, u64>{1, 0}; + const char* colon = std::strchr(text, ':'); + if (colon == nullptr) return std::pair<u64, u64>{0, ~u64{}}; + return std::pair<u64, u64>{std::strtoull(text, nullptr, 10), std::strtoull(colon + 1, nullptr, 10)}; + }(); + const u64 presents = Core::g_nx2_present_count.load(std::memory_order_relaxed); + const bool svc_trace = presents >= trace_range.first && presents <= trace_range.second; + if (svc_trace) { + LOG_INFO(Kernel_SVC, "svc {:#04x} in t{} [0]={:#x} [1]={:#x} [2]={:#x} [3]={:#x}", imm, + GetCurrentThread(kernel).GetThreadId(), + GetArg64(args, 0), GetArg64(args, 1), GetArg64(args, 2), GetArg64(args, 3)); + } + if (imm == 0x80) { + // Switch 2 only: svcSetGpuMemoryAttribute(address, size, mask, attr). nn::gpu::sdk and + // nn::gs native memory set the new "GPU accessible" attribute (0x20) before registering + // memory with the GPU and clear it on destruction; NVN can see it through QueryMemory. + // Same checks as svcSetMemoryAttribute, restricted to that bit (as in changes.patch). + const u64 address = GetArg64(args, 0); + const u64 size = GetArg64(args, 1); + const u32 mask = static_cast<u32>(GetArg64(args, 2)); + const u32 attr = static_cast<u32>(GetArg64(args, 3)); + constexpr u32 Supported = static_cast<u32>(Svc::MemoryAttribute::GpuAccessible); + auto& page_table = process.GetPageTable(); + Result result = ResultSuccess; + if (!Common::IsAligned(address, PageSize)) { + result = ResultInvalidAddress; + } else if (size == 0 || !Common::IsAligned(size, PageSize)) { + result = ResultInvalidSize; + } else if ((attr & ~mask) != 0 || ((mask | attr) & ~Supported) != 0) { + result = ResultInvalidCombination; + } else if (address + size <= address || !page_table.Contains(address, size)) { + result = ResultInvalidCurrentMemory; + } else { + result = page_table.SetMemoryAttribute(address, size, + static_cast<KMemoryAttribute>(mask), + static_cast<KMemoryAttribute>(attr)); + } + LOG_DEBUG(Kernel_SVC, "Switch 2 svc 0x80 addr={:#x} size={:#x} mask={:#x} attr={:#x} -> {:#x}", + address, size, mask, attr, result.raw); + args[0] = result.raw; + } else if (process.Is64Bit()) { Call64(system, imm, args); } else { Call32(system, imm, args); } + if (svc_trace) { + LOG_INFO(Kernel_SVC, "svc {:#04x} out t{} [0]={:#x} [1]={:#x} [2]={:#x}", imm, + GetCurrentThread(kernel).GetThreadId(), GetArg64(args, 0), GetArg64(args, 1), + GetArg64(args, 2)); + } //kernel.ExitSVCProfile(); kernel.CurrentPhysicalCore().LoadSvcArguments(process, args); } diff --git a/src/core/hle/kernel/svc/svc_info.cpp b/src/core/hle/kernel/svc/svc_info.cpp index 863bb58fa8..bbd7d32445 100644 --- a/src/core/hle/kernel/svc/svc_info.cpp +++ b/src/core/hle/kernel/svc/svc_info.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> + #include "core/core.h" #include "core/core_timing.h" #include "core/hle/kernel/k_memory_manager.h" @@ -40,6 +42,11 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle case InfoType::UsedNonSystemMemorySize: case InfoType::IsApplication: case InfoType::FreeThreadCount: + case InfoType::Nx2RegionAddress: + case InfoType::Nx2RegionSize: + case InfoType::Nx2VammSize: + case InfoType::Nx2RemoteMemorySize: + case InfoType::Nx2RemoteMemoryUsed: case InfoType::AliasRegionExtraSize: { R_UNLESS(info_sub_id == 0, ResultInvalidEnumValue); @@ -66,6 +73,35 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle case InfoType::HeapRegionSize: *result = process->GetPageTable().GetHeapRegionSize(); R_SUCCEED(); + case InfoType::Nx2RegionAddress: + case InfoType::Nx2RegionSize: { + // Looks like the region Switch 2 VAMM reserves address space in (a title asks for a + // ~7.5 GiB reservation). VAMM backs it with MapPhysicalMemory, which only works in + // the alias region here, so hand out the upper half of the alias region. + auto& page_table = process->GetPageTable(); + const u64 half = page_table.GetAliasRegionSize() / 2; + *result = info_id_type == InfoType::Nx2RegionAddress + ? GetInteger(page_table.GetAliasRegionStart()) + half + : half; + LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType {}, returning {:#x}", + static_cast<u32>(info_id_type), *result); + R_SUCCEED(); + } + case InfoType::Nx2RemoteMemorySize: + // Unknown real value; nothing backs it yet. + *result = 0; + LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType 31 (remote memory size), returning 0"); + R_SUCCEED(); + case InfoType::Nx2RemoteMemoryUsed: + *result = 0; + LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType 32 (remote memory used), returning 0"); + R_SUCCEED(); + case InfoType::Nx2VammSize: + // A page granule: the SDK checks module addresses/sizes against (value - 1) and + // VAMM maps one granule up front. 4 KiB matches what Switch 1 kernels use. + *result = Core::Memory::YUZU_PAGESIZE; + LOG_WARNING(Kernel_SVC, "(STUBBED) Switch 2 InfoType 33, returning {:#x}", *result); + R_SUCCEED(); case InfoType::AslrRegionAddress: *result = GetInteger(process->GetPageTable().GetAliasCodeRegionStart()); R_SUCCEED(); @@ -310,7 +346,16 @@ Result GetSystemInfo(Core::System& system, uint64_t* out, SystemInfoType info_ty Result GetInfo64(Core::System& system, uint64_t* out, InfoType info_type, Handle handle, uint64_t info_subtype) { - R_RETURN(GetInfo(system, out, info_type, handle, info_subtype)); + const Result result = GetInfo(system, out, info_type, handle, info_subtype); + // EDEN_NX2_MEM_TRACE: log the memory layout and size queries of a title (diagnostics). + static const bool trace = std::getenv("EDEN_NX2_MEM_TRACE") != nullptr; + const u32 type = static_cast<u32>(info_type); + if (static u32 logs{}; trace && logs < 200 && (type <= 7 || (type >= 12 && type <= 33))) { + ++logs; + LOG_INFO(Kernel_SVC, "mem trace: GetInfo type {} sub {:#x} -> {:#x} = {:#x}", type, + info_subtype, result.raw, *out); + } + R_RETURN(result); } Result GetSystemInfo64(Core::System& system, uint64_t* out, SystemInfoType info_type, Handle handle, diff --git a/src/core/hle/kernel/svc/svc_physical_memory.cpp b/src/core/hle/kernel/svc/svc_physical_memory.cpp index 48915b2b1e..4fde85fdaa 100644 --- a/src/core/hle/kernel/svc/svc_physical_memory.cpp +++ b/src/core/hle/kernel/svc/svc_physical_memory.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> + #include "core/core.h" #include "core/hle/kernel/k_process.h" #include "core/hle/kernel/svc.h" @@ -142,16 +144,47 @@ Result SetUnsafeLimit(Core::System& system, uint64_t limit) { R_THROW(ResultNotImplemented); } +// EDEN_NX2_MEM_TRACE: log the heap and physical memory requests of a title (diagnostics). +static bool MemTrace() { + static const bool trace = std::getenv("EDEN_NX2_MEM_TRACE") != nullptr; + return trace; +} + Result SetHeapSize64(Core::System& system, uint64_t* out_address, uint64_t size) { - R_RETURN(SetHeapSize(system, out_address, size)); + const Result result = SetHeapSize(system, out_address, size); + if (MemTrace()) { + LOG_INFO(Kernel_SVC, "mem trace: SetHeapSize {:#x} -> {:#x} at {:#x}", size, result.raw, + *out_address); + } + R_RETURN(result); } Result MapPhysicalMemory64(Core::System& system, uint64_t address, uint64_t size) { - R_RETURN(MapPhysicalMemory(system, address, size)); + const Result result = MapPhysicalMemory(system, address, size); + static u64 total{}; + if (MemTrace()) { + if (result.IsSuccess()) { + total += size; + } + static u32 logs{}; + if (result.IsError() || logs++ < 400) { + LOG_INFO(Kernel_SVC, "mem trace: MapPhysicalMemory {:#x}+{:#x} -> {:#x} (mapped total {:#x})", + address, size, result.raw, total); + } + } + R_RETURN(result); } Result UnmapPhysicalMemory64(Core::System& system, uint64_t address, uint64_t size) { - R_RETURN(UnmapPhysicalMemory(system, address, size)); + const Result result = UnmapPhysicalMemory(system, address, size); + if (MemTrace()) { + static u32 logs{}; + if (result.IsError() || logs++ < 400) { + LOG_INFO(Kernel_SVC, "mem trace: UnmapPhysicalMemory {:#x}+{:#x} -> {:#x}", address, + size, result.raw); + } + } + R_RETURN(result); } Result MapPhysicalMemoryUnsafe64(Core::System& system, uint64_t address, uint64_t size) { diff --git a/src/core/hle/kernel/svc/svc_thread.cpp b/src/core/hle/kernel/svc/svc_thread.cpp index dbca13dd26..7f6b959256 100644 --- a/src/core/hle/kernel/svc/svc_thread.cpp +++ b/src/core/hle/kernel/svc/svc_thread.cpp @@ -20,6 +20,18 @@ constexpr bool IsValidVirtualCoreId(int32_t core_id) { return (0 <= core_id && core_id < static_cast<int32_t>(Core::Hardware::NUM_CPU_CORES)); } +// Switch 2 hardware has eight cores. Six are emulated (the most an application is given), so +// fold any id past them back onto the low cores, both in core ids and in affinity masks. +constexpr int32_t FoldExtraCoreId(int32_t core_id) { + constexpr auto num_cores = static_cast<int32_t>(Core::Hardware::NUM_CPU_CORES); + return core_id >= num_cores && core_id < 2 * num_cores ? core_id - num_cores : core_id; +} + +constexpr u64 FoldExtraCoreMask(u64 mask) { + constexpr u64 low = (1ULL << Core::Hardware::NUM_CPU_CORES) - 1; + return (mask & low) | ((mask >> Core::Hardware::NUM_CPU_CORES) & low); +} + } // Anonymous namespace /// Creates a new thread @@ -36,6 +48,7 @@ Result CreateThread(Core::System& system, Handle* out_handle, u64 entry_point, u if (core_id == IdealCoreUseProcessValue) { core_id = process.GetIdealCoreId(); } + core_id = FoldExtraCoreId(core_id); // Validate arguments. R_UNLESS(IsValidVirtualCoreId(core_id), ResultInvalidCoreId); @@ -261,6 +274,10 @@ Result SetThreadCoreMask(Core::System& system, Handle thread_handle, s32 core_id R_UNLESS((affinity_mask | process_core_mask) == process_core_mask, ResultInvalidCoreId); R_UNLESS(affinity_mask != 0, ResultInvalidCombination); + // Validated against the real mask above; fold to the emulated cores from here on. + affinity_mask = FoldExtraCoreMask(affinity_mask); + core_id = FoldExtraCoreId(core_id); + // Validate the core id. if (IsValidVirtualCoreId(core_id)) { R_UNLESS(((1ULL << core_id) & affinity_mask) != 0, ResultInvalidCombination); diff --git a/src/core/hle/kernel/svc_types.h b/src/core/hle/kernel/svc_types.h index b0f4796da4..0bf2aaaa2d 100644 --- a/src/core/hle/kernel/svc_types.h +++ b/src/core/hle/kernel/svc_types.h @@ -50,6 +50,7 @@ enum class MemoryAttribute : u32 { DeviceShared = (1 << 2), Uncached = (1 << 3), PermissionLocked = (1 << 4), + GpuAccessible = (1 << 5), // Switch 2 (svc 0x80) }; DECLARE_ENUM_FLAG_OPERATORS(MemoryAttribute); @@ -157,6 +158,17 @@ enum class InfoType : u32 { IsSvcPermitted = 26, IoRegionHint = 27, AliasRegionExtraSize = 28, + // Switch 2 kernels: address/size of a region the SDK queries between the alias and + // ASLR regions during nnosInitialize; likely the VAMM address-space region. + Nx2RegionAddress = 29, + Nx2RegionSize = 30, + // Switch 2 kernels: nn::os::GetRemoteMemoryUsage returns {id 31, id 32}; used by + // nn::gsv/nn::nvsrv BootWithRemoteMemory (memory set aside for the graphics stack). + Nx2RemoteMemorySize = 31, + Nx2RemoteMemoryUsed = 32, + // Switch 2 kernels: a page granule. The SDK aligns module ranges and VAMM's first + // MapPhysicalMemory to it. + Nx2VammSize = 33, TransferMemoryHint = 34, diff --git a/src/core/hle/service/am/service/application_functions.cpp b/src/core/hle/service/am/service/application_functions.cpp index 90d292c4a2..d4f7cd6041 100644 --- a/src/core/hle/service/am/service/application_functions.cpp +++ b/src/core/hle/service/am/service/application_functions.cpp @@ -118,7 +118,8 @@ IApplicationFunctions::IApplicationFunctions(Core::System& system_, std::shared_ {220, nullptr, "Unknown220"}, // [20.0.0+] {300, nullptr, "CreateMovieWriter"}, // [19.0.0+] {310, nullptr, "Unknown310"}, // [20.0.0+] - {320, nullptr, "Unknown320"}, // [20.0.0+] + {320, D<&IApplicationFunctions::Unknown320>, "Unknown320"}, // [20.0.0+] + {321, D<&IApplicationFunctions::Unknown321>, "Unknown321"}, // Switch 2 {330, D<&IApplicationFunctions::Unknown330>, "Unknown330"}, // [20.0.0+] {500, nullptr, "StartContinuousRecordingFlushForDebug"}, {1000, nullptr, "CreateMovieMaker"}, @@ -352,10 +353,11 @@ Result IApplicationFunctions::GetPseudoDeviceId(Out<Common::UUID> out_pseudo_dev // This should be hashed with the device specific hash // for now this will do const auto res = FileSys::PatchManager::GetMetadataFromBaseOrUpdate(system, m_applet->program_id); - R_UNLESS(res.first != nullptr, ResultUnknown); std::array<u8, EVP_MAX_MD_SIZE> hash; unsigned int hash_len = 0; - auto const seed = res.first->raw.seed_for_pseudo_device_id; + // Loose ExeFS boots have no control.nacp; seed from the program id instead of failing. + auto const seed = res.first != nullptr ? res.first->raw.seed_for_pseudo_device_id + : m_applet->program_id; EVP_MD_CTX *ctx = EVP_MD_CTX_new(); auto const algorithm = EVP_sha1(); EVP_DigestInit_ex(ctx, algorithm, nullptr); @@ -544,6 +546,18 @@ Result IApplicationFunctions::GetUnknownEvent210( R_SUCCEED(); } +Result IApplicationFunctions::Unknown320() { + // Called by Switch 2 titles during nn::oe::Initialize; arguments unknown. + LOG_WARNING(Service_AM, "(STUBBED) called"); + R_SUCCEED(); +} + +Result IApplicationFunctions::Unknown321() { + // Called by Switch 2 titles right after 320; arguments unknown. + LOG_WARNING(Service_AM, "(STUBBED) called"); + R_SUCCEED(); +} + Result IApplicationFunctions::Unknown330(Out<u8> out) { LOG_DEBUG(Service_AM, "called"); *out = 0; diff --git a/src/core/hle/service/am/service/application_functions.h b/src/core/hle/service/am/service/application_functions.h index b97ed7e25e..72bf8998b1 100644 --- a/src/core/hle/service/am/service/application_functions.h +++ b/src/core/hle/service/am/service/application_functions.h @@ -82,6 +82,8 @@ private: Result GetHealthWarningDisappearedSystemEvent(OutCopyHandle<Kernel::KReadableEvent> out_event); Result CreateApplicationAndRequestToStart(u64 application_id); Result GetUnknownEvent210(OutCopyHandle<Kernel::KReadableEvent> out_event); + Result Unknown320(); + Result Unknown321(); Result Unknown330(Out<u8> out); Result PrepareForJit(); diff --git a/src/core/hle/service/am/service/application_proxy.cpp b/src/core/hle/service/am/service/application_proxy.cpp index df438e1e17..430f3c6888 100644 --- a/src/core/hle/service/am/service/application_proxy.cpp +++ b/src/core/hle/service/am/service/application_proxy.cpp @@ -11,11 +11,13 @@ #include "core/hle/service/am/service/common_state_getter.h" #include "core/hle/service/am/service/debug_functions.h" #include "core/hle/service/am/service/display_controller.h" +#include "core/hle/service/am/service/graphics_scale_controller.h" #include "core/hle/service/am/service/library_applet_creator.h" #include "core/hle/service/am/service/process_winding_controller.h" #include "core/hle/service/am/service/self_controller.h" #include "core/hle/service/am/service/window_controller.h" #include "core/hle/service/cmif_serialization.h" +#include "core/hle/service/nx2_stub_interface.h" namespace Service::AM { @@ -33,6 +35,13 @@ IApplicationProxy::IApplicationProxy(Core::System& system_, std::shared_ptr<Appl {10, D<&IApplicationProxy::GetProcessWindingController>, "GetProcessWindingController"}, {11, D<&IApplicationProxy::GetLibraryAppletCreator>, "GetLibraryAppletCreator"}, {20, D<&IApplicationProxy::GetApplicationFunctions>, "GetApplicationFunctions"}, + // Switch 2-only getters; see nx2_stub_interface.h. + {40, &IApplicationProxy::GetNx2Interface, "Nx2Getter40"}, + {50, &IApplicationProxy::GetNx2Interface, "Nx2Getter50"}, + {60, D<&IApplicationProxy::GetGraphicsScaleController>, "GetGraphicsScaleController (Switch 2)"}, + {70, &IApplicationProxy::GetNx2Interface, "Nx2Getter70"}, + {80, &IApplicationProxy::GetNx2Interface, "Nx2Getter80"}, + {90, &IApplicationProxy::GetNx2Interface, "Nx2Getter90"}, {1000, D<&IApplicationProxy::GetDebugFunctions>, "GetDebugFunctions"}, }; // clang-format on @@ -42,6 +51,17 @@ IApplicationProxy::IApplicationProxy(Core::System& system_, std::shared_ptr<Appl IApplicationProxy::~IApplicationProxy() = default; +Result IApplicationProxy::GetGraphicsScaleController( + Out<SharedPointer<IGraphicsScaleController>> out_controller) { + LOG_DEBUG(Service_AM, "called"); + *out_controller = std::make_shared<IGraphicsScaleController>(system); + R_SUCCEED(); +} + +void IApplicationProxy::GetNx2Interface(HLERequestContext& ctx) { + ReturnNx2StubInterface(system, ctx, "IApplicationProxy"); +} + Result IApplicationProxy::GetAudioController( Out<SharedPointer<IAudioController>> out_audio_controller) { LOG_DEBUG(Service_AM, "called"); diff --git a/src/core/hle/service/am/service/application_proxy.h b/src/core/hle/service/am/service/application_proxy.h index b56f597605..0a706be4fd 100644 --- a/src/core/hle/service/am/service/application_proxy.h +++ b/src/core/hle/service/am/service/application_proxy.h @@ -17,6 +17,7 @@ class IApplicationFunctions; class ICommonStateGetter; class IDebugFunctions; class IDisplayController; +class IGraphicsScaleController; class ILibraryAppletCreator; class IProcessWindingController; class ISelfController; @@ -30,6 +31,8 @@ public: ~IApplicationProxy(); private: + void GetNx2Interface(HLERequestContext& ctx); + Result GetGraphicsScaleController(Out<SharedPointer<IGraphicsScaleController>> out_controller); Result GetAudioController(Out<SharedPointer<IAudioController>> out_audio_controller); Result GetDisplayController(Out<SharedPointer<IDisplayController>> out_display_controller); Result GetProcessWindingController( diff --git a/src/core/hle/service/am/service/application_proxy_service.cpp b/src/core/hle/service/am/service/application_proxy_service.cpp index 7727951d8c..67306aadc0 100644 --- a/src/core/hle/service/am/service/application_proxy_service.cpp +++ b/src/core/hle/service/am/service/application_proxy_service.cpp @@ -19,6 +19,8 @@ IApplicationProxyService::IApplicationProxyService(Core::System& system_, : ServiceFramework{system_, "appletOE"}, m_window_system{window_system} { static const FunctionInfo functions[] = { {0, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy"}, + // Switch 2 SDKs open the proxy with command 10, same arguments. + {10, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy (Switch 2)"}, }; RegisterHandlers(functions); } diff --git a/src/core/hle/service/am/service/graphics_scale_controller.h b/src/core/hle/service/am/service/graphics_scale_controller.h new file mode 100644 index 0000000000..07e38c6ac8 --- /dev/null +++ b/src/core/hle/service/am/service/graphics_scale_controller.h @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "core/hle/service/cmif_types.h" +#include "core/hle/service/nx2_stub_interface.h" +#include "core/hle/service/service.h" + +namespace Service::AM { + +// Switch 2 interface returned by IApplicationProxy command 60 (name is a guess). nn::oe's +// GetCurrentGraphicsScale reads command 10 and accepts 10, 20 or 30 (scale enum 0, 1, 2). +// Commands without a handler are auto-stubbed because the name carries the nx2: prefix. +class IGraphicsScaleController final : public ServiceFramework<IGraphicsScaleController> { +public: + explicit IGraphicsScaleController(Core::System& system_) + : ServiceFramework{system_, "nx2:IGraphicsScaleController"} { + // clang-format off + static const FunctionInfo functions[] = { + {10, D<&IGraphicsScaleController::GetCurrentGraphicsScale>, "GetCurrentGraphicsScale"}, + }; + // clang-format on + RegisterHandlers(functions); + } + +private: + Result GetCurrentGraphicsScale(Out<u32> out_scale) { + LOG_WARNING(Service_AM, "(STUBBED) called, returning 10"); + *out_scale = 10; + R_SUCCEED(); + } +}; + +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/self_controller.cpp b/src/core/hle/service/am/service/self_controller.cpp index 3a654b9daa..a8078e6048 100644 --- a/src/core/hle/service/am/service/self_controller.cpp +++ b/src/core/hle/service/am/service/self_controller.cpp @@ -12,6 +12,7 @@ #include "core/hle/service/caps/caps_su.h" #include "core/hle/service/cmif_serialization.h" #include "core/hle/service/nvnflinger/nvnflinger.h" +#include "core/hle/service/nx2_gsv.h" #include "core/hle/service/sm/sm.h" #include "core/hle/service/vi/vi_results.h" @@ -73,6 +74,7 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet> {230, D<&ISelfController::Unknown230>, "Unknown230"}, {240, D<&ISelfController::Unknown240>, "Unknown240"}, {1000, nullptr, "GetDebugStorageChannel"}, + {2000, &ISelfController::GetNx2GraphicsSystemSession, "GetGraphicsSystemSession (Switch 2, guessed)"}, }; // clang-format on @@ -83,6 +85,12 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet> m_applet->library_applet_mode); } +void ISelfController::GetNx2GraphicsSystemSession(HLERequestContext& ctx) { + // nn::gsv::BootWithRemoteMemory sends TIPC on the handle this returns. + LOG_WARNING(Service_AM, "(STUBBED) Switch 2 ISelfController::2000, returning a nx2:gsv session"); + ReturnRawSession(system, ctx, std::make_shared<INx2GraphicsSystemSession>(system)); +} + ISelfController::~ISelfController() { std::scoped_lock lk{m_applet->lock}; m_applet->display_layer_manager.Finalize(); diff --git a/src/core/hle/service/am/service/self_controller.h b/src/core/hle/service/am/service/self_controller.h index c9f480f575..221fbb0981 100644 --- a/src/core/hle/service/am/service/self_controller.h +++ b/src/core/hle/service/am/service/self_controller.h @@ -30,6 +30,7 @@ public: ~ISelfController() override; private: + void GetNx2GraphicsSystemSession(HLERequestContext& ctx); Result Exit(); Result LockExit(); Result UnlockExit(); diff --git a/src/core/hle/service/audio/audio.cpp b/src/core/hle/service/audio/audio.cpp index fde766a275..211e1cb6bc 100644 --- a/src/core/hle/service/audio/audio.cpp +++ b/src/core/hle/service/audio/audio.cpp @@ -6,6 +6,7 @@ #include "core/core.h" #include "core/hle/service/audio/audio.h" +#include "core/hle/service/audio/nx2_audio_user_client.h" #include "core/hle/service/audio/audio_controller.h" #include "core/hle/service/audio/audio_in_manager.h" #include "core/hle/service/audio/audio_out_manager.h" @@ -203,6 +204,8 @@ void LoopProcess(Core::System& system) { server_manager->RegisterNamedService("audrec:a", std::make_shared<IFinalOutputRecorderManagerForApplet>(system), 30); server_manager->RegisterNamedService("audrec:u", std::make_shared<IFinalOutputRecorderManager>(system), 30); server_manager->RegisterNamedService("audren:u", std::make_shared<IAudioRendererManager>(system), 30); + // Switch 2 audio entry point (nn::audio::CreateUserClientSessionByHipc). + server_manager->RegisterNamedService("aud:u", std::make_shared<INx2AudioUserClient>(system), 30); server_manager->RegisterNamedService("audren:a", std::make_shared<IAudioRendererManagerForApplet>(system), 30); server_manager->RegisterNamedService("hwopus", std::make_shared<IHardwareOpusDecoderManager>(system), 25); ServerManager::RunServer(std::move(server_manager)); diff --git a/src/core/hle/service/audio/nx2_audio_user_client.h b/src/core/hle/service/audio/nx2_audio_user_client.h new file mode 100644 index 0000000000..2d1a158c84 --- /dev/null +++ b/src/core/hle/service/audio/nx2_audio_user_client.h @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "core/hle/service/audio/audio_in_manager.h" +#include "core/hle/service/audio/audio_out_manager.h" +#include "core/hle/service/audio/audio_renderer_manager.h" +#include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/service.h" + +namespace Service::Audio { + +// Switch 2 "aud:u" (nn::audio::CreateUserClientSessionByHipc). The SDK reaches the Switch 1 +// audio interfaces through it: command 2 returns an object used exactly like +// IAudioRendererManager (vtable slot 1 = GetWorkBufferSize with a Switch 1 parameter block), so +// it is backed by Eden's audren:u implementation. The nx2: name keeps unknown commands +// auto-stubbed and logged. +class INx2AudioUserClient final : public ServiceFramework<INx2AudioUserClient> { +public: + explicit INx2AudioUserClient(Core::System& system_) + : ServiceFramework{system_, "nx2:aud:u"} { + // clang-format off + static const FunctionInfo functions[] = { + {2, &INx2AudioUserClient::GetAudioRendererManager, "GetAudioRendererManager (guessed)"}, + }; + // clang-format on + RegisterHandlers(functions); + } + +private: + void GetAudioRendererManager(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const u32 arg = rp.Pop<u32>(); + LOG_WARNING(Service_Audio, "(STUBBED) Switch 2 aud:u 2 arg={}, returning IAudioRendererManager", + arg); + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface<IAudioRendererManager>(ctx, system); + } +}; + +} // namespace Service::Audio diff --git a/src/core/hle/service/hid/hid_server.cpp b/src/core/hle/service/hid/hid_server.cpp index 9689bce8dd..04b4663bc2 100644 --- a/src/core/hle/service/hid/hid_server.cpp +++ b/src/core/hle/service/hid/hid_server.cpp @@ -132,6 +132,7 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {210, C<&IHidServer::EndPermitVibrationSession>, "EndPermitVibrationSession"}, {211, C<&IHidServer::IsVibrationDeviceMounted>, "IsVibrationDeviceMounted"}, {212, C<&IHidServer::SendVibrationValueInBool>, "SendVibrationValueInBool"}, + {214, C<&IHidServer::SendNx2VibrationValues>, "SendNx2VibrationValues (guessed)"}, {300, C<&IHidServer::ActivateConsoleSixAxisSensor>, "ActivateConsoleSixAxisSensor"}, {301, C<&IHidServer::StartConsoleSixAxisSensor>, "StartConsoleSixAxisSensor"}, {302, C<&IHidServer::StopConsoleSixAxisSensor>, "StopConsoleSixAxisSensor"}, @@ -1069,6 +1070,11 @@ Result IHidServer::IsVibrationDeviceMounted( R_SUCCEED(); } +Result IHidServer::SendNx2VibrationValues() { + LOG_DEBUG(Service_HID, "(STUBBED) called"); + R_SUCCEED(); +} + Result IHidServer::SendVibrationValueInBool( bool is_vibrating, Core::HID::VibrationDeviceHandle vibration_device_handle, ClientAppletResourceUserId aruid) { diff --git a/src/core/hle/service/hid/hid_server.h b/src/core/hle/service/hid/hid_server.h index 57a1a04b9c..574d54c415 100644 --- a/src/core/hle/service/hid/hid_server.h +++ b/src/core/hle/service/hid/hid_server.h @@ -189,6 +189,9 @@ private: Result SendVibrationValueInBool(bool is_vibrating, Core::HID::VibrationDeviceHandle vibration_device_handle, ClientAppletResourceUserId aruid); + // Switch 2 sends vibration every frame with 214 (pointer buffers of device handles and + // values, like 206); Eden doesn't emulate vibration, so accept and drop it. + Result SendNx2VibrationValues(); Result ActivateConsoleSixAxisSensor(ClientAppletResourceUserId aruid); Result StartConsoleSixAxisSensor(Core::HID::ConsoleSixAxisSensorHandle console_sixaxis_handle, ClientAppletResourceUserId aruid); diff --git a/src/core/hle/service/kernel_helpers.cpp b/src/core/hle/service/kernel_helpers.cpp index f7a5cb7049..e4d6925f19 100644 --- a/src/core/hle/service/kernel_helpers.cpp +++ b/src/core/hle/service/kernel_helpers.cpp @@ -4,6 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <mutex> +#include <unordered_map> + #include "core/core.h" #include "core/core_timing.h" #include "core/hle/kernel/k_event.h" @@ -16,6 +19,12 @@ namespace Service::KernelHelpers { +namespace { +// Event names for the debugger's handle listing. +std::mutex event_names_mutex; +std::unordered_map<const Kernel::KEvent*, std::string> event_names; +} // namespace + ServiceContext::ServiceContext(Core::System& system_, std::string name_) : kernel(system_.Kernel()) { @@ -64,14 +73,28 @@ Kernel::KEvent* ServiceContext::CreateEvent(std::string&& name) { // Register the event. Kernel::KEvent::Register(kernel, event); + { + std::scoped_lock lk{event_names_mutex}; + event_names[event] = std::move(name); + } return event; } void ServiceContext::CloseEvent(Kernel::KEvent* event) { if (event) { + { + std::scoped_lock lk{event_names_mutex}; + event_names.erase(event); + } event->GetReadableEvent().Close(kernel); event->Close(kernel); } } +std::string ServiceContext::GetEventName(const Kernel::KEvent* event) { + std::scoped_lock lk{event_names_mutex}; + const auto it = event_names.find(event); + return it == event_names.end() ? std::string{} : it->second; +} + } // namespace Service::KernelHelpers diff --git a/src/core/hle/service/kernel_helpers.h b/src/core/hle/service/kernel_helpers.h index 4e21c8857f..a62dd838aa 100644 --- a/src/core/hle/service/kernel_helpers.h +++ b/src/core/hle/service/kernel_helpers.h @@ -29,6 +29,9 @@ public: void CloseEvent(Kernel::KEvent* event); + /// Name an event was created with (debugger handle listing); empty when unknown. + static std::string GetEventName(const Kernel::KEvent* event); + Kernel::KernelCore& kernel; Kernel::KProcess* process{}; bool process_created{false}; diff --git a/src/core/hle/service/ns/platform_service_manager.cpp b/src/core/hle/service/ns/platform_service_manager.cpp index bf93a2edfa..88aac56b1c 100644 --- a/src/core/hle/service/ns/platform_service_manager.cpp +++ b/src/core/hle/service/ns/platform_service_manager.cpp @@ -22,7 +22,9 @@ #include "core/hle/kernel/k_shared_memory.h" #include "core/hle/kernel/kernel.h" #include "core/hle/service/cmif_serialization.h" +#include "common/alignment.h" #include "core/hle/service/filesystem/filesystem.h" +#include "core/hle/service/ipc_helpers.h" #include "core/hle/service/ns/platform_service_manager.h" namespace Service::NS { @@ -83,6 +85,8 @@ struct IPlatformServiceManager::Impl { // Automatically populated based on shared_fonts dump or system archives. // 6 builtin fonts + extra 2 for whatever may come after boost::container::static_vector<FontRegion, 8> shared_font_regions; + /// Plain TTF copies of the same fonts (Switch 2 GetBuiltInFontData), after the BFTTF data. + boost::container::static_vector<FontRegion, 8> builtin_font_regions; /// Backing memory for the shared font data std::array<u8, SHARED_FONT_MEM_SIZE> shared_font; }; @@ -107,6 +111,8 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch {106, nullptr, "GetFunctionBlackListVersion"}, {1000, nullptr, "LoadNgWordDataForPlatformRegionChina"}, {1001, nullptr, "GetNgWordDataSizeForPlatformRegionChina"}, + {2000, &IPlatformServiceManager::GetBuiltInFontSharedMemory, "GetBuiltInFontSharedMemory (Switch 2)"}, + {2010, &IPlatformServiceManager::GetBuiltInFontData, "GetBuiltInFontData (Switch 2)"}, }; // clang-format on RegisterHandlers(functions); @@ -116,6 +122,7 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch // Attempt to load shared font data from disk const auto* nand = fsc.GetSystemNANDContents(); std::size_t offset = 0; + std::vector<std::vector<u8>> plain_fonts; // Rebuild shared fonts from data ncas or synthesize for (auto& font : SHARED_FONTS) { FileSys::VirtualFile romfs; @@ -151,9 +158,58 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch Common::swap32); // Font offset and size do not account for the header const FontRegion region{u32(offset + 8), u32((font_data_u32.size() * sizeof(u32)) - 8)}; + std::vector<u8> plain(region.size); + DecryptSharedFontToTTF(font_data_u32, plain); + plain_fonts.push_back(std::move(plain)); DecryptSharedFont(font_data_u32, impl->shared_font, offset); impl->shared_font_regions.push_back(region); } + + for (const auto& plain : plain_fonts) { + offset = Common::AlignUp(offset, 0x1000); + if (offset + plain.size() > SHARED_FONT_MEM_SIZE) { + LOG_WARNING(Service_NS, "No shared memory left for plain built-in font copies"); + break; + } + std::memcpy(impl->shared_font.data() + offset, plain.data(), plain.size()); + impl->builtin_font_regions.push_back({u32(offset), u32(plain.size())}); + offset += plain.size(); + } +} + +// Switch 2: the shared memory GetBuiltInFontData offsets point into (copy handle, u32 size); the +// SDK maps it read-only. Eden serves its font shared memory, which holds plain copies too. +void IPlatformServiceManager::GetBuiltInFontSharedMemory(HLERequestContext& ctx) { + LOG_WARNING(Service_NS, "(STUBBED) Switch 2 GetBuiltInFontSharedMemory"); + std::memcpy(kernel.GetFontSharedMem().GetPointer(), impl->shared_font.data(), impl->shared_font.size()); + IPC::ResponseBuilder rb{ctx, 3, 1}; + rb.Push(ResultSuccess); + rb.Push<u32>(static_cast<u32>(SHARED_FONT_MEM_SIZE)); + rb.PushCopyObjects(ctx, kernel.GetFontSharedMem()); +} + +// nn::pl::GetBuiltInFontData(BuiltInFontDataOunce*, BuiltInFontTypeOunce, bool): returns a 16-byte +// bool that says whether the font exists and a 16-byte descriptor (u8 flags, u8 flag bit, u16, +// u32 offset into the font shared memory, u32 size, u32). The client expects plain TrueType/OpenType data +// there. The type numbering is unknown, so types map onto Eden's shared fonts in order. +void IPlatformServiceManager::GetBuiltInFontData(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const u32 type = rp.Pop<u32>(); + const bool flag = rp.Pop<u8>() != 0; + const auto& regions = impl->builtin_font_regions; + const bool found = !regions.empty(); + const FontRegion region = found ? regions[std::min<size_t>(type, regions.size() - 1)] : EMPTY_REGION; + LOG_WARNING(Service_NS, "(STUBBED) Switch 2 GetBuiltInFontData type={} flag={} -> offset={:#x} size={:#x}", + type, flag, region.offset, region.size); + // Raw reply: the bool at offset 0, the descriptor at offset 8. + IPC::ResponseBuilder rb{ctx, 2 + 6}; + rb.Push(ResultSuccess); + rb.Push<u32>(found ? 1 : 0); + rb.Push<u32>(0); + rb.Push<u32>(0); + rb.Push<u32>(region.offset); + rb.Push<u32>(region.size); + rb.Push<u32>(0); } IPlatformServiceManager::~IPlatformServiceManager() = default; diff --git a/src/core/hle/service/ns/platform_service_manager.h b/src/core/hle/service/ns/platform_service_manager.h index 73521abbc2..ed5b1c1e39 100644 --- a/src/core/hle/service/ns/platform_service_manager.h +++ b/src/core/hle/service/ns/platform_service_manager.h @@ -73,6 +73,9 @@ private: Out<bool> out_fonts_are_loaded, Out<u32> out_font_count, Set::LanguageCode language_code); + void GetBuiltInFontSharedMemory(HLERequestContext& ctx); + void GetBuiltInFontData(HLERequestContext& ctx); + struct Impl; std::unique_ptr<Impl> impl; }; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index 3c9e42f602..2634902641 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -14,13 +14,16 @@ #include "common/logging.h" #include "core/core.h" #include "core/hle/service/nvdrv/core/container.h" +#include "core/hle/service/nvdrv/core/heap_mapper.h" #include "core/hle/service/nvdrv/core/nvmap.h" #include "core/hle/service/nvdrv/devices/ioctl_serialization.h" #include "core/hle/service/nvdrv/devices/nvhost_as_gpu.h" #include "core/hle/service/nvdrv/devices/nvhost_gpu.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/hle/service/nvdrv/nvdrv.h" #include "video_core/control/channel_state.h" #include "video_core/gpu.h" +#include "video_core/host1x/host1x.h" #include "video_core/memory_manager.h" #include "video_core/rasterizer_interface.h" @@ -30,7 +33,9 @@ nvhost_as_gpu::nvhost_as_gpu(Core::System& system_, Module& module_, NvCore::Con : nvdevice{system_}, module{module_}, container{core}, nvmap{core.GetNvMapFile()}, vm{}, gmmu{} {} -nvhost_as_gpu::~nvhost_as_gpu() = default; +nvhost_as_gpu::~nvhost_as_gpu() { + Nx2GpuMemoryRegistry::Get().RemoveListener(this); +} NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input, std::span<u8> output) { @@ -49,8 +54,24 @@ NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> i return WrapFixed(this, &nvhost_as_gpu::MapBufferEx, input, output); case 0x8: return WrapFixed(this, &nvhost_as_gpu::GetVARegions1, input, output); - case 0x9: - return WrapFixed(this, &nvhost_as_gpu::AllocAsEx, input, output); + case 0x9: { + const auto result = WrapFixed(this, &nvhost_as_gpu::AllocAsEx, input, output); + if (result == NvResult::Success && g_nx2_driver) { + Nx2GpuMemoryRegistry::Get().AddListener( + this, + [this](u64 gpu_address, u64 cpu_address, u64 size) { + Nx2Map(gpu_address, cpu_address, size); + }, + [this](u64 gpu_address, u64 size) { Nx2Unmap(gpu_address, size); }, + [this](u64 cpu_address, u64 size) { Nx2Release(cpu_address, size); }); + } + return result; + } + case 0xb: + if (g_nx2_driver) { + return WrapFixed(this, &nvhost_as_gpu::Nx2GetSyncpointShimInfo, input, output); + } + break; case 0x14: return WrapVariable(this, &nvhost_as_gpu::Remap, input, output); default: @@ -90,11 +111,105 @@ NvResult nvhost_as_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> i return NvResult::NotImplemented; } -void nvhost_as_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) {} +void nvhost_as_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) { + nx2_session = session_id; +} + +void nvhost_as_gpu::Nx2Map(u64 gpu_address, u64 cpu_address, u64 size) { + std::scoped_lock lock(mutex); + auto* session = container.GetSession(nx2_session); + if (!gmmu || session == nullptr) { + LOG_ERROR(Service_NVDRV, "Switch 2 GPU mapping {:#x}+{:#x}: no address space or session", + gpu_address, size); + return; + } + // The same CPU range is often mapped twice (at its own address and in a sparse region), so + // reuse its SMMU mapping. + DAddr device_address{}; + bool heap = false; + if (const auto it = nx2_device_ranges.find({cpu_address, size}); it != nx2_device_ranges.end()) { + device_address = it->second.address; + heap = it->second.heap; + } else if (session->has_preallocated_area && session->mapper->IsInBounds(cpu_address, size)) { + device_address = session->mapper->Map(cpu_address, size); + heap = true; + } else { + auto& smmu = system.Host1x().MemoryManager(); + device_address = smmu.Allocate(Common::AlignUp(size, 0x20000)); + if (device_address == 0) { + LOG_CRITICAL(Service_NVDRV, "Switch 2 GPU memory {:#x}+{:#x}: out of SMMU space", + cpu_address, size); + return; + } + smmu.Map(device_address, cpu_address, size, session->asid, true); + } + nx2_device_ranges[{cpu_address, size}] = {device_address, heap}; + Nx2GpuMemoryRegistry::Get().RecordDeviceRange(cpu_address, size, device_address); + gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false); + LOG_DEBUG(Service_NVDRV, "Switch 2 GPU mapping {:#x}+{:#x} -> CPU {:#x}", gpu_address, size, + cpu_address); +} + +void nvhost_as_gpu::Nx2Unmap(u64 gpu_address, u64 size) { + std::scoped_lock lock(mutex); + if (!gmmu) { + return; + } + // The GPU thread may still be running command lists that use the range. + system.GPU().RunInOrder([gmmu = gmmu.get(), gpu_address, size] { gmmu->Unmap(gpu_address, size); }); + LOG_DEBUG(Service_NVDRV, "Switch 2 GPU unmapping {:#x}+{:#x}", gpu_address, size); +} + +void nvhost_as_gpu::Nx2Release(u64 cpu_address, u64 size) { + std::scoped_lock lock(mutex); + const auto it = nx2_device_ranges.find({cpu_address, size}); + if (it == nx2_device_ranges.end()) { + return; + } + const Nx2DeviceRange range = it->second; + nx2_device_ranges.erase(it); + auto* session = container.GetSession(nx2_session); + auto* const mapper = range.heap && session != nullptr ? session->mapper.get() : nullptr; + auto& smmu = system.Host1x().MemoryManager(); + // Queued after the GPU unmaps of the range, so the GPU thread is done with it. + system.GPU().RunInOrder([&smmu, mapper, range, cpu_address, size] { + if (range.heap) { + if (mapper != nullptr) { + mapper->Unmap(cpu_address, size); + } + return; + } + smmu.Unmap(range.address, size); + smmu.Free(range.address, Common::AlignUp(size, 0x20000)); + }); + LOG_DEBUG(Service_NVDRV, "Switch 2 released backing {:#x}+{:#x}", cpu_address, size); +} + void nvhost_as_gpu::OnClose(DeviceFD fd) {} +// Switch 2 NvRm (with characteristics flag 26) asks for the read-only syncpoint shim: a GPU page +// whose u32 at id * stride reads as that syncpoint's value; NVN waits on syncpoints through it and +// asserts without one. Same scheme as changes.patch, placed on the last page of the VA range. +NvResult nvhost_as_gpu::Nx2GetSyncpointShimInfo(IoctlNx2SyncpointShimInfo& params) { + std::scoped_lock lock(mutex); + if (!vm.initialised || !gmmu) { + return NvResult::NotSupported; + } + const u64 address = vm.va_range_end - VM::YUZU_PAGESIZE; + if (!nx2_syncpoint_shim) { + nx2_syncpoint_shim = true; + gmmu->SetNativeSyncpointShim(address); + } + params.address = address; + params.stride = sizeof(u32); + params.count = Tegra::MemoryManager::NumShimSyncpoints; + LOG_INFO(Service_NVDRV, "Switch 2 syncpoint shim at {:#x} ({} syncpoints)", address, + params.count); + return NvResult::Success; +} + NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) { - LOG_DEBUG(Service_NVDRV, "called, big_page_size={:#x}", params.big_page_size); + LOG_DEBUG(Service_NVDRV, "called, flags={:#x} big_page_size={:#x} va_range_start={:#x} va_range_end={:#x} va_range_split={:#x}", params.flags, params.big_page_size, params.va_range_start, params.va_range_end, params.va_range_split); std::scoped_lock lock(mutex); @@ -125,6 +240,15 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) { vm.va_range_start = params.va_range_start; vm.va_range_split = params.va_range_split; vm.va_range_end = params.va_range_end; + if (vm.va_range_split == 0) { + // Switch 2 NvRm (flags 5, no big page size) asks for small pages over the whole range + // and then reserves 4 KiB VA up to 512 GiB. Keep a small big-page region past the end + // so both allocators stay valid. + vm.va_range_split = vm.va_range_end; + vm.va_range_end += 4ULL << 30; + vm.absolute_big_page_limit = true; + LOG_WARNING(Service_NVDRV, "No VA split given; small pages up to {:#x}", vm.va_range_split); + } } const u64 max_big_page_bits = Common::Log2Ceil(vm.va_range_end); @@ -134,8 +258,11 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) { vm.small_page_allocator.emplace(start_pages, end_pages); const auto start_big_pages{static_cast<u32>(vm.va_range_split >> vm.big_page_size_bits)}; - const auto end_big_pages{ - static_cast<u32>((vm.va_range_end - vm.va_range_split) >> vm.big_page_size_bits)}; + // The Switch 1 layout passes the big-page region length as the limit (so the limit stays above + // the start there); a split past the region length needs the absolute end instead. + const auto end_big_pages{static_cast<u32>( + (vm.absolute_big_page_limit ? vm.va_range_end : vm.va_range_end - vm.va_range_split) >> + vm.big_page_size_bits)}; vm.big_page_allocator.emplace(start_big_pages, end_big_pages); gmmu = std::make_unique<Tegra::MemoryManager>(system, max_big_page_bits, vm.va_range_split, vm.big_page_size_bits, VM::PAGE_SIZE_BITS); diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h index b13b1c9efa..d64ec85dec 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h @@ -218,8 +218,29 @@ private: std::optional<Allocator> small_page_allocator; //! Shared as this is also used by nvhost::GpuChannel bool initialised{}; + bool absolute_big_page_limit{}; //!< Switch 2 layout: big-page limit is va_range_end } vm; std::unique_ptr<Tegra::MemoryManager> gmmu; + + struct IoctlNx2SyncpointShimInfo { + u64_le address; + u32_le stride; + u32_le count; + }; + static_assert(sizeof(IoctlNx2SyncpointShimInfo) == 16); + NvResult Nx2GetSyncpointShimInfo(IoctlNx2SyncpointShimInfo& params); + bool nx2_syncpoint_shim{}; + + // Switch 2: maps GPU-accessible process memory (CPU range) at a GPU address. + void Nx2Map(u64 gpu_address, u64 cpu_address, u64 size); + void Nx2Unmap(u64 gpu_address, u64 size); + NvCore::SessionId nx2_session{}; + struct Nx2DeviceRange { + DAddr address; + bool heap; ///< Mapped through the session's heap mapper (else an SMMU allocation) + }; + std::map<std::pair<u64, u64>, Nx2DeviceRange> nx2_device_ranges; + void Nx2Release(u64 cpu_address, u64 size); }; } // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp index 14cdd8fda4..df2421cd13 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <array> +#include <algorithm> #include <cstring> #include "common/assert.h" #include "common/logging.h" @@ -11,10 +13,16 @@ #include "core/core_timing.h" #include "core/hle/service/nvdrv/devices/ioctl_serialization.h" #include "core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" +#include "video_core/nx2.h" #include "core/hle/service/nvdrv/nvdrv.h" namespace Service::Nvidia::Devices { +namespace { +void ApplyNx2CharacteristicsTail(std::span<u8> block); +} // namespace + nvhost_ctrl_gpu::nvhost_ctrl_gpu(Core::System& system_, EventInterface& events_interface_) : nvdevice{system_}, events_interface{events_interface_} { error_notifier_event = events_interface.CreateEvent("CtrlGpuErrorNotifier"); @@ -93,9 +101,14 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> switch (command.group) { case 'G': switch (command.cmd) { - case 0x5: - return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetCharacteristics3, input, output, - inline_output); + case 0x5: { + const auto result = WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetCharacteristics3, + input, output, inline_output); + if (nx2_caller) { + ApplyNx2CharacteristicsTail(inline_output); + } + return result; + } case 0x6: return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetTPCMasks3, input, output, inline_output); @@ -112,14 +125,108 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> default: break; } + if (nx2_caller && command.group == 'G' && command.cmd == 0x1F) { + return Nx2GetEngineInfo(input, output, inline_output); + } + if (nx2_caller) { + // Switch 2 NvRm uses ioctls Switch 1 never had; answer them with a zeroed success so + // device setup can continue, and log each one. + LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 ioctl={:08X} group={:01X} command={:01X}", + command.raw, command.group, command.cmd); + std::fill(output.begin(), output.end(), u8{0}); + std::fill(inline_output.begin(), inline_output.end(), u8{0}); + return NvResult::Success; + } UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}, group={:01X}, command={:01X}", command.raw, command.group, command.cmd); return NvResult::NotImplemented; } +// nvgpu GET_ENGINE_INFO as the Switch 2 NvRm uses it: args {u32 buf_size, u32 reserved, +// u64 buf_addr}, queried once for the size and once with an inline output buffer of 16-byte items +// {u32 engine_id, u32 engine_instance, s32 runlist_id, u32 reserved}. NvRm picks the channel +// runlist from this list. Report the graphics engine (runlist 0) and an async copy engine (1). +NvResult nvhost_ctrl_gpu::Nx2GetEngineInfo(std::span<const u8> input, std::span<u8> output, + std::span<u8> inline_output) { + struct Item { + u32 engine_id; + u32 engine_instance; + s32 runlist_id; + u32 reserved; + }; + static constexpr std::array<Item, 2> items{{ + {0, 0, 0, 0}, // NVGPU_ENGINE_GR + {2, 0, 1, 0}, // NVGPU_ENGINE_ASYNC_COPY + }}; + constexpr u32 needed = static_cast<u32>(items.size() * sizeof(Item)); + + u32 buf_size{}; + if (input.size() >= sizeof(buf_size)) { + std::memcpy(&buf_size, input.data(), sizeof(buf_size)); + } + LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 GetEngineInfo buf_size={:#x} inline_out={:#x}", + buf_size, inline_output.size()); + if (buf_size >= needed && inline_output.size() >= needed) { + std::memcpy(inline_output.data(), items.data(), needed); + } + std::fill(output.begin(), output.end(), u8{0}); + std::copy_n(input.begin(), std::min(input.size(), output.size()), output.begin()); + if (output.size() >= sizeof(needed)) { + std::memcpy(output.data(), &needed, sizeof(needed)); + } + return NvResult::Success; +} + void nvhost_ctrl_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) {} void nvhost_ctrl_gpu::OnClose(DeviceFD fd) {} +namespace { + +// Switch 2 SDKs ask for a 0xD0-byte characteristics block (Switch 1: 0xA0). Their NvRm only +// accepts chip ids 0x17E, 0x12B and 0x12E and its NVN path rejects the Maxwell ones, so report the +// Switch 2 Ampere chip. Fields past the Switch 1 layout are left zero. Values are best guesses. +constexpr u64 Nx2CharacteristicsSize = 0xD0; + +template <typename GC> +void ApplyNx2Characteristics(GC& gc) { + gc.arch = 0x170; // Ampere + gc.impl = 0xE; // chip id 0x17E as listed in the Switch 2 NvRm chip table + gc.num_tpc_per_gpc = 0x6; + gc.sm_arch_sm_version = 0x808; // NVN2 only accepts SM 0x808 (its class table lookup) + gc.sm_arch_spa_version = 0x808; + gc.threed_class = 0xC797; // AMPERE_B + gc.compute_class = 0xC7C0; // AMPERE_COMPUTE_B + gc.gpfifo_class = 0xC56F; // AMPERE_CHANNEL_GPFIFO_A + gc.dma_copy_class = 0xC7B5; // AMPERE_DMA_COPY_B + gc.chipname = 0x6631306167; // "ga10f" + gc.flags |= 0x100; // NVGPU_GPU_FLAGS_SUPPORT_TSG: NvRm binds channels to a TSG + gc.flags |= 1ULL << 22; // NVGPU_GPU_FLAGS_SUPPORT_TSG_SUBCONTEXTS (see below) + gc.flags |= 1ULL << 26; // NVGPU_GPU_FLAGS_SUPPORT_SYNCPOINT_ADDRESS: syncpoint shim + + // The Switch 2 layout keeps nine s16 interface versions at 0x70..0x80 (like nvgpu's + // *_ioctl_nr_last); its NvRm requires e.g. [0x78] >= 6 and enables features above 0xa, 0x1f + // and 0x26. In the Switch 1 layout these bytes are max_fbps_count..max_tex_per_tpc. + auto* const raw = reinterpret_cast<u8*>(&gc); + for (size_t off = 0x70; off <= 0x80; off += 2) { + const s16 version = 0x40; + std::memcpy(raw + off, &version, sizeof(version)); + } +} + +// Fields of the Switch 2 block past the Switch 1 layout. NvRm sizes each TSG's subcontext (VEID) +// allocator from the u32 at 0xb4, but only when the TSG interface version (0x72) is above 0xa and +// flags bits 8 and 22 are set; otherwise every TSG gets one subcontext and a second queue's +// NvRmGpuTSGSubcontextCreate fails (0x11), leaving that NVN queue without command memory. +void ApplyNx2CharacteristicsTail(std::span<u8> block) { + constexpr size_t MaxSubcontextsOffset = 0xb4; + constexpr u32 MaxSubcontexts = 64; // Ampere VEIDs per TSG + if (block.size() >= MaxSubcontextsOffset + sizeof(MaxSubcontexts)) { + std::memcpy(block.data() + MaxSubcontextsOffset, &MaxSubcontexts, sizeof(MaxSubcontexts)); + } +} + +} // namespace + NvResult nvhost_ctrl_gpu::GetCharacteristics1(IoctlCharacteristics& params) { LOG_DEBUG(Service_NVDRV, "called"); params.gc.arch = 0x120; @@ -164,7 +271,8 @@ NvResult nvhost_ctrl_gpu::GetCharacteristics1(IoctlCharacteristics& params) { NvResult nvhost_ctrl_gpu::GetCharacteristics3( IoctlCharacteristics& params, std::span<IoctlGpuCharacteristics> gpu_characteristics) { - LOG_DEBUG(Service_NVDRV, "called"); + LOG_DEBUG(Service_NVDRV, "called, buf_size={:#x}", params.gpu_characteristics_buf_size); + const bool nx2 = params.gpu_characteristics_buf_size >= Nx2CharacteristicsSize; params.gc.arch = 0x120; params.gc.impl = 0xb; @@ -203,6 +311,14 @@ NvResult nvhost_ctrl_gpu::GetCharacteristics3( params.gc.gr_compbit_store_base_hw = 0x0; params.gpu_characteristics_buf_size = 0xA0; params.gpu_characteristics_buf_addr = 0xdeadbeef; // Cannot be 0 (UNUSED) + if (nx2) { + nx2_caller = true; + g_nx2_driver = true; + Tegra::g_nx2_ampere = true; + ApplyNx2Characteristics(params.gc); + params.gpu_characteristics_buf_size = Nx2CharacteristicsSize; + LOG_WARNING(Service_NVDRV, "Switch 2 caller, reporting Ampere chip 0x17E"); + } if (!gpu_characteristics.empty()) { gpu_characteristics.front() = params.gc; } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h index 80b7abd6a1..507cdac71d 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h @@ -208,6 +208,11 @@ private: EventInterface& events_interface; // Events + NvResult Nx2GetEngineInfo(std::span<const u8> input, std::span<u8> output, + std::span<u8> inline_output); + + // Set once a caller asked for the Switch 2 characteristics layout. + bool nx2_caller{}; Kernel::KEvent* error_notifier_event; Kernel::KEvent* unknown_event; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp index 1e571be4c9..2a144cc7ab 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp @@ -4,7 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> #include <cstring> +#include <vector> #include "common/assert.h" #include "common/logging.h" #include "core/core.h" @@ -14,6 +16,7 @@ #include "core/hle/service/nvdrv/core/syncpoint_manager.h" #include "core/hle/service/nvdrv/devices/ioctl_serialization.h" #include "core/hle/service/nvdrv/devices/nvhost_gpu.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/hle/service/nvdrv/nvdrv.h" #include "core/memory.h" #include "video_core/control/channel_state.h" @@ -48,6 +51,9 @@ nvhost_gpu::~nvhost_gpu() { events_interface.FreeEvent(sm_exception_breakpoint_int_report_event); events_interface.FreeEvent(sm_exception_breakpoint_pause_report_event); events_interface.FreeEvent(error_notifier_event); + for (auto& [id, event] : nx2_events) { + events_interface.FreeEvent(event); + } syncpoint_manager.FreeSyncpoint(channel_syncpoint); } @@ -101,6 +107,9 @@ NvResult nvhost_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> inpu } break; } + if (g_nx2_driver) { + return Nx2UnknownIoctl("nvhost-gpu", command, input, output); + } UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}", command.raw); return NvResult::NotImplemented; }; @@ -116,12 +125,19 @@ NvResult nvhost_gpu::Ioctl2(DeviceFD fd, Ioctl command, std::span<const u8> inpu } break; } + if (g_nx2_driver) { + return Nx2UnknownIoctl("nvhost-gpu", command, input, output); + } UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}", command.raw); return NvResult::NotImplemented; } NvResult nvhost_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> input, std::span<u8> output, std::span<u8> inline_output) { + if (g_nx2_driver) { + std::fill(inline_output.begin(), inline_output.end(), u8{0}); + return Nx2UnknownIoctl("nvhost-gpu", command, input, output); + } UNIMPLEMENTED_MSG("Unimplemented ioctl={:08X}", command.raw); return NvResult::NotImplemented; } @@ -278,6 +294,35 @@ NvResult nvhost_gpu::AllocateObjectContext(IoctlAllocObjCtx& params) { params.flags = allowed_mask; } + if (g_nx2_driver) { + // Switch 2 NvRm asks for Ampere classes, or class 0 (it reads the class numbers from a + // characteristics layout we don't know yet). Map them onto the Maxwell engine slots. + u32 mapped = params.class_num; + switch (params.class_num) { + case 0xC797: case 0xC697: case 0xC597: mapped = static_cast<u32>(CtxClasses::Ctx3D); break; + case 0xC7C0: case 0xC6C0: case 0xC5C0: mapped = static_cast<u32>(CtxClasses::CtxCompute); break; + case 0xC7B5: case 0xC6B5: case 0xC5B5: mapped = static_cast<u32>(CtxClasses::CtxDMA); break; + case 0xC56F: case 0xC76F: mapped = static_cast<u32>(CtxClasses::CtxChannelGPFIFO); break; + case 0: { + for (const auto candidate : {CtxClasses::Ctx3D, CtxClasses::CtxCompute, CtxClasses::CtxDMA, + CtxClasses::Ctx2D, CtxClasses::CtxKepler}) { + const auto index = GetObjectContextClassNumberIndex(candidate); + if (index >= 0 && !ctxObjs[index].has_value()) { + mapped = static_cast<u32>(candidate); + break; + } + } + break; + } + default: + break; + } + if (mapped != params.class_num) { + LOG_WARNING(Service_NVDRV, "Switch 2 object class {:#x} -> {:#x}", params.class_num, mapped); + params.class_num = mapped; + } + } + s32_le ctx_class_number_index = GetObjectContextClassNumberIndex(static_cast<CtxClasses>(params.class_num)); if (ctx_class_number_index < 0) { @@ -327,6 +372,27 @@ static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementC return result; } +// Switch 2 pushbuffers signal completion with semaphore releases, never host1x syncpoint methods, +// but the driver still has the kernel raise the syncpoint max by the submission's increment value +// and then waits for that value. Append the increments after its commands. +static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementCountCommandList( + NvFence fence, u32 count) { + boost::container::small_vector<Tegra::CommandHeader, 512> result{ + Tegra::BuildCommandHeader(Tegra::BufferMethods::WaitForIdle, 1, + Tegra::SubmissionMode::Increasing), + {}, + Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointPayload, 1, + Tegra::SubmissionMode::Increasing), + {}}; + for (u32 i = 0; i < count; ++i) { + result.push_back(Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointOperation, 1, + Tegra::SubmissionMode::Increasing)); + result.push_back( + BuildFenceAction(Tegra::Engines::Puller::FenceOperation::Increment, fence.id)); + } + return result; +} + static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementWithWfiCommandList( NvFence fence) { boost::container::small_vector<Tegra::CommandHeader, 512> result{ @@ -353,6 +419,23 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL const auto bind_id = channel_state->bind_id; + if (g_nx2_driver) { + for (const auto& entry : entries.command_lists) { + LOG_TRACE(Service_NVDRV, "nx2 GPFIFO entry raw={:#018x} addr={:#x} size={:#x}", entry.raw, + entry.addr.Value(), entry.size.Value()); + } + static const bool check = std::getenv("EDEN_NX2_PB_CHECK") != nullptr; + if (check) { + std::vector<u32> words; + for (const auto& entry : entries.command_lists) { + words.resize(entry.size); + system.ApplicationMemory().ReadBlock(entry.addr.Value(), words.data(), + words.size() * sizeof(u32)); + entries.submit_hashes.push_back(Tegra::HashPushbufferWords(words)); + } + } + } + auto& flags = params.flags; if (flags.fence_wait.Value()) { @@ -372,6 +455,11 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL params.fence.value = syncpoint_manager.IncrementSyncpointMaxExt(channel_syncpoint, increment); gpu.PushGPUEntries(bind_id, std::move(entries)); + if (g_nx2_driver && !flags.fence_increment.Value() && increment > 0) { + gpu.PushGPUEntries(bind_id, + Tegra::CommandList{BuildIncrementCountCommandList(params.fence, increment)}); + } + if (flags.fence_increment.Value()) { if (flags.suppress_wfi.Value()) { gpu.PushGPUEntries(bind_id, @@ -382,6 +470,12 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL } } + if (g_nx2_driver) { + LOG_TRACE(Service_NVDRV, + "nx2 SubmitGPFIFO entries={} flags={:#x} fence_out=({}, {}) guest_value={}", + params.num_entries, flags.raw, params.fence.id, params.fence.value, + system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(params.fence.id)); + } flags.raw = 0; return NvResult::Success; @@ -464,6 +558,16 @@ Kernel::KEvent* nvhost_gpu::QueryEvent(u32 event_id) { case 3: return error_notifier_event; default: + if (g_nx2_driver) { + // Switch 2 NVN also asks for event 7 and multi-waits on it with its syncpoint events. + // Nothing signals it yet. + auto*& event = nx2_events[event_id]; + if (event == nullptr) { + event = events_interface.CreateEvent(fmt::format("GpuChannelNx2Event{}", event_id)); + } + LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 channel event {}", event_id); + return event; + } LOG_CRITICAL(Service_NVDRV, "Unknown Ctrl GPU Event {}", event_id); return nullptr; } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h index 70a6b1bd68..ab968d6c7e 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h @@ -6,6 +6,7 @@ #pragma once +#include <map> #include <memory> #include <vector> #include "common/bit_field.h" @@ -225,6 +226,7 @@ private: Kernel::KEvent* sm_exception_breakpoint_int_report_event; Kernel::KEvent* sm_exception_breakpoint_pause_report_event; Kernel::KEvent* error_notifier_event; + std::map<u32, Kernel::KEvent*> nx2_events; }; } // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h new file mode 100644 index 0000000000..b91a929f68 --- /dev/null +++ b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h @@ -0,0 +1,204 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <algorithm> +#include <atomic> +#include <functional> +#include <map> +#include <mutex> +#include <optional> +#include <string> +#include <utility> +#include <vector> + +#include "common/logging.h" +#include "core/hle/service/nvdrv/devices/nvdevice.h" + +namespace Service::Nvidia::Devices { + +// Set once a Switch 2 NvRm has identified itself (0xD0-byte GetCharacteristics). Devices then +// answer ioctls they don't know with a logged, input-echoing success instead of NotImplemented. +inline std::atomic<bool> g_nx2_driver{false}; + +inline NvResult Nx2UnknownIoctl(const char* device, Ioctl command, std::span<const u8> input, + std::span<u8> output) { + LOG_WARNING(Service_NVDRV, "(STUBBED) Switch 2 {} ioctl={:08X} group={:01X} command={:02X} in={:#x}", + device, command.raw, command.group, command.cmd, input.size()); + std::fill(output.begin(), output.end(), u8{0}); + std::copy_n(input.begin(), std::min(input.size(), output.size()), output.begin()); + return NvResult::Success; +} + +// GPU mappings a Switch 2 process made through nx2:gpu: CreateGpuAccessibleMemory (sub 101) maps +// the memory at GPU address == CPU address, and NormalMapGpuMemory4K (211) maps a memory handle at +// a chosen GPU address (also in sparse regions). Every nvhost-as-gpu address space applies each +// mapping, including ones made before the address space existed. +class Nx2GpuMemoryRegistry { +public: + using Listener = std::function<void(u64 gpu_address, u64 cpu_address, u64 size)>; + using UnmapListener = std::function<void(u64 gpu_address, u64 size)>; + using ReleaseListener = std::function<void(u64 cpu_address, u64 size)>; + + static Nx2GpuMemoryRegistry& Get() { + static Nx2GpuMemoryRegistry registry; + return registry; + } + + void Register(u64 address, u64 size) { + MapAt(address, address, size); + } + + void MapAt(u64 gpu_address, u64 cpu_address, u64 size) { + std::scoped_lock lk{mutex}; + TrimMappings(gpu_address, size); + mappings.push_back({gpu_address, cpu_address, size}); + for (auto& [owner, listener] : listeners) { + listener.map(gpu_address, cpu_address, size); + } + } + + // NormalUnmapGpuMemory4K, or destroying the memory: the GPU range stops reaching it. + void Unmap(u64 gpu_address, u64 size) { + std::scoped_lock lk{mutex}; + TrimMappings(gpu_address, size); + for (auto& [owner, listener] : listeners) { + listener.unmap(gpu_address, size); + } + } + + // Backing memory [cpu_address, +size) is about to be freed (its GPU mappings are gone): each + // address space drops its SMMU mapping of it, in GPU-thread order. + void ReleaseBacking(u64 cpu_address, u64 size) { + { + std::scoped_lock lk{device_mutex}; + device_ranges.erase(cpu_address); + } + std::scoped_lock lk{mutex}; + for (auto& [owner, listener] : listeners) { + if (listener.release) { + listener.release(cpu_address, size); + } + } + } + + void AddListener(const void* owner, Listener listener, UnmapListener unmap_listener, + ReleaseListener release_listener = {}) { + std::scoped_lock lk{mutex}; + for (const auto& m : mappings) { + listener(m.gpu_address, m.cpu_address, m.size); + } + listeners.insert_or_assign(owner, Listeners{std::move(listener), std::move(unmap_listener), + std::move(release_listener)}); + } + + void RemoveListener(const void* owner) { + std::scoped_lock lk{mutex}; + listeners.erase(owner); + } + + // Device (SMMU) address of a CPU range an address space mapped; used to present images. + void RecordDeviceRange(u64 cpu_address, u64 size, u64 device_address) { + std::scoped_lock lk{device_mutex}; + device_ranges[cpu_address] = {size, device_address}; + } + + std::optional<u64> DeviceAddressOf(u64 cpu_address) { + std::scoped_lock lk{device_mutex}; + auto it = device_ranges.upper_bound(cpu_address); + if (it == device_ranges.begin()) { + return std::nullopt; + } + --it; + const auto& [start, range] = *it; + if (cpu_address - start >= range.first) { + return std::nullopt; + } + return range.second + (cpu_address - start); + } + +private: + // Separate lock: ranges are recorded from listeners that run under `mutex`. + std::mutex device_mutex; + std::map<u64, std::pair<u64, u64>> device_ranges; // cpu start -> (size, device address) + + struct Mapping { + u64 gpu_address; + u64 cpu_address; + u64 size; + }; + + struct Listeners { + Listener map; + UnmapListener unmap; + ReleaseListener release; + }; + + // Drops [gpu_address, +size) from the recorded mappings, keeping the parts outside it. + void TrimMappings(u64 gpu_address, u64 size) { + const u64 end = gpu_address + size; + std::vector<Mapping> kept; + kept.reserve(mappings.size() + 1); + for (const auto& m : mappings) { + const u64 m_end = m.gpu_address + m.size; + if (m_end <= gpu_address || m.gpu_address >= end) { + kept.push_back(m); + continue; + } + if (m.gpu_address < gpu_address) { + kept.push_back({m.gpu_address, m.cpu_address, gpu_address - m.gpu_address}); + } + if (m_end > end) { + kept.push_back({end, m.cpu_address + (end - m.gpu_address), m_end - end}); + } + } + mappings = std::move(kept); + } + + std::mutex mutex; + std::vector<Mapping> mappings; + std::map<const void*, Listeners> listeners; +}; + +// Device node that Switch 2 nvgpu drivers open but Switch 1 never had (e.g. /dev/nvhost-tsg-gpu). +// Every ioctl succeeds with a zeroed reply and is logged, so the driver's use of it shows up. +class nx2_stub_device final : public nvdevice { +public: + explicit nx2_stub_device(Core::System& system_, std::string name_) + : nvdevice{system_}, name{std::move(name_)} {} + + NvResult Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input, + std::span<u8> output) override { + return Stub(fd, command, input, output, {}); + } + NvResult Ioctl2(DeviceFD fd, Ioctl command, std::span<const u8> input, + std::span<const u8> inline_input, std::span<u8> output) override { + return Stub(fd, command, input, output, {}); + } + NvResult Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> input, std::span<u8> output, + std::span<u8> inline_output) override { + return Stub(fd, command, input, output, inline_output); + } + + void OnOpen(NvCore::SessionId session_id, DeviceFD fd) override { + LOG_WARNING(Service_NVDRV, "(STUBBED) {} opened as fd {}", name, fd); + } + void OnClose(DeviceFD fd) override {} + +private: + NvResult Stub(DeviceFD fd, Ioctl command, std::span<const u8> input, std::span<u8> output, + std::span<u8> inline_output) { + LOG_WARNING(Service_NVDRV, "(STUBBED) {} fd={} ioctl={:08X} group={:01X} command={:02X} in={:#x}", + name, fd, command.raw, command.group, command.cmd, input.size()); + // Echo the input (in/out ioctls keep their fields), zero the rest. + std::fill(output.begin(), output.end(), u8{0}); + std::copy_n(input.begin(), std::min(input.size(), output.size()), output.begin()); + std::fill(inline_output.begin(), inline_output.end(), u8{0}); + return NvResult::Success; + } + + std::string name; +}; + +} // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/nvdrv.cpp b/src/core/hle/service/nvdrv/nvdrv.cpp index b252a2efa5..5cac2693c5 100644 --- a/src/core/hle/service/nvdrv/nvdrv.cpp +++ b/src/core/hle/service/nvdrv/nvdrv.cpp @@ -23,6 +23,7 @@ #include "core/hle/service/nvdrv/devices/nvhost_nvjpg.h" #include "core/hle/service/nvdrv/devices/nvhost_vic.h" #include "core/hle/service/nvdrv/devices/nvmap.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/hle/service/nvdrv/nvdrv.h" #include "core/hle/service/nvdrv/nvdrv_interface.h" #include "core/hle/service/nvdrv/nvmemp.h" @@ -125,6 +126,11 @@ void LoopProcess(Core::System& system) { Module::Module(Core::System& system) : container{system.Host1x()}, service_context{system, "nvdrv"}, events_interface{*this} { + nx2_stub_builder = [this, &system](DeviceFD fd, const std::string& name) { + auto device = std::make_shared<Devices::nx2_stub_device>(system, name); + open_files.emplace(fd, device); + return std::static_pointer_cast<Devices::nvdevice>(device); + }; builders["/dev/nvhost-as-gpu"] = [this, &system](DeviceFD fd) { auto device = std::make_shared<Devices::nvhost_as_gpu>(system, *this, container); return open_files.emplace(fd, std::move(device)).first; @@ -181,6 +187,15 @@ NvResult Module::VerifyFD(DeviceFD fd) const { DeviceFD Module::Open(const std::string& device_name, NvCore::SessionId session_id) { auto it = builders.find(device_name); + if (it == builders.end() && device_name.starts_with("/dev/nvhost-")) { + // Newer nvgpu nodes used by Switch 2 drivers (e.g. /dev/nvhost-tsg-gpu): serve them with + // a logged stub instead of failing the open. + LOG_WARNING(Service_NVDRV, "Opening unknown device {} as a Switch 2 stub", device_name); + const DeviceFD fd = next_fd++; + auto device = nx2_stub_builder(fd, device_name); + device->OnOpen(session_id, fd); + return fd; + } if (it == builders.end()) { LOG_ERROR(Service_NVDRV, "Trying to open unknown device {}", device_name); return INVALID_NVDRV_FD; diff --git a/src/core/hle/service/nvdrv/nvdrv.h b/src/core/hle/service/nvdrv/nvdrv.h index f503f108c3..ba8ab6a40e 100644 --- a/src/core/hle/service/nvdrv/nvdrv.h +++ b/src/core/hle/service/nvdrv/nvdrv.h @@ -98,6 +98,8 @@ private: friend class EventInterface; /// Manages syncpoints on the host + /// Builds a logged stub for nvgpu device nodes only Switch 2 drivers use. + std::function<std::shared_ptr<Devices::nvdevice>(DeviceFD, const std::string&)> nx2_stub_builder; NvCore::Container container; /// Id to use for the next open file descriptor. diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.cpp b/src/core/hle/service/nvdrv/nvdrv_interface.cpp index 5fae68adbb..020036d74b 100644 --- a/src/core/hle/service/nvdrv/nvdrv_interface.cpp +++ b/src/core/hle/service/nvdrv/nvdrv_interface.cpp @@ -34,6 +34,7 @@ void NVDRV::Open(HLERequestContext& ctx) { const auto& buffer = ctx.ReadBuffer(); const std::string device_name(Common::StringFromBuffer(buffer)); + LOG_DEBUG(Service_NVDRV, "device_name={}", device_name); if (device_name == "/dev/nvhost-prof-gpu") { rb.Push<DeviceFD>(0); @@ -181,6 +182,14 @@ void NVDRV::Initialize(HLERequestContext& ctx) { is_initialized = true; } +void NVDRV::InitializeForProcess(Kernel::KProcess* process) { + if (is_initialized) { + return; + } + session_id = nvdrv->GetContainer().OpenSession(process); + is_initialized = true; +} + void NVDRV::QueryEvent(HLERequestContext& ctx) { IPC::RequestParser rp{ctx}; const auto fd = rp.Pop<DeviceFD>(); diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.h b/src/core/hle/service/nvdrv/nvdrv_interface.h index f628fca7df..8fe3196176 100644 --- a/src/core/hle/service/nvdrv/nvdrv_interface.h +++ b/src/core/hle/service/nvdrv/nvdrv_interface.h @@ -23,6 +23,9 @@ public: return nvdrv; } + // For Switch 2 nvsrv sessions, which are set up through gsv instead of Initialize. + void InitializeForProcess(Kernel::KProcess* process); + private: void Open(HLERequestContext& ctx); void Ioctl1(HLERequestContext& ctx); diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h new file mode 100644 index 0000000000..740b5627af --- /dev/null +++ b/src/core/hle/service/nx2_gsv.h @@ -0,0 +1,1647 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <algorithm> +#include <array> +#include <thread> +#include <chrono> +#include <string> +#include <cstdlib> +#include <cstdio> +#include <initializer_list> +#include <atomic> +#include <deque> +#include <map> +#include <optional> +#include <memory> +#include <mutex> +#include <vector> + +#include "common/alignment.h" +#include "common/settings.h" +#include "core/core.h" +#include "video_core/rasterizer_interface.h" +#include "video_core/renderer_base.h" +#include "core/frontend/framebuffer_layout.h" +#include "core/perf_stats.h" +#include "video_core/gpu.h" +#include "video_core/framebuffer_config.h" +#include "core/core_timing.h" +#include "core/hle/kernel/k_client_session.h" +#include "core/hle/kernel/k_process.h" +#include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/k_system_resource.h" +#include "core/hle/kernel/k_memory_manager.h" +#include "core/hle/kernel/k_page_group.h" +#include "core/hle/kernel/k_resource_limit.h" +#include "core/hle/kernel/k_server_session.h" +#include "core/hle/kernel/k_event.h" +#include "core/hle/kernel/k_readable_event.h" +#include "core/hle/kernel/k_session.h" +#include "core/hle/kernel/k_transfer_memory.h" +#include "core/memory.h" +#include "video_core/host1x/host1x.h" +#include "video_core/nx2.h" +#include "core/hle/service/hle_ipc.h" +#include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/kernel_helpers.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" +#include "core/hle/service/nvdrv/nvdrv.h" +#include "core/hle/service/nvdrv/nvdrv_interface.h" +#include "core/hle/service/nx2_stub_interface.h" +#include "core/hle/service/sm/sm.h" +#include "core/hle/service/server_manager.h" +#include "core/hle/service/service.h" + +namespace Service { + +// Creates a session served by `iface` and returns its client end. Unlike PushIpcInterface this +// works from a domain too, for callers that want a raw session handle. +template <class T> +inline Kernel::KClientSession* CreateRawSession(Core::System& system, HLERequestContext& ctx, + std::shared_ptr<T> iface) { + auto& kernel = system.Kernel(); + ASSERT(Kernel::GetCurrentProcess(kernel).GetResourceLimit()->Reserve( + kernel, Kernel::LimitableResource::SessionCountMax, 1)); + + auto* session = Kernel::KSession::Create(kernel); + session->Initialize(kernel, nullptr, 0); + Kernel::KSession::Register(kernel, session); + + auto manager = + std::make_shared<SessionRequestManager>(kernel, ctx.GetManager()->GetServerManager()); + manager->SetSessionHandler(std::move(iface)); + ctx.GetManager()->GetServerManager().RegisterSession(&session->GetServerSession(), manager); + return &session->GetClientSession(); +} + +// Replies to a TIPC request with success and raw u32 data words. ResponseBuilder::Push(Result) +// writes the CMIF two-word result, which shifts TIPC data one word past where the client reads +// it (and past the copied size); TIPC has a one-word result followed directly by the data. +inline void TipcReplyWords(HLERequestContext& ctx, std::initializer_list<u32> words) { + IPC::ResponseBuilder rb{ctx, static_cast<u32>(2 + words.size())}; + rb.PushRaw<u32>(ResultSuccess.raw); + for (const u32 w : words) { + rb.Push<u32>(w); + } +} + +// Replies with success and a copied handle to a new session served by `iface`. +template <class T> +inline void ReturnRawSession(Core::System& system, HLERequestContext& ctx, std::shared_ptr<T> iface) { + auto* client = CreateRawSession(system, ctx, std::move(iface)); + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, *client); +} + +// Placeholder for the Switch 2 NVIDIA driver server (nn::nvsrv). It has no handlers, so every +// command (CMIF or TIPC) gets an empty success and is logged through the nx2: prefix. +class INx2NvsrvSession final : public ServiceFramework<INx2NvsrvSession> { +public: + explicit INx2NvsrvSession(Core::System& system_) : ServiceFramework{system_, "nx2:nvsrv"} {} +}; + +// Placeholder for the Switch 2 syncpoint server (nn::syncpt), handed out by gsv 202. TIPC 1 +// returns a u32 the SDK requires to be < 0x400 and then passes min(value, 63) to TIPC 21. +// Everything else is auto-stubbed through the nx2: prefix. +// - TIPC 0(u64): handshake, echo the token (newer SDKs). +// - TIPC 22(u32 slot): an event handle for that slot (copy handle at word 3, result at word 4). +// NVN's NvOsEventPool asks for slots and multi-waits on them. +// - TIPC 201(slot, counter, u64 value, mode): signal the slot's event once the counter reaches +// value. The counter is a SyncCounterId, the bitwise NOT of the host1x syncpoint id; mode 0 +// compares 32-bit wrapping, mode 1 64-bit. Eden's guest syncpoints back it. +// - TIPC 202(slot): cancel the slot's request. +class INx2SyncptSession final : public ServiceFramework<INx2SyncptSession> { +public: + explicit INx2SyncptSession(Core::System& system_) + : ServiceFramework{system_, "nx2:syncpt"}, m_context{system_, "nx2:syncpt"} { + RegisterHandlersTipc({ + {0, &INx2SyncptSession::Handshake, "Handshake"}, + {1, &INx2SyncptSession::GetCount, "GetCount (guessed)"}, + {22, &INx2SyncptSession::GetSlotEvent, "GetSlotEvent (guessed)"}, + {103, &INx2SyncptSession::QueryCounter, "QueryCounter"}, + {201, &INx2SyncptSession::RequestSlotEvent, "RequestSlotEvent (guessed)"}, + {202, &INx2SyncptSession::CancelSlotEvent, "CancelSlotEvent (guessed)"}, + }); + } + + ~INx2SyncptSession() override { + // A registered action may be running on the GPU thread right now: mark the session dead + // under the lock it signals under, so it can't touch a request or event closed below. + { + std::scoped_lock lk{m_shared->mutex}; + m_shared->alive = false; + } + for (auto& [slot, request] : m_requests) { + Cancel(request); + } + for (auto& [slot, event] : m_events) { + m_context.CloseEvent(event); + } + } + +private: + // Newer SDKs (title 1) open with TIPC 0(u64 token) and abort unless the reply echoes it. + void Handshake(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + LOG_INFO(Service, "nx2:syncpt TIPC 0 handshake {:#x}", u64{w[2]} | (u64{w[3]} << 32)); + TipcReplyWords(ctx, {w[2], w[3]}); + } + + void GetCount(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:syncpt TIPC 1, returning 32"); + TipcReplyWords(ctx, {32}); + } + + // From changes.patch: the counter is the bitwise NOT of the host1x syncpoint id. + void QueryCounter(HLERequestContext& ctx) { + const u32 syncpoint = ~ctx.CommandBuffer()[2]; + const u64 value = syncpoint < HostSyncpoints + ? system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(syncpoint) + : 0; + LOG_DEBUG(Service, "nx2:syncpt 103 syncpoint={} value={:#x}", syncpoint, value); + TipcReplyWords(ctx, {static_cast<u32>(value), static_cast<u32>(value >> 32), 0}); + } + + void GetSlotEvent(HLERequestContext& ctx) { + const u32 slot = ctx.CommandBuffer()[2]; + LOG_WARNING(Service, "(STUBBED) nx2:syncpt TIPC 22 slot={:#x}, returning an event", slot); + auto*& event = m_events[slot]; + if (event == nullptr) { + event = m_context.CreateEvent(fmt::format("nx2:syncpt:Slot{:x}", slot)); + } + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, event->GetReadableEvent()); + } + + // Map nodes are stable, so a registered action can hold a reference to its request. + struct SlotRequest { + u32 syncpoint{}; + std::atomic<bool> pending{}; + Tegra::Host1x::SyncpointManager::ActionHandle handle{}; + }; + + // Outlives the session (actions hold a reference): signalling happens under `mutex` and only + // while `alive`. + struct SharedState { + std::mutex mutex; + bool alive{true}; + }; + + void RequestSlotEvent(HLERequestContext& ctx) { + const auto* cmd_buf = ctx.CommandBuffer(); + const u32 slot = cmd_buf[2]; + const u32 syncpoint = ~cmd_buf[3]; + const u64 value = u64{cmd_buf[4]} | (u64{cmd_buf[5]} << 32); + const u32 mode = cmd_buf[6]; + LOG_INFO(Service, "nx2:syncpt TIPC 201 slot={:#x} syncpoint={} value={:#x} mode={}", slot, + syncpoint, value, mode); + auto*& event = m_events[slot]; + if (event == nullptr) { + event = m_context.CreateEvent(fmt::format("nx2:syncpt:Slot{:x}", slot)); + } + if (syncpoint >= HostSyncpoints) { + LOG_WARNING(Service, "nx2:syncpt TIPC 201 syncpoint {} out of range, signalling", + syncpoint); + event->Signal(system.Kernel()); + TipcReplyWords(ctx, {}); + return; + } + auto& request = m_requests[slot]; + Cancel(request); + request.syncpoint = syncpoint; + request.pending = true; + auto& manager = system.Host1x().GetSyncpointManager(); + // The action runs on the GPU thread under the manager's lock, or inline if the value is + // already reached. + request.handle = manager.RegisterGuestAction( + syncpoint, static_cast<u32>(value), + [shared = m_shared, &kernel = system.Kernel(), &request, event] { + std::scoped_lock lk{shared->mutex}; + if (!shared->alive) { + return; + } + request.pending = false; + event->Signal(kernel); + }); + TipcReplyWords(ctx, {}); + } + + void CancelSlotEvent(HLERequestContext& ctx) { + const u32 slot = ctx.CommandBuffer()[2]; + LOG_INFO(Service, "nx2:syncpt TIPC 202 slot={:#x}", slot); + if (auto it = m_requests.find(slot); it != m_requests.end() && Cancel(it->second)) { + // The client waits on the slot's event after a cancel (NvOsEventPool, after its + // ~100 ms wait times out), so a cancel must signal it like a completion; it rechecks + // the counter (TIPC 103) itself. Title 1 hung forever there when the GPU thread ran + // a frame behind (EDEN_NX2_FRAMES_AHEAD) and a mid-frame wait outlasted 100 ms. + if (const auto event = m_events.find(slot); event != m_events.end()) { + event->second->Signal(system.Kernel()); + } + } + TipcReplyWords(ctx, {}); + } + + /// Returns true if the request was still pending. + bool Cancel(SlotRequest& request) { + if (request.pending.exchange(false)) { + system.Host1x().GetSyncpointManager().DeregisterGuestAction(request.syncpoint, + request.handle); + return true; + } + return false; + } + + static constexpr u32 HostSyncpoints = 192; // Tegra::Host1x::SyncpointManager + + KernelHelpers::ServiceContext m_context; + std::map<u32, Kernel::KEvent*> m_events; + std::map<u32, SlotRequest> m_requests; + std::shared_ptr<SharedState> m_shared = std::make_shared<SharedState>(); +}; + +// Placeholder for the Switch 2 GPU session (nn::gpu::sdk), handed out by gsv 497. +// - TIPC 11: [first, last] id range (first <= min(last, 64)); ids outside it are masked out. +// - TIPC 12(id): event handle for that id (handle at word 3, result at word 4); the SDK attaches +// it as a SystemEvent. +// - TIPC 21(id): starts an operation; the SDK then waits on the id's event. Nothing runs here, so +// the event is signalled at once. +// - TIPC 22: another handle the SDK keeps like a session; a placeholder session is returned. +class INx2GpuSession final : public ServiceFramework<INx2GpuSession> { +public: + explicit INx2GpuSession(Core::System& system_) + : ServiceFramework{system_, "nx2:gpu"}, m_context{system_, "nx2:gpu"} { + RegisterHandlersTipc({ + {11, &INx2GpuSession::GetIdRange, "GetIdRange (guessed)"}, + {12, &INx2GpuSession::GetIdEvent, "GetIdEvent (guessed)"}, + {21, &INx2GpuSession::Start, "Start (guessed)"}, + {22, &INx2GpuSession::OpenSubSession, "OpenSubSession (guessed)"}, + {33, &INx2GpuSession::GetCount33, "Unknown33 (u32 count)"}, + {201, &INx2GpuSession::GetUserPartition, "GetUserPartition (newer SDKs)"}, + {202, &INx2GpuSession::GetGsvPartition, "GetGsvPartition (newer SDKs)"}, + {203, &INx2GpuSession::GetVaRange, "GetVaRange (guessed)"}, + // Sub-session commands (gpu 22 hands out a second session to this same object). + {101, &INx2GpuSession::SubRegisterMemory, "CreateGpuAccessibleMemory"}, + {102, &INx2GpuSession::SubDestroyMemory, "DestroyGpuAccessibleMemory (guessed)"}, + {211, &INx2GpuSession::SubSubmitJob, "NormalMapGpuMemory4K"}, + {213, &INx2GpuSession::SubUnmapJob, "NormalUnmapGpuMemory4K"}, + {214, &INx2GpuSession::SubCreateGpuOnlyRegion, "CreateSparseRegion"}, + {215, &INx2GpuSession::SubDestroySparseRegion, "DestroySparseRegion"}, + }); + } + + ~INx2GpuSession() override { + for (auto* event : m_events) { + if (event != nullptr) { + m_context.CloseEvent(event); + } + } + } + +private: + static constexpr u32 NumIds = 32; + + Kernel::KEvent* EventFor(u32 id) { + id %= NumIds; + if (m_events[id] == nullptr) { + m_events[id] = m_context.CreateEvent(fmt::format("nx2:gpu:Event{}", id)); + } + return m_events[id]; + } + + void GetIdRange(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 11, returning [0, {}]", NumIds - 1); + TipcReplyWords(ctx, {0, NumIds - 1}); + } + + void GetIdEvent(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 12 id={}", id); + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, EventFor(id)->GetReadableEvent()); + } + + void Start(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 21 id={}, signalling completion", id); + EventFor(id)->Signal(system.Kernel()); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void GetCount33(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 33 [2]={:#x}, returning 1", ctx.CommandBuffer()[2]); + TipcReplyWords(ctx, {1}); + } + + // Used by nn::gpu::sdk::CreateGpuAddressSpace: three u64s after the result, a kind (< 5, 0 = + // none) and the GPU VA range of the in-process VirtualMachine that serves AllocSpace + // (nn::gpu::sdk::util::ExecuteVirtualMachineAllocSpace). NVN reserves fixed VA from 0x8000000 + // up to 512 GiB, so report the same range NvRm gives AllocAsEx: [0x8000000, 0xC000000000). + void GetVaRange(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 203 [2]={:#x}", ctx.CommandBuffer()[2]); + constexpr u64 kind = 1; + constexpr u64 start = 0x8000000ULL; + constexpr u64 size = 0xC000000000ULL - start; + TipcReplyWords(ctx, {static_cast<u32>(kind), static_cast<u32>(kind >> 32), + static_cast<u32>(start), static_cast<u32>(start >> 32), + static_cast<u32>(size), static_cast<u32>(size >> 32)}); + } + + // Newer SDKs (title 1) query the address space's user and gsv partitions with 201/202(id): + // two u64s each, stored as the partition's min/max. The user value works whether it is read + // as (start, size) or (min, max). Title 4 never needed the gsv partition, so it stays empty. + void GetUserPartition(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 201 [2]={:#x}", ctx.CommandBuffer()[2]); + constexpr u64 first = 0x8000000ULL; + constexpr u64 second = 0xC000000000ULL - first; + TipcReplyWords(ctx, {static_cast<u32>(first), static_cast<u32>(first >> 32), + static_cast<u32>(second), static_cast<u32>(second >> 32)}); + } + + void GetGsvPartition(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 202 [2]={:#x}, empty", ctx.CommandBuffer()[2]); + TipcReplyWords(ctx, {0, 0, 0, 0}); + } + + void OpenSubSession(HLERequestContext& ctx) { + // Served by this object so sub-session jobs can signal the per-id events. + LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 22, returning a sub-session on nx2:gpu"); + ReturnRawSession(system, ctx, std::static_pointer_cast<INx2GpuSession>(shared_from_this())); + } + + // Sub 101: (start u64, last u64, flags...) after CreateGpuAccessibleMemory's svc 0x80. + void SubRegisterMemory(HLERequestContext& ctx) { + const auto* b = ctx.CommandBuffer(); + const u64 start = u64{b[2]} | (u64{b[3]} << 32); + const u64 last = u64{b[4]} | (u64{b[5]} << 32); + LOG_INFO(Service, "nx2:gpu sub 101 start={:#x} last={:#x} [6]={:#x} [7]={:#x}", start, + last, b[6], b[7]); + if (last > start) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Register(start, last - start + 1); + // Two-page regions hold queue semaphores: GPU releases go to the second page, waits + // read the first (see video_core/nx2.h). + if (last - start + 1 == 0x2000) { + Tegra::Nx2AddSemaphoreRegion(start); + LOG_INFO(Service, "nx2:gpu semaphore page {:#x} mirrored to {:#x}", start + 0x1000, + start); + } + } + // nn::gpu::sdk::CreateGpuAccessibleMemory stores word 3 of the reply as the + // GpuMemoryHandle that later map jobs (211) name. + const u32 handle = m_next_memory_handle++; + m_memory[handle] = {start, last - start + 1}; + TipcReplyWords(ctx, {handle}); + } + + // Jobs that complete through a job-slot event (word Slot of the request), like 211. + template <size_t Slot> + void SubSlotJob(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + LOG_INFO(Service, "(STUBBED) nx2:gpu {} words {:08x} slot={}", ctx.GetCommand(), + fmt::join(std::span<const u32>{w + 2, 18}, " "), w[Slot]); + EventFor(w[Slot])->Signal(system.Kernel()); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Sub 211: job submission (nn::gpu::sdk job tracking). Word 2 is a constant id; word 17 is + // the job slot whose event (gpu TIPC 12) the SDK then waits on. Nothing runs here, so + // complete it at once. + void SubSubmitJob(HLERequestContext& ctx) { + const auto* cmd_buf = ctx.CommandBuffer(); + const u32 slot = cmd_buf[17]; + const u64 a = u64{cmd_buf[3]} | (u64{cmd_buf[4]} << 32); + const u64 b = u64{cmd_buf[5]} | (u64{cmd_buf[6]} << 32); + const u64 c = u64{cmd_buf[8]} | (u64{cmd_buf[9]} << 32); + // Words 3-4: first GPU address, 5-6: last, 7: memory handle, 8-9: offset into it. + const u32 handle = cmd_buf[7]; + LOG_INFO(Service, "nx2:gpu 211 map [{:#x}, {:#x}] handle={} offset={:#x} slot={}", a, b, + handle, c, slot); + if (const auto it = m_memory.find(handle); it != m_memory.end() && b > a) { + // Also at the memory's own address: an earlier 213 may have unmapped the mapping 101 + // made there. + const u64 cpu = it->second.address + c; + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().MapAt(a, cpu, b - a + 1); + } else { + LOG_WARNING(Service, "nx2:gpu 211 map of unknown handle {}", handle); + } + EventFor(slot)->Signal(system.Kernel()); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Sub 213: words 3-4 first GPU address, 5-6 last, job slot in word 8. + void SubUnmapJob(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u64 first = u64{w[3]} | (u64{w[4]} << 32); + const u64 last = u64{w[5]} | (u64{w[6]} << 32); + LOG_DEBUG(Service, "nx2:gpu 213 unmap [{:#x}, {:#x}] slot={}", first, last, w[8]); + if (last > first) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Unmap(first, last - first + 1); + } + SignalAfterGpu(w[8]); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Signals a job slot's event once the GPU thread has run everything queued so far, as the + // console's graphics server completes unmap jobs behind the submitted work. + void SignalAfterGpu(u32 slot) { + auto* const event = EventFor(slot); + system.GPU().RunInOrder([this, event] { event->Signal(system.Kernel()); }); + } + + // Sub 102: word 2 is the memory handle 101 returned, word 3 the job slot. The game frees the + // memory (svc 0x80 clears GPU access) and reuses it as ordinary heap right after, so nothing + // on the GPU side may keep it: drop the mapping 101 made at its own address. + void SubDestroyMemory(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u32 handle = w[2]; + if (const auto it = m_memory.find(handle); it != m_memory.end()) { + const auto [address, size] = it->second; + LOG_DEBUG(Service, "nx2:gpu 102 destroy handle={} [{:#x}, +{:#x}] slot={}", handle, + address, size, w[3]); + if (size != 0) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Unmap(address, size); + } + if (size == 0x2000) { + Tegra::Nx2RemoveSemaphoreRegion(address); + } + m_memory.erase(it); + } else { + LOG_WARNING(Service, "nx2:gpu 102 destroy of unknown handle {}", handle); + } + SignalAfterGpu(w[3]); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Sub 214: GPU-only memory [start, last] (words 3-6, job slot in word 11). The graphics + // server backs it itself: the client never maps anything there but renders into it (title 4 + // keeps its render targets in six 300 MB regions at 0x81_0000_0000). Back each region with + // kernel memory placed in the process's alias code region and map that for the GPU. + void SubCreateGpuOnlyRegion(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u64 start = u64{w[3]} | (u64{w[4]} << 32); + const u64 last = u64{w[5]} | (u64{w[6]} << 32); + const u32 slot = w[11]; + LOG_INFO(Service, "nx2:gpu 214 words {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}", + w[1], w[2], w[3], w[4], w[5], w[6], w[7], w[8], w[9], w[10], w[11], w[12]); + if (last > start && ((last + 1 - start) % Core::Memory::YUZU_PAGESIZE) == 0) { + const u64 size = last + 1 - start; + FreeReleasedRegions(); + if (const auto address = BackGpuOnlyRegion(start, size)) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().MapAt(start, *address, size); + LOG_INFO(Service, "nx2:gpu 214 GPU-only region [{:#x}, {:#x}] backed at {:#x}", + start, last, *address); + } + } else { + LOG_WARNING(Service, "nx2:gpu 214 unexpected region [{:#x}, {:#x}]", start, last); + } + EventFor(slot)->Signal(system.Kernel()); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Sub 215 (nn::gpu::sdk::DestroySparseRegion): words 3-6 the range, word 8 the job slot, like + // 213. Drop the GPU mapping, then the backing 214 added: its SMMU mapping (in GPU-thread + // order) and, once the GPU thread is past that, its pages. Kept allocated, title 1 ran the + // pool dry within minutes and later regions had no memory at all (GPU writes to them lost). + void SubDestroySparseRegion(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u64 first = u64{w[3]} | (u64{w[4]} << 32); + const u64 last = u64{w[5]} | (u64{w[6]} << 32); + LOG_INFO(Service, "nx2:gpu 215 destroy sparse region [{:#x}, {:#x}] words {:#x} {:#x} {:#x} {:#x}", + first, last, w[1], w[2], w[7], w[8]); + FreeReleasedRegions(); + if (last > first) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Unmap(first, last - first + 1); + const auto region = m_gpu_only_regions.find(first); + if (region != m_gpu_only_regions.end() && region->second.size == last - first + 1) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().ReleaseBacking( + region->second.cpu_address, region->second.size); + auto done = std::make_shared<std::atomic<bool>>(false); + system.GPU().RunInOrder([done] { done->store(true, std::memory_order_release); }); + m_released_regions.push_back({std::move(region->second), std::move(done)}); + m_gpu_only_regions.erase(region); + } + } + SignalAfterGpu(w[8]); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Returns the pages of destroyed regions the GPU thread is done with to the kernel. + void FreeReleasedRegions() { + auto* const process = system.ApplicationProcess(); + std::erase_if(m_released_regions, [&](ReleasedRegion& released) { + if (!released.done->load(std::memory_order_acquire)) { + return false; + } + auto& region = released.region; + if (process != nullptr && + R_FAILED(process->GetPageTable().UnmapPageGroup( + region.cpu_address, *region.pages, Kernel::KMemoryState::IoMemory))) { + LOG_ERROR(Service, "nx2:gpu could not unmap GPU-only backing {:#x}+{:#x}", + region.cpu_address, region.size); + return true; // still mapped: keep its pages + } + region.pages->Close(system.Kernel()); + return true; + }); + } + + std::optional<u64> BackGpuOnlyRegion(u64 gpu_address, u64 size) { + using Core::Memory::YUZU_PAGESIZE; + auto& kernel_core = system.Kernel(); + auto* const process = system.ApplicationProcess(); + if (process == nullptr) { + return std::nullopt; + } + auto pg = std::make_unique<Kernel::KPageGroup>( + kernel_core, std::addressof(kernel_core.GetSystemSystemResource().GetBlockInfoManager())); + // The graphics server's own memory, not the application's: taken from the application + // pool, title 4's six regions left it 1.8 GB less than the kernel reports to it. Not + // cleared: the GPU writes these before reading them, and touching 300 MB would commit it + // on the host. + if (R_FAILED(kernel_core.MemoryManager().AllocateAndOpen( + pg.get(), size / YUZU_PAGESIZE, + Kernel::KMemoryManager::EncodeOption(Kernel::KMemoryManager::Pool::SystemNonSecure, + Kernel::KMemoryManager::Direction::FromBack)))) { + LOG_ERROR(Service, "nx2:gpu 214 could not allocate {:#x} bytes", size); + return std::nullopt; + } + auto& page_table = process->GetPageTable(); + const u64 region_start = GetInteger(page_table.GetAliasCodeRegionStart()); + const u64 region_size = page_table.GetAliasCodeRegionSize(); + constexpr u64 Alignment = 0x200000; // 2 MiB + for (u64 offset = 0; offset + size <= region_size; offset += Alignment) { + const Common::ProcessAddress address{region_start + offset}; + if (R_SUCCEEDED(page_table.MapPageGroup(address, *pg, Kernel::KMemoryState::IoMemory, + Kernel::KMemoryPermission::UserReadWrite))) { + m_gpu_only_regions[gpu_address] = {GetInteger(address), size, std::move(pg)}; + return GetInteger(address); + } + } + LOG_ERROR(Service, "nx2:gpu 214 no room for {:#x} bytes in the alias code region " + "[{:#x}, +{:#x})", size, region_start, region_size); + pg->Close(kernel_core); + return std::nullopt; + } + + struct GpuOnlyRegion { + u64 cpu_address; + u64 size; + std::unique_ptr<Kernel::KPageGroup> pages; + }; + struct ReleasedRegion { + GpuOnlyRegion region; + std::shared_ptr<std::atomic<bool>> done; ///< Set by the GPU thread past the region's last use + }; + std::map<u64, GpuOnlyRegion> m_gpu_only_regions; ///< By GPU address + std::vector<ReleasedRegion> m_released_regions; + + struct GpuMemory { + u64 address; + u64 size; + }; + + KernelHelpers::ServiceContext m_context; + std::array<Kernel::KEvent*, NumIds> m_events{}; + std::map<u32, GpuMemory> m_memory; + u32 m_next_memory_handle{1}; +}; + +// Native memory and images handed out by the Switch 2 graphics server (gsv 111/114/423/429/315), +// shared with the vi2 session so presents can find an image's memory. Command meanings follow +// the shared changes.patch (IGraphicsSupervisor), re-checked against title 4's SDK. +struct Nx2NativeObjects { + struct Memory { + u64 size{}; + u64 address{}; // CPU (== GPU) address once bound by 114/423 + }; + struct Image { + u32 memory{}; + std::array<u32, 16> preference{}; + u64 size{}; + bool window_pool{}; + }; + std::mutex mutex; + std::map<u32, Memory> memories; + std::map<u32, Image> images; + u32 next_memory{1}; + u32 next_image{1}; +}; + +// Block-linear layout of an InterModuleUserImagePreference (16 words): word 2 holds +// (width - 1) | (height - 1) << 16, word 4 the block height (log2 of GOB rows), word 5 the row +// pitch in 64-byte GOBs. Formula from changes.patch (native_image_layout.h); preferences outside +// it fall back to 4 bytes per pixel and 128-row blocks. +struct Nx2ImageLayout { + u32 width; + u32 height; + u64 pitch; + u32 block_height_log2; + u64 size; + u64 alignment; +}; + +inline Nx2ImageLayout Nx2CalculateImageLayout(const u32* preference) { + Nx2ImageLayout layout{}; + layout.width = (preference[2] & 0xFFFF) + 1; + layout.height = (preference[2] >> 16) + 1; + const u64 pitch_gobs = preference[5]; + const u32 block_height_log2 = preference[4]; + if (block_height_log2 <= 8 && pitch_gobs >= (u64{layout.width} * 4 + 63) / 64 && + pitch_gobs <= 4096) { + layout.pitch = pitch_gobs * 64; + layout.block_height_log2 = block_height_log2; + } else { + layout.pitch = Common::AlignUp(u64{layout.width} * 4, 64); + layout.block_height_log2 = 7; + } + const u64 block_rows = u64{1} << layout.block_height_log2; + layout.size = layout.pitch * Common::AlignUp(u64{layout.height}, block_rows); + layout.alignment = std::max<u64>(64 * block_rows, 0x1000); + return layout; +} + +// Placeholder for the Switch 2 display session (nn::vi2::sdk), handed out by gsv 305. Reply +// shapes read from the SDK: TIPC 1 -> u32 < 0x400, TIPC 3 -> u32 < 8, TIPC 11/31(u32) and +// 32(handle, u32) -> one handle at word 3 then the result. Handles returned here are events (a +// guess); everything else is auto-stubbed through the nx2: prefix. +class INx2Vi2Session final : public ServiceFramework<INx2Vi2Session> { +public: + explicit INx2Vi2Session(Core::System& system_, std::shared_ptr<Nx2NativeObjects> objects_) + : ServiceFramework{system_, "nx2:vi2"}, m_context{system_, "nx2:vi2"}, + m_objects{std::move(objects_)} { + RegisterHandlersTipc({ + {1, &INx2Vi2Session::ReturnOne, "Unknown1"}, + {3, &INx2Vi2Session::ReturnOne, "GetDisplayCount (guessed)"}, + {5, &INx2Vi2Session::GetTimestampRecorderCount, "GetTimestampRecorderCount (guessed)"}, + {11, &INx2Vi2Session::ReturnEvent, "Unknown11"}, + {31, &INx2Vi2Session::GetVsyncEvent, "GetVsyncEvent (guessed)"}, + {32, &INx2Vi2Session::GetVsyncEvent, "GetDisplayVsyncEvent (guessed)"}, + // Title 1 calls these ~75 times a second with the display name "default"; the + // auto stub's empty success reply works, but logged two lines per call (a 50 MB log + // in an 18 minute session). + {104, &INx2Vi2Session::SucceedQuietly, "Unknown104 (display \"default\")"}, + {105, &INx2Vi2Session::SucceedQuietly, "Unknown105 (display \"default\")"}, + {228, &INx2Vi2Session::CancelClientBuffer, "CancelClientBuffer"}, + {229, &INx2Vi2Session::PresentClientBuffer, "PresentClientBuffer"}, + }); + } + + ~INx2Vi2Session() override { + // The vsync callback may be running on the HostTiming thread: empty its event list under + // its lock first (it holds the state, not this session), then close the events. + { + std::scoped_lock lk{m_vsync->mutex}; + m_vsync->events.clear(); + } + if (m_vsync_timer) { + system.CoreTiming().UnscheduleEvent(m_vsync_timer); + } + for (auto* event : m_events) { + m_context.CloseEvent(event); + } + } + +private: + // 229(u64 client token, u32 slot, u32 image, ...fences): result, u64 mask of buffer slots the + // client may acquire next, four zero words. Nothing is shown yet; every slot is handed back + // at once (changes.patch's default mask is 0x7fff). + void PresentClientBuffer(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + ++m_presents; + Core::g_nx2_present_count.fetch_add(1, std::memory_order_relaxed); + if (m_presents <= 4 || (m_presents % 600) == 0) { + LOG_INFO(Service, "nx2:vi2 229 present #{} token={:#x} slot={} image={}", m_presents, + u64{w[2]} | (u64{w[3]} << 32), w[4], w[5]); + } + // Wait for the previous frame's fence instead of this one, so the game prepares frame N+1 + // while the GPU thread runs frame N (the console keeps several frames in flight). Waiting + // for this frame serialized the two: 8-56 ms of every 34-105 ms frame on title 1. + // EDEN_NX2_FRAMES_AHEAD=0 restores the wait for the current frame. + static const bool frames_ahead = [] { + const char* value = std::getenv("EDEN_NX2_FRAMES_AHEAD"); + return value == nullptr || value[0] != '0'; + }(); + const auto entered = std::chrono::steady_clock::now(); + if (frames_ahead) { + LOG_DEBUG(Service, "nx2:vi2 present #{} fence ({}, {:#x}), waiting for ({}, {:#x}); now {:#x}", + m_presents, ~w[22], w[24], m_previous_fence_id, m_previous_fence_value, + ~w[22] < 192 ? system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(~w[22]) : 0U); + WaitForFrame(m_previous_fence_id, m_previous_fence_value); + m_previous_fence_id = ~w[22]; + m_previous_fence_value = w[24]; + } else { + WaitForFrame(~w[22], w[24]); + } + const auto waited = std::chrono::steady_clock::now(); + Composite(w[5], ~w[22], w[24]); + const auto composited = std::chrono::steady_clock::now(); + PaceToVblank(); + const auto paced = std::chrono::steady_clock::now(); + // Where each frame's time goes on the present thread, averaged per 600 presents: the + // game is blocked here while Eden's GPU thread catches up (fence) and composites. + { + using us = std::chrono::microseconds; + const auto to_us = [](auto d) { return std::chrono::duration_cast<us>(d).count(); }; + m_stats.fence_us += to_us(waited - entered); + m_stats.composite_us += to_us(composited - waited); + m_stats.pace_us += to_us(paced - composited); + m_stats.max_fence_us = (std::max)(m_stats.max_fence_us, to_us(waited - entered)); + if (m_stats.last_present != std::chrono::steady_clock::time_point{}) { + const long long frame = to_us(entered - m_stats.last_present); + m_stats.frame_us += frame; + m_stats.max_frame_us = (std::max)(m_stats.max_frame_us, frame); + m_stats.hitches += frame > 100'000 ? 1 : 0; + } + m_stats.last_present = entered; + if (++m_stats.presents == 600) { + LOG_INFO(Service, + "nx2:vi2 last 600 presents: frame {} us (max {}, {} over 100 ms), " + "blocked on the GPU fence {} us (max {}), composite {} us, vblank " + "pacing {} us", + m_stats.frame_us / 599, m_stats.max_frame_us, m_stats.hitches, + m_stats.fence_us / 600, m_stats.max_fence_us, m_stats.composite_us / 600, + m_stats.pace_us / 600); + m_stats = {.last_present = entered}; + } + } + constexpr u64 AvailableSlots = 0x7fff; + TipcReplyWords(ctx, {static_cast<u32>(AvailableSlots), + static_cast<u32>(AvailableSlots >> 32), 0, 0, 0, 0}); + } + + // The console's compositor frees a buffer only once its frame's GPU fence has passed, which + // is what keeps a game from running ahead of the GPU. This reply always reports every buffer + // free, so pace the game here instead: wait (bounded, in case the GPU waits on the CPU) until + // Eden's GPU thread has processed the frame. Otherwise the game laps the GPU thread and + // rewrites command memory it hasn't read yet. + u32 m_wait_timeouts{}; + struct PresentStats { + long long frame_us{}, fence_us{}, composite_us{}, pace_us{}, max_fence_us{}; + long long max_frame_us{}, hitches{}; + u32 presents{}; + std::chrono::steady_clock::time_point last_present{}; + } m_stats{}; + u32 m_previous_fence_id{~0U}; // none yet (WaitForFrame ignores ids >= 192) + u32 m_previous_fence_value{}; + + // The console flips one frame per 60 Hz vblank and keeps the rest queued, so a game can't + // present faster than that. Every slot is reported free here: hold each present until the + // next vblank instead (title 4 ran at up to 160 fps, its timing and audio with it). + std::chrono::steady_clock::time_point m_next_vblank{}; + + void PaceToVblank() { + if (!Settings::values.use_speed_limit.GetValue() || Settings::SpeedLimit() == 0) { + return; + } + const auto interval = std::chrono::nanoseconds{100'000'000'000LL / 60 / Settings::SpeedLimit()}; + const auto now = std::chrono::steady_clock::now(); + if (now > m_next_vblank + interval) { + // Running slower than 60 fps: show at once and don't bank the lost time. + m_next_vblank = now; + } else if (now < m_next_vblank) { + std::this_thread::sleep_until(m_next_vblank); + } + m_next_vblank += interval; + } + + void WaitForFrame(u32 fence_id, u32 fence_value) { + if (fence_id >= 192) { + return; + } + auto& syncpoints = system.Host1x().GetSyncpointManager(); + const auto start = std::chrono::steady_clock::now(); + // Long enough to cover shader/pipeline compile stalls: a game that gets more than a few + // command buffers ahead misses the equal-value gates its other channels acquire. + const auto deadline = start + std::chrono::milliseconds{4000}; + if (!syncpoints.IsReadyGuest(fence_id, fence_value)) { + // Woken by the GPU thread as it reaches the fence. Polling with 200 us sleeps slept + // about 1 ms per check on Windows, added to every frame. An action left registered + // after a timeout fires later on a state nobody waits on. + struct Wake { + std::mutex mutex; + std::condition_variable cv; + bool done{}; + }; + auto wake = std::make_shared<Wake>(); + syncpoints.RegisterGuestAction(fence_id, fence_value, [wake] { + { + std::scoped_lock lk{wake->mutex}; + wake->done = true; + } + wake->cv.notify_all(); + }); + std::unique_lock lk{wake->mutex}; + while (!wake->done && system.IsPoweredOn() && + std::chrono::steady_clock::now() < deadline) { + wake->cv.wait_for(lk, std::chrono::milliseconds{50}); + } + } + if (!syncpoints.IsReadyGuest(fence_id, fence_value) && m_wait_timeouts++ < 64) { + LOG_WARNING(Service, "nx2:vi2 present #{} fence ({}, {}) not reached in 4 s (now {})", + m_presents, fence_id, fence_value, syncpoints.GetGuestSyncpointValue(fence_id)); + } + if (m_presents <= 8 || (m_presents % 600) == 0) { + LOG_INFO(Service, "nx2:vi2 present #{} fence ({}, {}) now {}, waited {} us", + m_presents, fence_id, fence_value, syncpoints.GetGuestSyncpointValue(fence_id), + std::chrono::duration_cast<std::chrono::microseconds>( + std::chrono::steady_clock::now() - start) + .count()); + } + } + + // Shows a presented native image: word 5 is the image id, ~word 22 / word 24 the GPU fence + // (syncpoint, value) to wait for (layout from changes.patch). The image is a block-linear + // RGBA8 surface in the console's native GOB layout at GPU address == CPU address. + void Composite(u32 image_id, u32 fence_id, u32 fence_value) { + u64 address{}; + Nx2ImageLayout layout{}; + { + std::scoped_lock lk{m_objects->mutex}; + const auto image = m_objects->images.find(image_id); + if (image == m_objects->images.end()) { + return; + } + const auto memory = m_objects->memories.find(image->second.memory); + if (memory == m_objects->memories.end() || memory->second.address == 0) { + return; + } + address = memory->second.address; + layout = Nx2CalculateImageLayout(image->second.preference.data()); + } + const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address); + if (!device) { + LOG_WARNING(Service, "nx2:vi2 present: image {} at {:#x} has no device mapping", + image_id, address); + return; + } + std::vector<Tegra::FramebufferConfig> layers{Tegra::FramebufferConfig{ + .address = *device, + .offset = 0, + .width = layout.width, + .height = layout.height, + .stride = static_cast<u32>(layout.pitch / 4), + .pixel_format = android::PixelFormat::Rgba8888, + .transform_flags = android::BufferTransformFlags::Unset, + .crop_rect = {0, 0, static_cast<int>(layout.width), static_cast<int>(layout.height)}, + .blending = Tegra::BlendMode::Opaque, + .native_tiling = true, + }}; + std::vector<Nvidia::NvFence> fences; + if (fence_id < 192) { + fences.push_back({static_cast<s32>(fence_id), fence_value}); + } + system.GPU().RequestComposite(std::move(layers), std::move(fences)); + CaptureFrame(); + DumpRanges(); + system.SpeedLimiter().DoSpeedLimiting(system.CoreTiming().GetGlobalTimeUs()); + system.GetPerfStats().EndSystemFrame(); + system.GetPerfStats().BeginSystemFrame(); + } + + // EDEN_NX2_FRAME_CAPTURE=<dir>: every 120th present, save what the renderer shows as + // nx2-frame-<n>.tga in <dir> (lets hidden test runs check the output; idea from changes.patch). + void CaptureFrame() { + static const char* const directory = std::getenv("EDEN_NX2_FRAME_CAPTURE"); + if (directory == nullptr || (m_presents % 120) != 1 || m_capture_pending->exchange(true)) { + return; + } + const auto layout = Layout::DefaultFrameLayout(1280, 720); + auto pixels = std::make_shared<std::vector<u8>>(layout.width * layout.height * 4); + const std::string path = + fmt::format("{}/nx2-frame-{:05}.tga", directory, m_presents); + auto pending = m_capture_pending; // the screenshot callback can outlive the session + system.GPU().Renderer().RequestScreenshot( + pixels->data(), + [pixels, layout, path, pending](bool invert_y) { + std::array<u8, 18> header{}; + header[2] = 2; // uncompressed true colour + header[12] = static_cast<u8>(layout.width); + header[13] = static_cast<u8>(layout.width >> 8); + header[14] = static_cast<u8>(layout.height); + header[15] = static_cast<u8>(layout.height >> 8); + header[16] = 32; + // Descriptor bit 5: rows top to bottom. invert_y means the data is bottom-up. + header[17] = invert_y ? 0x08 : 0x28; + if (std::FILE* file = std::fopen(path.c_str(), "wb")) { + std::fwrite(header.data(), 1, header.size(), file); + std::fwrite(pixels->data(), 1, pixels->size(), file); + std::fclose(file); + } + pending->store(false); + }, + layout); + } + + // EDEN_NX2_DUMP=<present>:<hex address>:<hex size>[,<hex address>:<hex size>...]: at that + // present, flush those GPU ranges back from the host GPU and save their bytes as + // nx2-dump-<address>.bin in the EDEN_NX2_FRAME_CAPTURE directory (to inspect what a pass + // rendered, e.g. a 3D volume, offline). + void DumpRanges() { + static const std::string spec = [] { + const char* text = std::getenv("EDEN_NX2_DUMP"); + return text ? std::string{text} : std::string{}; + }(); + if (spec.empty() || m_presents != std::strtoull(spec.c_str(), nullptr, 10)) { + return; + } + const char* const directory = std::getenv("EDEN_NX2_FRAME_CAPTURE"); + size_t position = spec.find(':'); + while (position != std::string::npos) { + char* end{}; + const u64 address = std::strtoull(spec.c_str() + position + 1, &end, 16); + const u64 size = *end == ':' ? std::strtoull(end + 1, &end, 16) : 0; + position = spec.find(',', position + 1); // the next "address:size" + const std::string path = fmt::format("{}/nx2-dump-{:x}.bin", directory ? directory : ".", address); + // On the GPU thread: waiting for it here can deadlock with a GPU wait on this present. + static_cast<void>(system.GPU().RunOnGpuThread([&system = system, address, size, path] { + std::vector<u8> bytes(size); + for (u64 current = address; current < address + size;) { + const u64 chunk = std::min((current | 0xfff) + 1, address + size) - current; + if (const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(current)) { + system.GPU().Renderer().ReadRasterizer()->FlushRegion(*device, chunk); + system.Host1x().MemoryManager().ReadBlock(*device, bytes.data() + (current - address), chunk); + } + current += chunk; + } + if (std::FILE* file = std::fopen(path.c_str(), "wb")) { + std::fwrite(bytes.data(), 1, bytes.size(), file); + std::fclose(file); + } + LOG_INFO(Service, "nx2: dumped {:#x} bytes at {:#x} to {}", size, address, path); + })); + } + } + + // 228(u64 token, u32 slot): return a buffer without presenting it. + void CancelClientBuffer(HLERequestContext& ctx) { + constexpr u64 AvailableSlots = 0x7fff; + TipcReplyWords(ctx, {static_cast<u32>(AvailableSlots), + static_cast<u32>(AvailableSlots >> 32), 0, 0, 0, 0}); + } + + void ReturnOne(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC {}, returning 1", ctx.GetCommand()); + TipcReplyWords(ctx, {1}); + } + + // The same reply as the auto stub (success, no data), without its log lines. + void SucceedQuietly(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Must be < 8; nn::vi::detail::LayerHistory creates 7 timestamp recorders per layer. + void GetTimestampRecorderCount(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC 5, returning 7"); + TipcReplyWords(ctx, {7}); + } + + void ReturnEvent(HLERequestContext& ctx) { + LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC {} [2]={:#x} [3]={:#x}, returning an event", + ctx.GetCommand(), ctx.CommandBuffer()[2], ctx.CommandBuffer()[3]); + auto* event = m_context.CreateEvent(fmt::format("nx2:vi2:Event{}", m_events.size())); + m_events.push_back(event); + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, event->GetReadableEvent()); + } + + // 31(u32 kind) and 32(u32 display): events signalled on every vblank. Title 1's main thread + // waits on the 32 event until it has fired twice before it submits any GPU work, and hung + // there while it never fired. + void GetVsyncEvent(HLERequestContext& ctx) { + LOG_INFO(Service, "nx2:vi2 {} vsync event, [2]={:#x}", ctx.GetCommand(), ctx.CommandBuffer()[2]); + auto* event = m_context.CreateEvent(fmt::format("nx2:vi2:Vsync{}", m_events.size())); + m_events.push_back(event); + { + std::scoped_lock lk{m_vsync->mutex}; + m_vsync->events.push_back(event); + } + if (!m_vsync_timer) { + m_vsync_timer = Core::Timing::CreateEvent( + "nx2:vi2 vsync", + [vsync_state = m_vsync, &kernel = system.Kernel()]( + s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> { + std::scoped_lock lk{vsync_state->mutex}; + for (auto* vsync : vsync_state->events) { + vsync->Signal(kernel); + } + return std::nullopt; + }); + // EDEN_NX2_VSYNC_HZ=<rate> (experiment): the game starts frames on this event, so a + // frame that misses a 60 Hz tick idles until the next one. + static const std::chrono::nanoseconds VblankInterval{[] { + const char* text = std::getenv("EDEN_NX2_VSYNC_HZ"); + const long long hz = text ? (std::max)(std::strtoll(text, nullptr, 10), 1LL) : 60LL; + return 1'000'000'000LL / hz; + }()}; + system.CoreTiming().ScheduleLoopingEvent(VblankInterval, VblankInterval, m_vsync_timer); + } + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, event->GetReadableEvent()); + } + + KernelHelpers::ServiceContext m_context; + std::shared_ptr<Nx2NativeObjects> m_objects; + std::vector<Kernel::KEvent*> m_events; + // Shared with the vsync timer callback, which can outlive the session by one call. + struct VsyncState { + std::mutex mutex; + std::vector<Kernel::KEvent*> events; + }; + std::shared_ptr<VsyncState> m_vsync = std::make_shared<VsyncState>(); + std::shared_ptr<Core::Timing::EventType> m_vsync_timer; + u64 m_presents{}; + std::shared_ptr<std::atomic<bool>> m_capture_pending = std::make_shared<std::atomic<bool>>(false); +}; + +// The Switch 2 video decoder (nn::codec::video::detail::ipc). gsv 902/1002 hand out a session +// whose TIPC 1 (CreateVideoDecoderService: handle at word 3, result at word 4) creates a decoder +// service, another session of this class. Layouts read from the SDK's nn::codec VP9 decoder: +// - 200(copy handle: work buffer transfer memory, u64 size, u32, u32, u32): initialize. +// - 101(X: codec configuration, u32 0, u32 width, u32 height, u32 rate, u32 max input): configure. +// - 3(): an event (copy handle at word 3, result at word 4) the client attaches as its +// submission-ready event. +// - 6 / 207 / 8: start / stop / flush; 115: finalize. +// - 311(u32 image id): register an output image (gsv 911 creates it); reply a 16-byte +// NativeSyncHandle, all zero reading as already signalled. The client acquires output images +// in registration/release order from a ring of its own (no IPC). +// - 314(u32 image id, u32, NativeSyncHandle): give an output image back; reply a new handle. +// - 212(u32 image id): unregister it. +// - 109(A: one compressed frame, u64 pts in us, u8 end of stream): submit input; reply a u64. +// - 213(): the next finished frame (16 words): u32 image id, u32 width, u32 height, u32 width, +// u32 height, u32 colour space, u64 pts, u8 end of stream. +// Frames are not decoded yet: each submitted frame completes at once into the next free output +// image, so movies run to their end (with no picture) instead of stalling. +class INx2VideoSession final : public ServiceFramework<INx2VideoSession> { +public: + explicit INx2VideoSession(Core::System& system_, std::shared_ptr<Nx2NativeObjects> objects_, + const char* name) + : ServiceFramework{system_, name}, m_context{system_, name}, + m_objects{std::move(objects_)} { + RegisterHandlersTipc({ + {1, &INx2VideoSession::CreateDecoderService, "CreateVideoDecoderService"}, + {3, &INx2VideoSession::GetEvent, "GetEvent"}, + {6, &INx2VideoSession::Succeed, "Start"}, + {8, &INx2VideoSession::Succeed, "Flush"}, + {101, &INx2VideoSession::Configure, "Configure"}, + {109, &INx2VideoSession::SubmitInput, "SubmitInputBuffer"}, + {115, &INx2VideoSession::Succeed, "Finalize"}, + {200, &INx2VideoSession::Succeed, "Initialize"}, + {207, &INx2VideoSession::Succeed, "Stop"}, + {212, &INx2VideoSession::UnregisterBuffer, "UnregisterBuffer"}, + {213, &INx2VideoSession::GetOutputInfo, "GetOutputInfo"}, + {311, &INx2VideoSession::RegisterBuffer, "RegisterBuffer"}, + {314, &INx2VideoSession::ReleaseBuffer, "ReleaseBuffer"}, + }); + } + + ~INx2VideoSession() override { + // The ready timer runs on the HostTiming thread: clear its event under its lock first. + { + std::scoped_lock lk{m_ready->mutex}; + m_ready->event = nullptr; + } + if (m_ready_timer) { + system.CoreTiming().UnscheduleEvent(m_ready_timer); + } + if (m_event != nullptr) { + m_context.CloseEvent(m_event); + } + } + +private: + struct Output { + u32 image; + u64 pts; + bool end; + }; + + void Log(HLERequestContext& ctx, std::string_view what) { + if (m_logged++ < 64) { + const auto* w = ctx.CommandBuffer(); + LOG_INFO(Service, "{} TIPC {} {}: [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x} [6]={:#x} [7]={:#x}", + GetServiceName(), ctx.GetCommand(), what, w[2], w[3], w[4], w[5], w[6], w[7]); + } + } + + void CreateDecoderService(HLERequestContext& ctx) { + Log(ctx, "create decoder service"); + ReturnRawSession(system, ctx, + std::make_shared<INx2VideoSession>(system, m_objects, "nx2:vdecsvc")); + } + + void Succeed(HLERequestContext& ctx) { + Log(ctx, ""); + TipcReplyWords(ctx, {}); + } + + void Configure(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + // Words 2-3 are the X descriptor of the configuration data. + m_width = w[5]; + m_height = w[6]; + Log(ctx, "configure"); + TipcReplyWords(ctx, {}); + } + + void GetEvent(HLERequestContext& ctx) { + Log(ctx, "event"); + if (m_event == nullptr) { + m_event = m_context.CreateEvent(fmt::format("{}:Ready", GetServiceName())); + // Ready for input at all times (frames complete as they are submitted). The client + // attaches it auto-clearing, so one signal is consumed by the first wait: re-signal + // every 4 ms; a spurious wake only makes it check again. + { + std::scoped_lock lk{m_ready->mutex}; + m_ready->event = m_event; + } + m_ready_timer = Core::Timing::CreateEvent( + "nx2:vdec ready", + [ready = m_ready, &kernel = system.Kernel()]( + s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> { + std::scoped_lock lk{ready->mutex}; + if (ready->event != nullptr) { + ready->event->Signal(kernel); + } + return std::nullopt; + }); + constexpr std::chrono::nanoseconds ReadyInterval{std::chrono::milliseconds{4}}; + system.CoreTiming().ScheduleLoopingEvent(ReadyInterval, ReadyInterval, m_ready_timer); + } + m_event->Signal(system.Kernel()); + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, m_event->GetReadableEvent()); + } + + void RegisterBuffer(HLERequestContext& ctx) { + Log(ctx, "register output"); + GiveBack(ctx.CommandBuffer()[2]); + TipcReplyWords(ctx, {0, 0, 0, 0}); + } + + void ReleaseBuffer(HLERequestContext& ctx) { + Log(ctx, "release output"); + GiveBack(ctx.CommandBuffer()[2]); + TipcReplyWords(ctx, {0, 0, 0, 0}); + } + + void UnregisterBuffer(HLERequestContext& ctx) { + Log(ctx, "unregister output"); + std::erase(m_free, ctx.CommandBuffer()[2]); + TipcReplyWords(ctx, {}); + } + + // An output image is free again: a frame waiting for one completes into it. + void GiveBack(u32 image) { + if (!m_waiting.empty()) { + m_outputs.push_back({image, m_waiting.front().pts, m_waiting.front().end}); + m_waiting.pop_front(); + return; + } + m_free.push_back(image); + } + + void SubmitInput(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + // Words 2-4 are the A descriptor of the compressed frame. + const u64 pts = u64{w[5]} | (u64{w[6]} << 32); + const bool end = (w[7] & 0xFF) != 0; + Log(ctx, "submit"); + if (!m_free.empty()) { + m_outputs.push_back({m_free.front(), pts, end}); + m_free.pop_front(); + } else { + m_waiting.push_back({0, pts, end}); + } + const u64 id = ++m_submitted; + if (m_event != nullptr) { + m_event->Signal(system.Kernel()); + } + TipcReplyWords(ctx, {static_cast<u32>(id), static_cast<u32>(id >> 32)}); + } + + void GetOutputInfo(HLERequestContext& ctx) { + Log(ctx, "output info"); + if (m_outputs.empty()) { + // Nothing finished: the code the client's own Acquire uses for an empty ring. + IPC::ResponseBuilder rb{ctx, 2}; + rb.PushRaw<u32>(0x1c286f); + return; + } + const Output output = m_outputs.front(); + m_outputs.pop_front(); + TipcReplyWords(ctx, {output.image, m_width, m_height, m_width, m_height, 0, + static_cast<u32>(output.pts), static_cast<u32>(output.pts >> 32), + output.end ? 1U : 0U, 0, 0, 0, 0, 0, 0, 0}); + } + + struct ReadyState { + std::mutex mutex; + Kernel::KEvent* event{}; + }; + + KernelHelpers::ServiceContext m_context; + std::shared_ptr<Nx2NativeObjects> m_objects; + Kernel::KEvent* m_event{}; + std::shared_ptr<ReadyState> m_ready = std::make_shared<ReadyState>(); + std::shared_ptr<Core::Timing::EventType> m_ready_timer; + u32 m_width{}; + u32 m_height{}; + u64 m_submitted{}; + u32 m_logged{}; + std::deque<u32> m_free; + std::deque<Output> m_waiting; + std::deque<Output> m_outputs; +}; + +// Placeholder for the Switch 2 graphics-system server that nn::gsv::BootWithRemoteMemory talks +// to over TIPC. Nothing behind it is emulated: commands succeed and are logged, so boot can show +// what comes next. Commands 202 and 1402 hand out the syncpt and nvsrv sessions the SDK uses. +class INx2GraphicsSystemSession final : public ServiceFramework<INx2GraphicsSystemSession> { +public: + explicit INx2GraphicsSystemSession(Core::System& system_) + : ServiceFramework{system_, "nx2:gsv"}, m_context{system_, "nx2:gsv"} { + RegisterHandlersTipc({ + {4, &INx2GraphicsSystemSession::Stub, "Boot (Switch 2, guessed)"}, + {5, &INx2GraphicsSystemSession::Stub, "Unknown5"}, + {101, &INx2GraphicsSystemSession::Stub, "Unknown101"}, + {202, &INx2GraphicsSystemSession::OpenSyncptSession, "OpenSyncptSession (guessed)"}, + {204, &INx2GraphicsSystemSession::AttachSyncptMemory, "AttachSyncptMemory (guessed)"}, + {305, &INx2GraphicsSystemSession::OpenViSession, "OpenViSession (guessed)"}, + {497, &INx2GraphicsSystemSession::OpenGpuSession, "OpenGpuSession (guessed)"}, + {111, &INx2GraphicsSystemSession::CreateNativeMemory, "CreateNativeMemory"}, + {112, &INx2GraphicsSystemSession::DestroyNativeMemory, "DestroyNativeMemory"}, + {114, &INx2GraphicsSystemSession::BindNativeMemory, "BindNativeMemory"}, + {314, &INx2GraphicsSystemSession::ConfigureWindowImagePool, "DestroyNativeWindowAliasImagePool"}, + {315, &INx2GraphicsSystemSession::ConfigureWindowImagePool, "CreateNativeWindowAliasImagePool"}, + {423, &INx2GraphicsSystemSession::BindNativeGpuMemory, "BindNativeGpuMemory"}, + {429, &INx2GraphicsSystemSession::CreateNativeImage, "CreateNativeImage"}, + {430, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyNativeImage"}, + {435, &INx2GraphicsSystemSession::CalculateNativeImageSize, "CalculateNativeImageSize"}, + {1402, &INx2GraphicsSystemSession::OpenNvsrvSession, "OpenNvsrvSession (guessed)"}, + {1510, &INx2GraphicsSystemSession::CalculateRequirements, "CalculateRequirements"}, + {991, &INx2GraphicsSystemSession::CalculateDecoderImagePoolBuffer, + "CalculateDecoderImagePoolBuffer"}, + {901, &INx2GraphicsSystemSession::Stub, "VideoInitialize (guessed)"}, + {902, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSession (guessed)"}, + {1001, &INx2GraphicsSystemSession::Stub, "VideoInitialize2 (guessed)"}, + {1002, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSession2 (guessed)"}, + // Same request as 429: the image backing a decoder output (nn::codec image pools). + {911, &INx2GraphicsSystemSession::CreateNativeImage, "CreateDecoderImage (guessed)"}, + }); + } + + ~INx2GraphicsSystemSession() override { + if (m_syncpt_poll) { + system.CoreTiming().UnscheduleEvent(m_syncpt_poll); + } + for (auto* event : m_syncpt_events) { + if (event != nullptr) { + m_context.CloseEvent(event); + } + } + } + +private: + // Index of the first raw data word of a TIPC request: header (2 words), then an optional + // special header with pid and copy/move handle words. + static size_t TipcDataOffset(const u32* cmd_buf) { + if ((cmd_buf[1] & 0x80000000) == 0) { + return 2; + } + const u32 desc = cmd_buf[2]; + return 3 + ((desc & 1) ? 2 : 0) + ((desc >> 1) & 0xF) + ((desc >> 5) & 0xF); + } + + void Stub(HLERequestContext& ctx) { + const auto* cmd_buf = ctx.CommandBuffer(); + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC {} [2]={:#x} [3]={:#x} [4]={:#x} [5]={:#x}", + ctx.GetCommand(), cmd_buf[2], cmd_buf[3], cmd_buf[4], cmd_buf[5]); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // nn::codec::video::detail::ipc::Initialize sends 901, 902, 1001, 1002. 902 and 1002 return + // a session handle at word 3 and the result at word 4; left to the auto stub, the handle was 0 + // and InitializeVp9Decoder failed with 2001-0114 (title 1 aborts when a movie starts). + void OpenVideoSession(HLERequestContext& ctx) { + const bool second = ctx.GetCommand() == 1002; + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC {}, returning a video decoder session", + ctx.GetCommand()); + ReturnRawSession(system, ctx, + std::make_shared<INx2VideoSession>(system, m_objects, + second ? "nx2:vdec2" : "nx2:vdec")); + } + + void OpenSyncptSession(HLERequestContext& ctx) { + // Used by nn::syncpt::sdk::InitializeEx; handle at word 3, result at word 4. + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 202, returning a nx2:syncpt session"); + ReturnRawSession(system, ctx, std::make_shared<INx2SyncptSession>(system)); + } + + // nn::syncpt::sdk::InitializeEx sends two transfer memories (0x1000 bytes RW, then 0x2000 + // bytes the client maps read-only) and expects the server to have written a header into the + // second one: u32 version (must be 1) at +0, u32 base id at +4, u32 count at +8 (the client + // requires ((count + 0x40) >> 6) - 1 < 0x40), then offsets of: the u64 syncpoint values (RO, + // +0xc), the RW "armed" summary word (+0x10), the RO summary word (+0x14), the RW armed bits + // (+0x18) and the RO acknowledge bits (+0x1c). + // To wait, the client makes its RW bit differ from the RO bit, signals the first returned + // event (it attaches that handle as writable), and waits on the second until the server makes + // the bits equal again; it then rereads the value and rearms if the threshold isn't reached. + // Reply: the two handles (words 3 and 4), then the result. + static constexpr u32 SyncptValuesOffset = 0x40; // RO, u64 per syncpoint + static constexpr u32 SyncptRoSummaryOffset = 0x1040; + static constexpr u32 SyncptRoBitsOffset = 0x1100; // RO, up to 0x40 u64 words + static constexpr u32 SyncptRwSummaryOffset = 0x0; + static constexpr u32 SyncptRwBitsOffset = 0x100; // RW, up to 0x40 u64 words + static constexpr u32 HostSyncpoints = 192; // Tegra::Host1x::SyncpointManager + + void AttachSyncptMemory(HLERequestContext& ctx) { + // RequestParser doesn't skip TIPC handle words, so read the raw data words. + const auto* cmd_buf = ctx.CommandBuffer(); + const size_t data = TipcDataOffset(cmd_buf); + const u32 version_requested = cmd_buf[data]; + const u32 max_syncpoints = std::min<u32>(cmd_buf[data + 1], (0x1000 - 0x40) / 8); + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 204, version={} max_syncpoints={:#x}", + version_requested, max_syncpoints); + + auto rw = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(ctx.GetCopyHandle(0)); + auto ro = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(ctx.GetCopyHandle(1)); + if (rw.IsNotNull() && ro.IsNotNull() && ro->GetSize() >= 0x2000) { + m_syncpt_memory = &ro->GetOwner()->GetMemory(); + m_syncpt_rw = GetInteger(rw->GetSourceAddress()); + m_syncpt_ro = GetInteger(ro->GetSourceAddress()); + m_syncpt_count = max_syncpoints; + auto& memory = *m_syncpt_memory; + memory.Write32(m_syncpt_ro + 0x00, 1); // version + memory.Write32(m_syncpt_ro + 0x04, 0); // base id (guess) + memory.Write32(m_syncpt_ro + 0x08, max_syncpoints); + memory.Write32(m_syncpt_ro + 0x0c, SyncptValuesOffset); + memory.Write32(m_syncpt_ro + 0x10, SyncptRwSummaryOffset); + memory.Write32(m_syncpt_ro + 0x14, SyncptRoSummaryOffset); + memory.Write32(m_syncpt_ro + 0x18, SyncptRwBitsOffset); + memory.Write32(m_syncpt_ro + 0x1c, SyncptRoBitsOffset); + UpdateSyncptValues(); + if (!m_syncpt_poll) { + m_syncpt_poll = Core::Timing::CreateEvent( + "nx2:gsv:SyncptPoll", + [this](s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> { + PollSyncpts(); + return std::nullopt; + }); + system.CoreTiming().ScheduleLoopingEvent(SyncptPollInterval, SyncptPollInterval, + m_syncpt_poll); + } + } else { + LOG_ERROR(Service, "nx2:gsv TIPC 204: missing syncpt transfer memory"); + } + + if (m_syncpt_events[0] == nullptr) { + m_syncpt_events[0] = m_context.CreateEvent("nx2:gsv:SyncptRequest"); + m_syncpt_events[1] = m_context.CreateEvent("nx2:gsv:SyncptNotify"); + } + IPC::ResponseBuilder rb{ctx, 2, 2, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, *m_syncpt_events[0]); // signalled by the client + rb.PushCopyObjects(ctx, m_syncpt_events[1]->GetReadableEvent()); + } + + // Mirrors Eden's guest syncpoint values into the shared RO page. Returns true if any changed. + bool UpdateSyncptValues() { + auto& manager = system.Host1x().GetSyncpointManager(); + bool changed = false; + for (u32 id = 0; id < std::min(m_syncpt_count, HostSyncpoints); ++id) { + const u64 value = manager.GetGuestSyncpointValue(id); + const u64 address = m_syncpt_ro + SyncptValuesOffset + id * 8; + if (m_syncpt_memory->Read64(address) != value) { + m_syncpt_memory->Write64(address, value); + changed = true; + } + } + return changed; + } + + // Answers every armed wait: the value page is current, so the client rechecks its threshold + // and rearms if it still has to wait. Acknowledging without a value change costs the client a + // wakeup per poll, which keeps waits whose syncpoint we never see from hanging. + void PollSyncpts() { + if (m_syncpt_memory == nullptr) { + return; + } + UpdateSyncptValues(); + bool acked = false; + const u32 words = (m_syncpt_count + 63) / 64; + for (u32 w = 0; w < words; ++w) { + const u64 rw_bits = m_syncpt_memory->Read64(m_syncpt_rw + SyncptRwBitsOffset + w * 8); + const u64 ro_address = m_syncpt_ro + SyncptRoBitsOffset + w * 8; + if (m_syncpt_memory->Read64(ro_address) != rw_bits) { + m_syncpt_memory->Write64(ro_address, rw_bits); + acked = true; + } + } + m_syncpt_memory->Write64(m_syncpt_ro + SyncptRoSummaryOffset, + m_syncpt_memory->Read64(m_syncpt_rw + SyncptRwSummaryOffset)); + if (acked) { + m_syncpt_events[1]->Signal(system.Kernel()); + } + } + + // nn::gsv::shim::CalculateRequirements(u64* size, u64* alignment, preference, u32): the 0x40-byte + // InterModuleUserImagePreference (words 2..17) and a u32 (word 18). A block-linear preference + // (MakeInterModuleBlockUserImagePreference) holds kind 2 at word 3, (width - 1) | (height - 1) + // << 16 at word 4, two u32s at words 6-7 and the pixel format at word 8. Reply: result, u64 + // size, u64 alignment; the client raises the alignment to at least 4 KiB and rounds the size. + void CalculateRequirements(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const auto layout = Nx2CalculateImageLayout(w + 2); + LOG_DEBUG(Service, "nx2:gsv 1510 kind={} {}x{} block={} pitch={:#x} format={} -> size={:#x} align={:#x}", + w[3], layout.width, layout.height, layout.block_height_log2, layout.pitch, w[8], + layout.size, layout.alignment); + TipcReplyWords(ctx, {static_cast<u32>(layout.size), static_cast<u32>(layout.size >> 32), + static_cast<u32>(layout.alignment), + static_cast<u32>(layout.alignment >> 32)}); + } + + // nn::codec::CalculateDecoderImagePoolBuffer(size*, alignment*, VideoDecoderImagePreference, + // u32 count) sends 991(preference[16], u32 count) and reads result, u64 size, u64 alignment; + // the client raises the alignment to at least 4 KiB and rounds the size up to it. The + // preference (MakeDecodeImagePreference) holds an output layout code at word 1 (1/4 for + // format 3, which the movie player uses, read here as NV12) and (width - 1) | (height - 1) << 16 + // at word 2. Left to the auto stub, the reply words keep the request's, so the size became + // 0x167027f'00000001 and title 1's movie player aborted (main+0x19e5b58) when its first movie + // started. Sizes are block-linear NV12 (luma, then half-height chroma, 128-row blocks) or 4 + // bytes per pixel for the other codes; nothing is decoded into them yet. + void CalculateDecoderImagePoolBuffer(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u32 code = w[3]; + const u64 width = (w[4] & 0xFFFF) + 1; + const u64 height = (w[4] >> 16) + 1; + const u32 count = std::max<u32>(w[18], 1); + constexpr u64 BlockRows = 128; + const bool nv12 = code == 1 || code == 4; + const u64 pitch = Common::AlignUp(width * (nv12 ? 1 : 4), 64); + u64 size = pitch * Common::AlignUp(height, BlockRows); + if (nv12) { + size += pitch * Common::AlignUp((height + 1) / 2, BlockRows); + } + size *= count; + const u64 alignment = 64 * BlockRows; + LOG_INFO(Service, "nx2:gsv 991 decoder image pool code={} {}x{} count={} -> size={:#x}", + code, width, height, count, size); + TipcReplyWords(ctx, {static_cast<u32>(size), static_cast<u32>(size >> 32), + static_cast<u32>(alignment), static_cast<u32>(alignment >> 32)}); + } + + // 435(preference[16], u32 count): result, u64 size of a GPU native image. + void CalculateNativeImageSize(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u64 size = Nx2CalculateImageLayout(w + 2).size * std::max<u32>(w[18], 1); + LOG_DEBUG(Service, "nx2:gsv 435 count={} -> size={:#x}", w[18], size); + TipcReplyWords(ctx, {static_cast<u32>(size), static_cast<u32>(size >> 32)}); + } + + static u64 Pair(const u32* w, size_t i) { + return u64{w[i]} | (u64{w[i + 1]} << 32); + } + + // 111(u8 mode at word 2, u64 size): result, u32 native memory id. + void CreateNativeMemory(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u64 size = Pair(w, 3); + std::scoped_lock lk{m_objects->mutex}; + const u32 id = m_objects->next_memory++; + m_objects->memories[id] = {size, 0}; + LOG_INFO(Service, "nx2:gsv 111 CreateNativeMemory mode={} size={:#x} -> id {}", w[2] & 0xFF, + size, id); + TipcReplyWords(ctx, {id}); + } + + void DestroyNativeMemory(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + std::scoped_lock lk{m_objects->mutex}; + m_objects->memories.erase(id); + LOG_DEBUG(Service, "nx2:gsv 112 DestroyNativeMemory id {}", id); + TipcReplyWords(ctx, {}); + } + + // 114(u32 id, u64 address, u64 size, u64 flags, u64 offset): back native memory with process + // memory. + void BindNativeMemory(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + BindMemory(w[2], Pair(w, 3), Pair(w, 5), "114"); + TipcReplyWords(ctx, {}); + } + + // 423(u32 id, u64 address, u64 size, u32 parent GpuMemoryHandle, u64 offset, ...): back + // native memory with part of GPU-accessible memory. + void BindNativeGpuMemory(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + LOG_DEBUG(Service, "nx2:gsv 423 parent={} offset={:#x}", w[7], Pair(w, 8)); + BindMemory(w[2], Pair(w, 3), Pair(w, 5), "423"); + TipcReplyWords(ctx, {}); + } + + void BindMemory(u32 id, u64 address, u64 size, const char* command) { + std::scoped_lock lk{m_objects->mutex}; + auto& memory = m_objects->memories[id]; + memory.address = address; + if (memory.size == 0) { + memory.size = size; + } + LOG_INFO(Service, "nx2:gsv {} bind native memory {} at {:#x}+{:#x}", command, id, address, + size); + } + + // 429(preference[16], u32 count, u32 memory id): result, u32 native image id. + void CreateNativeImage(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + std::scoped_lock lk{m_objects->mutex}; + const u32 id = m_objects->next_image++; + auto& image = m_objects->images[id]; + image.memory = w[19]; + std::copy_n(w + 2, image.preference.size(), image.preference.begin()); + image.size = Nx2CalculateImageLayout(w + 2).size; + LOG_INFO(Service, "nx2:gsv 429 CreateNativeImage memory={} count={} size={:#x} -> id {}", + w[19], w[18], image.size, id); + TipcReplyWords(ctx, {id}); + } + + void DestroyNativeImage(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + std::scoped_lock lk{m_objects->mutex}; + m_objects->images.erase(id); + TipcReplyWords(ctx, {}); + } + + // 315(u32 image, u32 count) / 314(u32 image): add/remove a window alias image pool. + void ConfigureWindowImagePool(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + const bool create = ctx.GetCommand() == 315; + std::scoped_lock lk{m_objects->mutex}; + if (auto it = m_objects->images.find(id); it != m_objects->images.end()) { + it->second.window_pool = create; + } + LOG_INFO(Service, "nx2:gsv {} window image pool image {}", ctx.GetCommand(), id); + TipcReplyWords(ctx, {}); + } + + void OpenViSession(HLERequestContext& ctx) { + // Used by nn::vi2::sdk::Initialize; takes one u64, returns a session (handle at word 3, + // result at word 4). + const auto* cmd_buf = ctx.CommandBuffer(); + const size_t data = TipcDataOffset(cmd_buf); + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 305 arg={:#x}, returning a nx2:vi2 session", + u64{cmd_buf[data]} | (u64{cmd_buf[data + 1]} << 32)); + ReturnRawSession(system, ctx, std::make_shared<INx2Vi2Session>(system, m_objects)); + } + + void OpenGpuSession(HLERequestContext& ctx) { + // Used by nn::gpu::sdk::BootWithRemoteMemory; takes one u64, returns a session (handle at + // word 3, result at word 4) that the GPU SDK keeps. + const auto* cmd_buf = ctx.CommandBuffer(); + const size_t data = TipcDataOffset(cmd_buf); + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 497 arg={:#x}, returning a nx2:gpu session", + u64{cmd_buf[data]} | (u64{cmd_buf[data + 1]} << 32)); + ReturnRawSession(system, ctx, std::make_shared<INx2GpuSession>(system)); + } + + void OpenNvsrvSession(HLERequestContext& ctx) { + // The SDK reads the handle at word 3 and the result at word 4. + // nvsrv speaks the nvdrv INvDrvServices protocol (Open/Ioctl/Close/..., + // 13 = SetGraphicsFirmwareMemoryMarginEnabled), so serve it with Eden's nvdrv on the shared + // module. The nx2: name keeps unknown commands auto-stubbed and logged. + LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 1402, returning an nvdrv-backed nx2:nvsrv session"); + auto module = system.ServiceManager().GetService<Nvidia::NVDRV>("nvdrv", true)->GetModule(); + auto nvsrv = std::make_shared<Nvidia::NVDRV>(system, module, "nx2:nvsrv"); + nvsrv->InitializeForProcess(ctx.GetThread().GetOwnerProcess()); + ReturnRawSession(system, ctx, std::move(nvsrv)); + } + + static constexpr std::chrono::nanoseconds SyncptPollInterval{std::chrono::microseconds{500}}; + + std::shared_ptr<Nx2NativeObjects> m_objects{std::make_shared<Nx2NativeObjects>()}; + KernelHelpers::ServiceContext m_context; + std::array<Kernel::KEvent*, 2> m_syncpt_events{}; + Core::Memory::Memory* m_syncpt_memory{}; + u64 m_syncpt_rw{}; + u64 m_syncpt_ro{}; + u32 m_syncpt_count{}; + std::shared_ptr<Core::Timing::EventType> m_syncpt_poll; +}; + +} // namespace Service diff --git a/src/core/hle/service/nx2_stub_interface.h b/src/core/hle/service/nx2_stub_interface.h new file mode 100644 index 0000000000..6890298d1b --- /dev/null +++ b/src/core/hle/service/nx2_stub_interface.h @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <deque> +#include <mutex> +#include <string> +#include <string_view> + +#include <fmt/format.h> + +#include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/service.h" + +namespace Service { + +// Prefix of placeholder interfaces for Switch 2-only IPC. ReportUnimplementedFunction answers +// every call on such an interface with an empty success, so boot can go on while the log shows +// which commands are used. +inline constexpr std::string_view Nx2StubPrefix = "nx2:"; + +class INx2StubInterface final : public ServiceFramework<INx2StubInterface> { +public: + explicit INx2StubInterface(Core::System& system_, const char* name) + : ServiceFramework{system_, name} {} + + // Names must outlive the interface, so they are interned here. + static const char* InternName(std::string name) { + static std::mutex mutex; + static std::deque<std::string> names; + std::scoped_lock lk{mutex}; + for (const auto& n : names) { + if (n == name) { + return n.c_str(); + } + } + return names.emplace_back(std::move(name)).c_str(); + } +}; + +// Handler for an unknown Switch 2 command that returns an interface: hands back a stub named +// after the owning service and the command id. +inline void ReturnNx2StubInterface(Core::System& system, HLERequestContext& ctx, + std::string_view owner) { + const char* name = + INx2StubInterface::InternName(fmt::format("{}{}::{}", Nx2StubPrefix, owner, ctx.GetCommand())); + LOG_WARNING(Service, "(STUBBED) Switch 2 {}::{} -> stub interface {}", owner, ctx.GetCommand(), + name); + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface<INx2StubInterface>(ctx, system, name); +} + +} // namespace Service diff --git a/src/core/hle/service/service.cpp b/src/core/hle/service/service.cpp index b4fe2b3ed5..5a779af3ca 100644 --- a/src/core/hle/service/service.cpp +++ b/src/core/hle/service/service.cpp @@ -4,6 +4,14 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include "common/scope_exit.h" +#include <vector> +#include <tuple> +#include <mutex> +#include <map> +#include <chrono> +#include <algorithm> +#include <cstdlib> #include <fmt/ranges.h> #include "common/assert.h" #include "common/logging.h" @@ -13,6 +21,7 @@ #include "core/hle/kernel/kernel.h" #include "core/hle/service/ipc_helpers.h" #include "core/hle/service/service.h" +#include "core/hle/service/nx2_stub_interface.h" #include "core/hle/service/sm/sm.h" #include "core/reporter.h" @@ -63,13 +72,18 @@ void ServiceFrameworkBase::ReportUnimplementedFunction(HLERequestContext& ctx, fmt::memory_buffer buf; fmt::format_to(std::back_inserter(buf), "function '{}({})': port='{}' cmd_buf={{[0]={:#x}", ctx.GetCommand(), function_name, service_name, cmd_buf[0]); - for (int i = 1; i <= 8; ++i) + // The Switch 2 video decoder sessions are being reverse engineered: dump whole requests. + const int last_word = std::string_view{service_name}.starts_with("nx2:vdec") ? 47 : 8; + for (int i = 1; i <= last_word; ++i) fmt::format_to(std::back_inserter(buf), ", [{}]={:#x}", i, cmd_buf[i]); buf.push_back('}'); system.GetReporter().SaveUnimplementedFunctionReport(ctx, ctx.GetCommand(), function_name, service_name); UNIMPLEMENTED_MSG("Unknown / unimplemented {}", fmt::to_string(buf)); - if (Settings::values.use_auto_stub) { + // EDEN_NX2_AUTO_STUB=1 auto-stubs every unknown command without touching the config. + static const bool env_auto_stub = std::getenv("EDEN_NX2_AUTO_STUB") != nullptr; + if (Settings::values.use_auto_stub || env_auto_stub || + std::string_view{service_name}.starts_with(Nx2StubPrefix)) { LOG_WARNING(Service, "Using auto stub fallback!"); IPC::ResponseBuilder rb{ctx, 2}; rb.Push(ResultSuccess); @@ -83,7 +97,25 @@ void ServiceFrameworkBase::InvokeRequest(HLERequestContext& ctx) { return ReportUnimplementedFunction(ctx, info); LOG_TRACE(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer())); + // EDEN_NX2_IPC_TRACE=1 logs every dispatched command and the result word it returned. + static const bool ipc_trace = std::getenv("EDEN_NX2_IPC_TRACE") != nullptr; + if (ipc_trace) { + LOG_INFO(Service, "ipc {}::{} ({})", service_name, ctx.GetCommand(), info->name); + } handler_invoker(this, info->handler_callback, ctx); + if (ipc_trace) { + // Response layout: [0..1] header, then 'SFCO', version, result. + const auto* cmd_buf = ctx.CommandBuffer(); + for (int i = 0; i < 12; ++i) { + if (cmd_buf[i] == 0x4F434653) { // "SFCO" + if (cmd_buf[i + 2] != 0) { + LOG_INFO(Service, "ipc {}::{} -> result {:#x}", service_name, ctx.GetCommand(), + cmd_buf[i + 2]); + } + break; + } + } + } } void ServiceFrameworkBase::InvokeRequestTipc(HLERequestContext& ctx) { @@ -93,12 +125,68 @@ void ServiceFrameworkBase::InvokeRequestTipc(HLERequestContext& ctx) { return ReportUnimplementedFunction(ctx, info); LOG_TRACE(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer())); + static const bool ipc_trace = std::getenv("EDEN_NX2_IPC_TRACE") != nullptr; + if (ipc_trace) { + const auto* w = ctx.CommandBuffer(); + LOG_INFO(Service, "tipc {}::{} ({}) [2]={:#x} [3]={:#x} [4]={:#x}", service_name, + ctx.GetCommand(), info->name, w[2], w[3], w[4]); + } handler_invoker(this, info->handler_callback, ctx); } +namespace { +// EDEN_NX2_IPC_SUMMARY: every 5 s, log how often each service command ran and with which result +// since the last summary (finds a game polling something that never becomes ready). +void CountRequest(const std::string& service, u32 command, u32 type, u32 result) { + static const bool enabled = std::getenv("EDEN_NX2_IPC_SUMMARY") != nullptr; + if (!enabled) { + return; + } + static std::mutex mutex; + static std::map<std::tuple<std::string, u32, u32, u32>, u32> counts; + static auto last = std::chrono::steady_clock::now(); + std::scoped_lock lock{mutex}; + ++counts[{service, command, type, result}]; + const auto now = std::chrono::steady_clock::now(); + if (now - last < std::chrono::seconds{5}) { + return; + } + last = now; + std::vector<std::pair<u32, std::string>> lines; + for (const auto& [key, count] : counts) { + lines.emplace_back(count, fmt::format("{}::{}{} -> {:#x}", std::get<0>(key), std::get<1>(key), + std::get<2>(key) >= 16 ? " (tipc)" : "", + std::get<3>(key))); + } + std::sort(lines.begin(), lines.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); + std::string text; + for (size_t i = 0; i < std::min<size_t>(lines.size(), 24); ++i) { + text += fmt::format(" | {}x {}", lines[i].first, lines[i].second); + } + LOG_INFO(Service, "ipc summary ({} kinds){}", lines.size(), text); + counts.clear(); +} +} // Anonymous namespace + Result ServiceFrameworkBase::HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) { const auto guard = LockService(); + SCOPE_EXIT { + const auto* cmd_buf = ctx.CommandBuffer(); + u32 result_word = 0; + const u32 type = static_cast<u32>(ctx.GetCommandType()); + if (type >= 16) { + result_word = cmd_buf[2]; + } else { + for (int i = 0; i < 12; ++i) { + if (cmd_buf[i] == 0x4F434653) { // "SFCO" + result_word = cmd_buf[i + 2]; + break; + } + } + } + CountRequest(service_name, ctx.GetCommand(), type, result_word); + }; Result result = ResultSuccess; diff --git a/src/core/launch_params.cpp b/src/core/launch_params.cpp index a0483226ec..25fdd45fdb 100644 --- a/src/core/launch_params.cpp +++ b/src/core/launch_params.cpp @@ -94,7 +94,7 @@ LaunchParams ParseLaunchParams(Core::System& system, int argc, char *argv[], wch break; case 'u': { // Launch game with a specific user - bool argument_ok = isdigit(optarg[0]); + bool argument_ok = isdigit(optarg[0]) != 0; p.selected_user = atoi(optarg); if (!argument_ok) { // try to look it up by username, only finds the first username that matches. diff --git a/src/core/loader/deconstructed_rom_directory.cpp b/src/core/loader/deconstructed_rom_directory.cpp index 5e2ec7e0b9..458e0210a0 100644 --- a/src/core/loader/deconstructed_rom_directory.cpp +++ b/src/core/loader/deconstructed_rom_directory.cpp @@ -12,6 +12,7 @@ #include "core/file_sys/content_archive.h" #include "core/file_sys/control_metadata.h" #include "core/file_sys/patch_manager.h" +#include "core/file_sys/romfs.h" #include "core/file_sys/romfs_factory.h" #include "core/hle/kernel/k_page_table.h" #include "core/hle/kernel/k_process.h" @@ -71,6 +72,30 @@ struct PatchCollection { std::array<s32, 13> module_patcher_indices{}; }; +namespace { + +// RomFS for a loose ExeFS: the first .romfs file next to main, else a romfs/ directory next to +// the exefs/ directory (the usual layout of an extracted title). +FileSys::VirtualFile FindLooseRomFS(const FileSys::VirtualDir& exefs_dir) { + if (exefs_dir == nullptr) { + return nullptr; + } + for (const auto& f : exefs_dir->GetFiles()) { + if (f->GetExtension() == "romfs") { + return f; + } + } + if (const auto parent = exefs_dir->GetParentDirectory()) { + if (const auto romfs_dir = parent->GetSubdirectory("romfs")) { + LOG_INFO(Loader, "Using RomFS directory next to the ExeFS"); + return FileSys::CreateRomFS(romfs_dir); + } + } + return nullptr; +} + +} // namespace + AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(FileSys::VirtualFile file_, bool override_update_) : AppLoader(std::move(file_)) , override_update(override_update_) @@ -122,6 +147,8 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(FileSys FileSys::NACP nacp(nacp_file); name = nacp.GetApplicationName(); } + + romfs = FindLooseRomFS(file_dir); } AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory( @@ -129,7 +156,9 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory( : AppLoader(directory->GetFile("main")) , dir(std::move(directory)) , override_update(override_update_) -{} +{ + romfs = FindLooseRomFS(dir); +} FileType AppLoader_DeconstructedRomDirectory::IdentifyType(const FileSys::VirtualFile& dir_file) { if (FileSys::IsDirectoryExeFS(dir_file->GetContainingDirectory())) { @@ -145,6 +174,9 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect return {ResultStatus::ErrorAlreadyLoaded, {}}; } + // Loaded directly from a loose ExeFS (not wrapped by the NCA/NSP loaders, which register + // the process with the filesystem controller themselves). + const bool standalone = dir == nullptr; if (dir == nullptr) { if (file == nullptr) { return {ResultStatus::ErrorNullFile, {}}; @@ -266,6 +298,14 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect LOG_DEBUG(Loader, "loaded module {} @ {:#x}", module, load_addr); } + if (standalone) { + // Without this, fsp-srv SetCurrentProcess fails with ResultTargetNotFound. + system.GetFileSystemController().RegisterProcess( + process.GetProcessId(), metadata.GetTitleID(), + std::make_shared<FileSys::RomFSFactory>(*this, system.GetContentProvider(), + system.GetFileSystemController())); + } + is_loaded = true; return {ResultStatus::Success, LoadParameters{metadata.GetMainThreadPriority(), metadata.GetMainThreadStackSize()}}; diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp index 92e2e2f602..5871ac412f 100644 --- a/src/core/loader/loader.cpp +++ b/src/core/loader/loader.cpp @@ -393,6 +393,13 @@ std::unique_ptr<AppLoader> GetLoader(Core::System& system, FileSys::VirtualFile FileType type = IdentifyFile(file); const FileType filename_type = GuessFromFilename(file->GetName()); + // A "main" inside an ExeFS directory is also a valid NSO, but loading it alone skips + // main.npdm, rtld and the sdk/subsdk modules. Load the whole directory instead. + if (type == FileType::NSO && filename_type == FileType::DeconstructedRomDirectory && + IdentifyFileLoader<AppLoader_DeconstructedRomDirectory>(file)) { + type = FileType::DeconstructedRomDirectory; + } + // Special case: 00 is either a NCA or NAX. if (type != filename_type && !(file->GetName() == "00" && type == FileType::NAX)) { LOG_WARNING(Loader, "File {} has a different type ({}) than its extension.", diff --git a/src/core/loader/nso.cpp b/src/core/loader/nso.cpp index ea1d1c9ecd..7e3b9a54c2 100644 --- a/src/core/loader/nso.cpp +++ b/src/core/loader/nso.cpp @@ -247,6 +247,7 @@ std::optional<VAddr> AppLoader_NSO::LoadModule(Kernel::KProcess& process, Core:: } // Load codeset for current process + LOG_INFO(Loader, "Module {} loaded at {:#x} (size {:#x})", nso_file.GetName(), load_base, image_size); process.LoadModule(system.Kernel(), std::move(codeset), load_base); return load_base + image_size; } diff --git a/src/core/memory.cpp b/src/core/memory.cpp index eabc6dfccb..6e0ab0ee6d 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -5,6 +5,7 @@ // SPDX-FileCopyrightText: 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <atomic> #include <algorithm> #include <cstring> #include <mutex> @@ -36,6 +37,11 @@ namespace Core::Memory { +u32& UnmappedReadFlag() { + thread_local u32 flag{}; + return flag; +} + static inline bool AddressSpaceContains(const Common::PageTable& table, const Common::ProcessAddress addr, const std::size_t size) { const Common::ProcessAddress max_addr = 1ULL << table.GetAddressSpaceBits(); return addr + size >= addr && addr + size <= max_addr; @@ -623,7 +629,13 @@ struct Memory::Impl { inline T Read(Common::ProcessAddress vaddr) noexcept requires(std::is_trivially_copyable_v<T>) { const u64 addr = GetInteger(vaddr); if (auto const ptr = GetPointerImpl(addr, [addr]() { - LOG_ERROR(HW_Memory, "Unmapped Read{} @ {:#016x}", sizeof(T) * 8, addr); + // A guest spinning on a bad pointer reads unmapped memory tens of thousands of times + // a second: keep the first ones and then a sample. + static std::atomic<u64> unmapped_reads{}; + if (const u64 count = unmapped_reads.fetch_add(1); count < 1000 || count % 100000 == 0) { + LOG_ERROR(HW_Memory, "Unmapped Read{} @ {:#016x} (#{})", sizeof(T) * 8, addr, count); + } + ++UnmappedReadFlag(); }, [&]() { HandleRasterizerDownload(addr, sizeof(T)); }); ptr) [[likely]] { diff --git a/src/core/memory.h b/src/core/memory.h index 7167efbb84..204b7514f6 100644 --- a/src/core/memory.h +++ b/src/core/memory.h @@ -38,6 +38,9 @@ class MemoryManager; namespace Core::Memory { +/// Counts unmapped reads on this host thread since the CPU callbacks last checked (debug aid). +u32& UnmappedReadFlag(); + /** * Page size used by the ARM architecture. This is the smallest granularity with which memory can * be mapped. diff --git a/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc b/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc index 8a3fab7ac1..ee9f0faa18 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc +++ b/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc @@ -2,6 +2,24 @@ // SPDX-License-Identifier: GPL-3.0-or-later // DO NOT REORDER +// ARMv8.1+ encodings used by Switch 2 code. Kept first so they win over broader patterns. +INST(LDAPR_sz, "LDAPRB, LDAPRH, LDAPR", "zz11100010111111110000nnnnnttttt") +INST(AtomicMemOp, "LDADD/LDCLR/LDEOR/LDSET/LDSMAX/LDSMIN/LDUMAX/LDUMIN/SWP", "zz111000AR1sssssoooo00nnnnnttttt") +INST(CompareAndSwap, "CASB, CASH, CAS", "zz0010001L1sssssO11111nnnnnttttt") +INST(CompareAndSwapPair, "CASP", "0z0010000L1sssssO11111nnnnnttttt") +INST(PACAUT_reg, "PACIA/PACIB/PACDA/PACDB/AUTIA/AUTIB/AUTDA/AUTDB", "110110101100000100Zooonnnnnddddd") +INST(XPAC_1, "XPACI, XPACD", "110110101100000101000D11111ddddd") +INST(PACGA, "PACGA", "10011010110mmmmm001100nnnnnddddd") +INST(RETA, "RETAA, RETAB", "110101100101111100001M1111111111") +INST(BRA_BLRA, "BRAA/BRAB/BLRAA/BLRAB (+Z)", "1101011Z00L1111100001Mnnnnnmmmmm") +INST(LDRA, "LDRAA, LDRAB", "11111000MS1iiiiiiiiiW1nnnnnttttt") +INST(STG_family, "STG/STZG/ST2G/STZ2G/LDG", "11011001oo1iiiiiiiiixxnnnnnttttt") +INST(STGP, "STGP", "0110100xx0iiiiiiiuuuuunnnnnttttt") +INST(IRG, "IRG", "10011010110mmmmm000100nnnnnddddd") +INST(GMI, "GMI", "10011010110mmmmm000101nnnnnddddd") +INST(ADDG, "ADDG", "1001000110iiiiii00ttttnnnnnddddd") +INST(SUBG, "SUBG", "1101000110iiiiii00ttttnnnnnddddd") + INST(FMOV_3, "FMOV (vector, immediate)", "0Q00111100000abc111111defghddddd") INST(FMOV_2, "FMOV (vector, immediate)", "0Qo0111100000abc111101defghddddd") INST(MOVI, "MOVI, MVNI, ORR, BIC (vector, immediate)", "0Qo0111100000abcmmmm01defghddddd") diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp index 7840c63d79..70d5098a65 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.cpp @@ -82,6 +82,9 @@ bool TranslateSingleInstruction(IR::Block& block, LocationDescriptor descriptor, #include "dynarmic/frontend/A64/translate/impl/impl.cpp" #include "dynarmic/frontend/A64/translate/impl/impl.h" #include "dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp" +#include "dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp" +#include "dynarmic/frontend/A64/translate/impl/memory_tagging.cpp" +#include "dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp" #include "dynarmic/frontend/A64/translate/impl/load_store_load_literal.cpp" #include "dynarmic/frontend/A64/translate/impl/load_store_multiple_structures.cpp" #include "dynarmic/frontend/A64/translate/impl/load_store_no_allocate_pair.cpp" diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h index c8afef3283..d8e6f2b739 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h @@ -326,6 +326,17 @@ struct TranslatorVisitor final { bool SWP(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); bool LDAPR(Reg Rn, Reg Rt); + // ARMv8.1+ forms decoded for Switch 2 code (load_store_atomic.cpp, pointer_authentication.cpp, + // memory_tagging.cpp) + bool AtomicMemOp(Imm<2> size, bool A, bool R, Reg Rs, Imm<4> op, Reg Rn, Reg Rt); + bool LDAPR_sz(Imm<2> size, Reg Rn, Reg Rt); + bool CompareAndSwap(Imm<2> size, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt); + bool CompareAndSwapPair(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt); + bool PACAUT_reg(bool Z, Imm<3> op, Reg Rn, Reg Rd); + bool BRA_BLRA(bool Z, bool L, bool M, Reg Rn, Reg Rm); + bool STG_family(Imm<2> opc, Imm<9> imm9, Imm<2> mode, Reg Rn, Reg Rt); + bool STGP(Imm<2> mode, Imm<7> imm7, Reg Rt2, Reg Rn, Reg Rt); + // Loads and stores - Load/Store register (register offset) bool STRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, Reg Rt); bool LDRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, Reg Rt); diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp new file mode 100644 index 0000000000..eafc993988 --- /dev/null +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_atomic.cpp @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// ARMv8.1 LSE atomics (LDADD/LDCLR/LDEOR/LDSET/LD{S,U}{MAX,MIN}/SWP, CAS, CASP) and +// ARMv8.3 RCpc LDAPR. +// +// Each read-modify-write is built from an exclusive load and an exclusive store. If the +// store fails (another core touched the monitor in between) no register is changed and the +// same instruction is executed again, so the operation stays atomic with multiple cores. + +#include "dynarmic/frontend/A64/translate/impl/impl.h" + +namespace Dynarmic::A64 { + +static Reg AtomicNextReg(Reg r) { + return static_cast<Reg>(static_cast<size_t>(r) + 1); +} + +static IR::U64 AtomicAddress(TranslatorVisitor& v, Reg Rn) { + return Rn == Reg::SP ? IR::U64{v.SP(64)} : IR::U64{v.X(64, Rn)}; +} + +// All-ones when value == 0, otherwise zero. +static IR::U64 AtomicMaskIfZero(TranslatorVisitor& v, IR::U64 value) { + const IR::U64 nonzero = v.ir.LogicalShiftRight(v.ir.Or(value, v.ir.Sub(v.ir.Imm64(0), value)), v.ir.Imm8(63)); + return v.ir.Sub(nonzero, v.ir.Imm64(1)); +} + +static IR::U64 AtomicSelect(TranslatorVisitor& v, IR::U64 mask, IR::U64 if_set, IR::U64 if_clear) { + return v.ir.Or(v.ir.And(if_set, mask), v.ir.AndNot(if_clear, mask)); +} + +static IR::UAny AtomicTruncate(TranslatorVisitor& v, IR::U64 value, size_t bytes) { + switch (bytes) { + case 1: + return v.ir.LeastSignificantByte(value); + case 2: + return v.ir.LeastSignificantHalf(value); + case 4: + return v.ir.LeastSignificantWord(value); + default: + return value; + } +} + +static IR::U64 AtomicLoadZext(TranslatorVisitor& v, IR::U64 address, size_t bytes, IR::AccType acctype) { + const IR::UAny value = v.ExclusiveMem(address, bytes, acctype); + return bytes == 8 ? IR::U64{value} : v.ir.ZeroExtendToLong(value); +} + +static IR::U64 AtomicSignExtend(TranslatorVisitor& v, IR::U64 value, size_t bytes) { + return bytes == 8 ? value : v.ir.SignExtendToLong(AtomicTruncate(v, value, bytes)); +} + +static IR::U64 AtomicZeroExtendReg(TranslatorVisitor& v, Reg r, size_t bytes) { + const IR::U64 value = v.X(64, r); + return bytes == 8 ? value : v.ir.ZeroExtendToLong(AtomicTruncate(v, value, bytes)); +} + +// Writes `value` to `r` only when the exclusive store succeeded (ok_mask all-ones). +static void AtomicCommitReg(TranslatorVisitor& v, Reg r, IR::U64 ok_mask, IR::U64 value) { + if (r == Reg::ZR) { + return; + } + v.X(64, r, AtomicSelect(v, ok_mask, value, v.X(64, r))); +} + +// Ends the block: continue with the next instruction on success, retry this one on failure. +static bool AtomicFinish(TranslatorVisitor& v, IR::U32 status) { + v.ir.SetCheckBit(v.ir.IsZero(status)); + v.ir.SetTerm(IR::Term::CheckBit{IR::Term::LinkBlock{v.ir.current_location->AdvancePC(4)}, + IR::Term::LinkBlock{*v.ir.current_location}}); + return false; +} + +static IR::AccType AtomicAccType(bool acquire, bool release) { + return acquire || release ? IR::AccType::ORDERED : IR::AccType::ATOMIC; +} + +bool TranslatorVisitor::AtomicMemOp(Imm<2> size, bool A, bool R, Reg Rs, Imm<4> op, Reg Rn, Reg Rt) { + const size_t bytes = size_t{1} << size.ZeroExtend<size_t>(); + const size_t opv = op.ZeroExtend<size_t>(); + if (opv > 0b1000) { + return UnallocatedEncoding(); + } + + const IR::U64 address = AtomicAddress(*this, Rn); + const IR::AccType acctype = AtomicAccType(A, R); + const IR::U64 old = AtomicLoadZext(*this, address, bytes, acctype); + const IR::U64 s = AtomicZeroExtendReg(*this, Rs, bytes); + + IR::U64 result; + switch (opv) { + case 0b0000: // LDADD + result = ir.Add(old, s); + break; + case 0b0001: // LDCLR + result = ir.AndNot(old, s); + break; + case 0b0010: // LDEOR + result = ir.Eor(old, s); + break; + case 0b0011: // LDSET + result = ir.Or(old, s); + break; + case 0b0100: // LDSMAX + result = ir.MaxSigned(AtomicSignExtend(*this, old, bytes), AtomicSignExtend(*this, s, bytes)); + break; + case 0b0101: // LDSMIN + result = ir.MinSigned(AtomicSignExtend(*this, old, bytes), AtomicSignExtend(*this, s, bytes)); + break; + case 0b0110: // LDUMAX + result = ir.MaxUnsigned(old, s); + break; + case 0b0111: // LDUMIN + result = ir.MinUnsigned(old, s); + break; + default: // SWP + result = s; + break; + } + + const IR::U32 status = ExclusiveMem(address, bytes, acctype, AtomicTruncate(*this, result, bytes)); + const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status)); + AtomicCommitReg(*this, Rt, ok, old); + return AtomicFinish(*this, status); +} + +bool TranslatorVisitor::LDAPR_sz(Imm<2> size, Reg Rn, Reg Rt) { + const size_t bytes = size_t{1} << size.ZeroExtend<size_t>(); + const size_t regsize = bytes == 8 ? 64 : 32; + const IR::UAny data = Mem(AtomicAddress(*this, Rn), bytes, IR::AccType::ORDERED); + X(regsize, Rt, ZeroExtend(data, regsize)); + return true; +} + +bool TranslatorVisitor::CompareAndSwap(Imm<2> size, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) { + const size_t bytes = size_t{1} << size.ZeroExtend<size_t>(); + const IR::U64 address = AtomicAddress(*this, Rn); + const IR::AccType acctype = AtomicAccType(L, o0); + + const IR::U64 old = AtomicLoadZext(*this, address, bytes, acctype); + const IR::U64 expected = AtomicZeroExtendReg(*this, Rs, bytes); + const IR::U64 desired = AtomicZeroExtendReg(*this, Rt, bytes); + const IR::U64 equal = AtomicMaskIfZero(*this, ir.Eor(old, expected)); + const IR::U64 result = AtomicSelect(*this, equal, desired, old); + + const IR::U32 status = ExclusiveMem(address, bytes, acctype, AtomicTruncate(*this, result, bytes)); + const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status)); + AtomicCommitReg(*this, Rs, ok, old); + return AtomicFinish(*this, status); +} + +bool TranslatorVisitor::CompareAndSwapPair(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) { + if ((static_cast<size_t>(Rs) & 1) != 0 || (static_cast<size_t>(Rt) & 1) != 0) { + return UnallocatedEncoding(); + } + const Reg Rs2 = AtomicNextReg(Rs); + const Reg Rt2 = AtomicNextReg(Rt); + const IR::U64 address = AtomicAddress(*this, Rn); + const IR::AccType acctype = AtomicAccType(L, o0); + + if (!sz) { + // Two 32-bit words, accessed as one 64-bit exclusive. + const IR::U64 old = ExclusiveMem(address, 8, acctype); + const IR::U64 expected = ir.Pack2x32To1x64(X(32, Rs), X(32, Rs2)); + const IR::U64 desired = ir.Pack2x32To1x64(X(32, Rt), X(32, Rt2)); + const IR::U64 equal = AtomicMaskIfZero(*this, ir.Eor(old, expected)); + const IR::U32 status = ExclusiveMem(address, 8, acctype, AtomicSelect(*this, equal, desired, old)); + const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status)); + AtomicCommitReg(*this, Rs, ok, ir.ZeroExtendToLong(ir.LeastSignificantWord(old))); + AtomicCommitReg(*this, Rs2, ok, ir.LogicalShiftRight(old, ir.Imm8(32))); + return AtomicFinish(*this, status); + } + + const IR::U128 old = ExclusiveMem(address, 16, acctype); + const IR::U64 old_lo = ir.VectorGetElement(64, old, 0); + const IR::U64 old_hi = ir.VectorGetElement(64, old, 1); + const IR::U64 diff = ir.Or(ir.Eor(old_lo, X(64, Rs)), ir.Eor(old_hi, X(64, Rs2))); + const IR::U64 equal = AtomicMaskIfZero(*this, diff); + const IR::U64 new_lo = AtomicSelect(*this, equal, X(64, Rt), old_lo); + const IR::U64 new_hi = AtomicSelect(*this, equal, X(64, Rt2), old_hi); + const IR::U32 status = ExclusiveMem(address, 16, acctype, ir.Pack2x64To1x128(new_lo, new_hi)); + const IR::U64 ok = AtomicMaskIfZero(*this, ir.ZeroExtendToLong(status)); + AtomicCommitReg(*this, Rs, ok, old_lo); + AtomicCommitReg(*this, Rs2, ok, old_hi); + return AtomicFinish(*this, status); +} + +} // namespace Dynarmic::A64 diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/memory_tagging.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/memory_tagging.cpp new file mode 100644 index 0000000000..9c6bcb7872 --- /dev/null +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/memory_tagging.cpp @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// ARMv8.5 memory tagging, with tags ignored: every allocation tag is 0, tag stores only do +// their address writeback, and the STZG forms still zero the granules they cover. + +#include "dynarmic/frontend/A64/translate/impl/impl.h" + +namespace Dynarmic::A64 { + +static IR::U64 TagBase(TranslatorVisitor& v, Reg Rn) { + return Rn == Reg::SP ? IR::U64{v.SP(64)} : IR::U64{v.X(64, Rn)}; +} + +static void TagSetBase(TranslatorVisitor& v, Reg Rn, IR::U64 value) { + if (Rn == Reg::SP) { + v.SP(64, value); + } else { + v.X(64, Rn, value); + } +} + +bool TranslatorVisitor::STG_family(Imm<2> opc, Imm<9> imm9, Imm<2> mode, Reg Rn, Reg Rt) { + const size_t opcv = opc.ZeroExtend<size_t>(); + const size_t modev = mode.ZeroExtend<size_t>(); + if (modev == 0b00) { + // LDG (opc 01) leaves Xt as-is since tags are 0; STZGM/STGM/LDGM are EL1-only. + if (opcv == 0b01) { + return true; + } + return UnallocatedEncoding(); + } + + const s64 offset = imm9.SignExtend<s64>() << 4; + const IR::U64 base = TagBase(*this, Rn); + const IR::U64 offset_address = ir.Add(base, ir.Imm64(static_cast<u64>(offset))); + const IR::U64 address = modev == 0b01 ? base : offset_address; // 01 = post-index + + const bool zero = (opcv & 1) != 0; // STZG, STZ2G + const size_t granules = (opcv & 2) ? 2 : 1; // ST2G, STZ2G + if (zero) { + const IR::U64 zero64 = ir.Imm64(0); + for (size_t i = 0; i < granules * 2; ++i) { + Mem(ir.Add(address, ir.Imm64(i * 8)), 8, IR::AccType::NORMAL, zero64); + } + } + (void)Rt; + if (modev != 0b10) { + TagSetBase(*this, Rn, offset_address); + } + return true; +} + +bool TranslatorVisitor::STGP(Imm<2> mode, Imm<7> imm7, Reg Rt2, Reg Rn, Reg Rt) { + const size_t modev = mode.ZeroExtend<size_t>(); + if (modev == 0b00) { + return UnallocatedEncoding(); + } + const s64 offset = imm7.SignExtend<s64>() << 4; + const IR::U64 base = TagBase(*this, Rn); + const IR::U64 offset_address = ir.Add(base, ir.Imm64(static_cast<u64>(offset))); + const IR::U64 address = modev == 0b01 ? base : offset_address; + Mem(address, 8, IR::AccType::NORMAL, X(64, Rt)); + Mem(ir.Add(address, ir.Imm64(8)), 8, IR::AccType::NORMAL, X(64, Rt2)); + if (modev != 0b10) { + TagSetBase(*this, Rn, offset_address); + } + return true; +} + +bool TranslatorVisitor::IRG(Reg Rm, Reg Rn, Reg Rd) { + (void)Rm; + TagSetBase(*this, Rd, TagBase(*this, Rn)); + return true; +} + +bool TranslatorVisitor::GMI(Reg Rm, Reg Rn, Reg Rd) { + (void)Rn; + // Tag of Xn is always 0, so the exclusion mask gains bit 0. + X(64, Rd, ir.Or(X(64, Rm), ir.Imm64(1))); + return true; +} + +bool TranslatorVisitor::ADDG(Imm<6> offset_imm, Imm<4> tag_offset, Reg Rn, Reg Rd) { + (void)tag_offset; + const u64 offset = offset_imm.ZeroExtend<u64>() << 4; + TagSetBase(*this, Rd, ir.Add(TagBase(*this, Rn), ir.Imm64(offset))); + return true; +} + +bool TranslatorVisitor::SUBG(Imm<6> offset_imm, Imm<4> tag_offset, Reg Rn, Reg Rd) { + (void)tag_offset; + const u64 offset = offset_imm.ZeroExtend<u64>() << 4; + TagSetBase(*this, Rd, ir.Sub(TagBase(*this, Rn), ir.Imm64(offset))); + return true; +} + +} // namespace Dynarmic::A64 diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp new file mode 100644 index 0000000000..e36942c942 --- /dev/null +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/pointer_authentication.cpp @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// ARMv8.3 pointer authentication, implemented as the identity: PAC* never adds a code and +// AUT*/XPAC* have nothing to strip, so signed pointers are just plain pointers. The HINT-space +// forms (PACIASP, AUTIBSP, ...) already decode as HINT/NOP. + +#include "dynarmic/frontend/A64/translate/impl/impl.h" + +namespace Dynarmic::A64 { + +bool TranslatorVisitor::PACAUT_reg(bool Z, Imm<3> op, Reg Rn, Reg Rd) { + (void)op; + (void)Rd; + if (Z && Rn != Reg::R31) { + return UnallocatedEncoding(); + } + return true; +} + +bool TranslatorVisitor::XPAC_1(bool D, Reg Rd) { + (void)D; + (void)Rd; + return true; +} + +bool TranslatorVisitor::PACGA(Reg Rm, Reg Rn, Reg Rd) { + // Any deterministic value works, since the code is only ever compared to a recomputation. + const IR::U64 m = Rm == Reg::SP ? IR::U64{SP(64)} : IR::U64{X(64, Rm)}; + X(64, Rd, ir.And(ir.Eor(X(64, Rn), m), ir.Imm64(0xFFFFFFFF00000000ULL))); + return true; +} + +bool TranslatorVisitor::RETA(bool M) { + (void)M; + ir.SetPC(X(64, Reg::R30)); + ir.SetTerm(IR::Term::PopRSBHint{}); + return false; +} + +bool TranslatorVisitor::BRA_BLRA(bool Z, bool L, bool M, Reg Rn, Reg Rm) { + (void)M; + if (!Z && Rm != Reg::R31) { + return UnallocatedEncoding(); + } + const auto target = X(64, Rn); + if (L) { + X(64, Reg::R30, ir.Imm64(ir.PC() + 4)); + ir.PushRSB(ir.current_location->AdvancePC(4)); + } + ir.SetPC(target); + ir.SetTerm(IR::Term::FastDispatchHint{}); + return false; +} + +bool TranslatorVisitor::LDRA(bool M, bool S, Imm<9> imm9, bool W, Reg Rn, Reg Rt) { + (void)M; + const s64 offset = concatenate(Imm<1>{S ? 1u : 0u}, imm9).SignExtend<s64>() << 3; + const IR::U64 base = Rn == Reg::SP ? IR::U64{SP(64)} : IR::U64{X(64, Rn)}; + const IR::U64 address = ir.Add(base, ir.Imm64(static_cast<u64>(offset))); + const IR::U64 data = Mem(address, 8, IR::AccType::NORMAL); + if (W) { + if (Rn == Reg::SP) { + SP(64, address); + } else { + X(64, Rn, address); + } + } + X(64, Rt, data); + return true; +} + +} // namespace Dynarmic::A64 diff --git a/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h b/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h index 05b067bdd8..46fce86bda 100644 --- a/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h +++ b/src/dynarmic/src/dynarmic/interface/exclusive_monitor.h @@ -82,7 +82,7 @@ private: static constexpr VAddr RESERVATION_GRANULE_MASK = 0xFFFF'FFFF'FFFF'FFFFull; static constexpr VAddr INVALID_EXCLUSIVE_ADDRESS = 0xDEAD'DEAD'DEAD'DEADull; - static constexpr size_t MAX_NUM_CPU_CORES = 4; // Sync with src/core/hardware_properties + static constexpr size_t MAX_NUM_CPU_CORES = 6; // Sync with src/core/hardware_properties boost::container::static_vector<VAddr, MAX_NUM_CPU_CORES> exclusive_addresses; boost::container::static_vector<Vector, MAX_NUM_CPU_CORES> exclusive_values; SpinLock lock; diff --git a/src/shader_recompiler/CMakeLists.txt b/src/shader_recompiler/CMakeLists.txt index cc9ce95d92..39484a1bd5 100644 --- a/src/shader_recompiler/CMakeLists.txt +++ b/src/shader_recompiler/CMakeLists.txt @@ -58,6 +58,9 @@ add_library(shader_recompiler STATIC frontend/ir/type.h frontend/ir/value.cpp frontend/ir/value.h + frontend/ampere/lower_to_maxwell.h + frontend/ampere/program_override.h + frontend/ampere/program_address_map.h frontend/maxwell/control_flow.cpp frontend/maxwell/control_flow.h frontend/maxwell/decode.cpp diff --git a/src/shader_recompiler/environment.h b/src/shader_recompiler/environment.h index 184b56ebf3..67486430c8 100644 --- a/src/shader_recompiler/environment.h +++ b/src/shader_recompiler/environment.h @@ -74,6 +74,8 @@ public: [[nodiscard]] virtual u64 ReadInstruction(u32 address) = 0; + [[nodiscard]] virtual u32 TranslateProgramAddress(u32 address) const { return address; } + [[nodiscard]] virtual u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) = 0; [[nodiscard]] virtual TextureType ReadTextureType(u32 raw_handle) = 0; diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h new file mode 100644 index 0000000000..365efa1118 --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h @@ -0,0 +1,2249 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +// Encoding reference: Mesa NAK sm70_encode.rs (MIT), Copyright 2022 Collabora, Ltd. +#pragma once + +#include <cstdlib> +#include <algorithm> +#include <array> +#include <bit> +#include <span> +#include <optional> +#include <string_view> +#include <vector> +#include "common/common_types.h" + +namespace Shader::Ampere { + +struct Instruction { + std::array<u32, 4> words; + + u32 Bits(u32 offset, u32 count) const { + u64 bits = words[offset / 32]; + if (offset / 32 + 1 < words.size()) { + bits |= u64{words[offset / 32 + 1]} << 32; + } + return static_cast<u32>((bits >> (offset % 32)) & ((u64{1} << count) - 1)); + } +}; +static_assert(sizeof(Instruction) == 16); + +struct LoweredProgram { + std::vector<u64> instructions; + std::vector<u32> source_instruction_offsets; + u32 source_size{}; + u32 required_texture_buffer = ~0U; + u32 error_offset{}; + std::string_view error; + + explicit operator bool() const { return error.empty(); } +}; + +// Lower only operations whose semantics are representable by the existing +// Maxwell frontend. These are new instructions, not reinterpretations of the +// native bytes. Scheduling/control words are added by the environment adapter. +inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { + LoweredProgram result; + struct Relocation { u32 output_index; s64 target; }; + std::vector<Relocation> relocations; + std::vector<u32> barrier_tokens; + std::vector<u32> call_targets; + std::vector<u32> call_links; + bool main_exited = false; + bool computed_branch = false; // a jump table: the program ends at its self-branch, not an EXIT + bool uses_uniform_bank = false; + for (const auto& instruction : code) { + const u32 op = instruction.Bits(0, 12); + uses_uniform_bank |= op == 0x3c4 || op == 0x2bd || op == 0xc0c || op == 0x986 || op == 0x9a8 || op == 0xc12 || op == 0xc17 || op == 0xe24 || op == 0xc24 || op == 0xd00 || op == 0xd09 || op == 0x9c3 || op == 0xc45 || op == 0x2a4 || op == 0x891 || op == 0x890 || op == 0x290 || op == 0x28c || op == 0x88c || op == 0x287 || op == 0x887 || op == 0x958 || op == 0xab9 || op == 0xabb || op == 0x899 || op == 0xc20 || op == 0xe21 || op == 0xe23 || op == 0xc02 || + op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || op == 0x292 || op == 0x892 || op == 0x291 || op == 0x981 || op == 0xc13 || op == 0xc10 || op == 0xc23 || op == 0x526 || op == 0x2be || op == 0x8a4 || op == 0xd05 || (op == 0x984 && instruction.Bits(91, 1)) || op == 0x89c || op == 0x883 || op == 0x2a5 || op == 0xd06 || op == 0x299 || op == 0xc09 || (op == 0x81c && instruction.Bits(67, 1)); + } + const auto address_of = [](u32 index) { return 0x50U + index * 8 + ((index + 1) / 3) * 8; }; + const auto fail = [&](u32 index, std::string_view reason) { + result.instructions.clear(); + result.source_instruction_offsets.clear(); + result.error_offset = index * sizeof(Instruction); + result.error = reason; + return result; + }; + const auto finish = [&](u32 index) { + for (const auto& relocation : relocations) { + if (relocation.target > index) return fail(index, "native branch or call leaves captured program"); + const u32 target = result.source_instruction_offsets[static_cast<size_t>(relocation.target)]; + const s64 relative = s64{address_of(target)} - address_of(relocation.output_index) - 8; + if (relative < -0x800000 || relative > 0x7fffff) return fail(index, "lowered branch distance"); + result.instructions[relocation.output_index] |= (static_cast<u64>(relative) & 0xffffff) << 20; + } + result.source_size = (index + 1) * sizeof(Instruction); + return result; + }; + if (code.empty() || code.size() > 65536) return fail(0, "invalid native program size"); + for (u32 index = 0; index < code.size(); ++index) { + auto insn = code[index]; + result.source_instruction_offsets.push_back(static_cast<u32>(result.instructions.size())); + u32 opcode = insn.Bits(0, 12); + if (insn.Bits(12, 4) == 15) continue; // @!PT: never executes (compiler padding) + std::optional<u32> active_link; + u32 active_entry = 0; + if (main_exited) { + for (size_t call = 0; call < call_targets.size(); ++call) { + if (call_targets[call] <= index && call_targets[call] >= active_entry) { + active_entry = call_targets[call]; + active_link = call_links[call]; + } + } + if (active_link && insn.Bits(16, 8) == *active_link) + return fail(index, "native callee overwrites link register"); + } + // Scratch values and emulated uniform state occupy reserved high GPRs. + // Reject native accesses that could alias that state. + for (u32 bit : {16U, 24U, 32U, 64U}) { + if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66) continue; + const u32 reg = insn.Bits(bit, 8); + if (reg >= (uses_uniform_bank ? 216U : 246U) && reg < 255) return fail(index, "native program uses lowering scratch registers"); + } + // UR0-UR21 live in R216-R237 and UR22-UR27 in R240-R245 (R239 holds the uniform + // predicates), so even/odd pairs stay adjacent. Sites that still check reg < 16 rely on + // UR0-UR15 being R216-R231. + const auto uniform_register = [&](u32 reg) -> u32 { + if (reg < 22) return 216 + reg; + if (reg < 28) return 218 + reg; + if (reg == 63) return 255; + return ~0U; + }; + bool rewritten_uniform = false; + bool uniform_guard = false; + u32 uniform_output = 7; // ULOP3's UPn output: computed in P(6-n), then merged into R239 + if (opcode == 0x2a4 || opcode == 0x8a4 || opcode == 0x891 || opcode == 0x890 || opcode == 0x290 || + opcode == 0x292 || opcode == 0x892 || opcode == 0x291) { + if (insn.Bits(12, 4) != 7 && opcode != 0x292 && opcode != 0x892) + return fail(index, "native uniform arithmetic predication"); + uniform_guard = insn.Bits(12, 4) != 7; + rewritten_uniform = true; + const u32 dst = uniform_register(insn.Bits(16, 8)); + const u32 a = uniform_register(insn.Bits(24, 8)); + const u32 c = uniform_register(insn.Bits(64, 8)); + if (dst == ~0U || a == ~0U || c == ~0U) return fail(index, "native uniform arithmetic register"); + insn.words[0] = (insn.words[0] & 0xffff) | (dst << 16) | (a << 24); + insn.words[2] = (insn.words[2] & ~0x080000ffU) | c; + if (opcode == 0x2a4 || opcode == 0x290 || opcode == 0x292 || opcode == 0x291) { + const u32 b = uniform_register(insn.Bits(32, 8)); + if (b == ~0U) return fail(index, "native uniform arithmetic second register"); + insn.words[1] = (insn.words[1] & ~0xffU) | b; + } + if ((opcode == 0x292 || opcode == 0x892) && insn.Bits(80, 4) != 0xe) { + if (insn.Bits(80, 1)) return fail(index, "native uniform logic predicate output"); + uniform_output = insn.Bits(81, 3); + insn.words[2] = (insn.words[2] & ~(7U << 17)) | ((6 - uniform_output) << 17); + } + opcode = opcode == 0x2a4 ? 0x224 : opcode == 0x8a4 ? 0x824 : opcode == 0x891 ? 0x811 : opcode == 0x890 ? 0x810 + : opcode == 0x292 ? 0x212 : opcode == 0x892 ? 0x812 : opcode == 0x291 ? 0x211 : 0x210; + } + const bool uniform_predication = opcode == 0xabb || opcode == 0xab9 || opcode == 0xc82 || opcode == 0x882; + const u32 native_predicate = insn.Bits(12, 4); + const bool guarded_uniform = (uniform_predication || uniform_guard) && native_predicate != 7; + // A uniform guard UPn lives in R239 bit 6-n and is swapped into P(6-n) for the op. + const u32 guard_slot = 6 - (native_predicate & 7); + u64 predicate = u64{guarded_uniform ? ((native_predicate & 8) | guard_slot) : native_predicate} << 16; + const u64 dest = insn.Bits(16, 8); + const auto base = [&](u64 opcode_bits) { return opcode_bits | predicate | dest; }; + // NVN2 fullscreen vertex shaders read bank 24, which the game never binds (the console reads + // zeros from an unbound bank); Maxwell has 18, so read it from bank 17, also never bound. + const auto bank = [&]() -> u32 { + const u32 value = insn.Bits(54, 5); + return value == 24 ? 17U : value; + }; + const auto cbuf = [&]() -> u64 { + return (u64{insn.Bits(38, 16) / 4} << 20) | (u64{bank()} << 34); + }; + const auto valid_cbuf = [&] { + return !insn.Bits(91, 1) && bank() < 18 && + (insn.Bits(38, 16) % 4) == 0; + }; + std::string_view operand_error; + const auto emit = [&](u64 word) { result.instructions.push_back(word); }; + // P(guard slot) while a uniform guard is swapped in; P6 for lowerings that use it as + // scratch (R2UR, MATCH, HFMA2.RELU), which save and restore it with the defaults below. + const u32 guard_mask = guard_slot < 7 ? 1U << guard_slot : 0U; + const auto save_predicate = [&](u32 mask = 1U << 6) { + emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + }; + const auto restore_predicate = [&](u32 mask = 1U << 6) { + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + }; + if (guarded_uniform) { + save_predicate(guard_mask); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{guard_mask} << 20)); + } + const u32 output_mask = uniform_output == 7 ? 0U : 1U << (6 - uniform_output); + if (output_mask) { + // Keep the guest P(6-n) in R254 and start from the current UPn (a false guard keeps it). + emit(0x38e8000000070000ULL | 254 | (u64{255} << 8) | (u64{output_mask} << 20)); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{output_mask} << 20)); + } + // Pd = LUT(a, b, c) over predicate operands (index | negate << 3): each input becomes + // 0 or ~0 (SEL), and LOP3 sets Pd when the result is nonzero. + const auto predicate_lut = [&](u32 lut, u32 a, u32 b, u32 c, u32 target, bool uniform_c = false) { + const u32 inputs[3]{a, b, c}; + for (u32 i = 0; i < 3; ++i) { + if (i == 2 && uniform_c) { + // UPn from R239 bit 6-n as 0 or ~0 (UPT is ~0); a negation flips the table. + if ((c & 7) == 7) { + emit(0x0100000000000000ULL | predicate | 252 | (u64{15} << 12) | (u64{~0U} << 20)); + } else { + emit(0x3800000000000000ULL | predicate | 252 | (u64{239} << 8) | + (u64{(6 - (c & 7)) | (1U << 8)} << 20)); + emit(0x5c10000000000000ULL | predicate | 252 | (u64{255} << 8) | + (u64{252} << 20) | (u64{1} << 48)); + } + if (c & 8) lut = ((lut & 0x55) << 1) | ((lut & 0xaa) >> 1); + continue; + } + emit(0x38a0000000000000ULL | predicate | (250 + i) | (u64{255} << 8) | + (u64{0x7ffff} << 20) | (u64{1} << 56) | (u64{inputs[i] ^ 8U} << 39)); + } + emit(0x5be0000000000000ULL | predicate | 255 | (u64{250} << 8) | (u64{251} << 20) | + (u64{lut} << 28) | (u64{3} << 36) | (u64{252} << 39) | (u64{target} << 48)); + }; + const auto mov_imm = [&](u32 reg, u32 value) { + emit(0x0100000000000000ULL | predicate | reg | (u64{15} << 12) | (u64{value} << 20)); + }; + const auto materialize = [&](u32 kind, u32 reg, u32 scratch, bool absolute, bool negate) -> u32 { + if (kind == 3) { + const u32 mapped = uniform_register(reg); + if (mapped == ~0U) { operand_error = "native uniform source register"; return 255; } + reg = mapped; + } else if (kind == 1) { + mov_imm(scratch, insn.words[1]); + reg = scratch; + } else if (kind == 2) { + if (!valid_cbuf()) { operand_error = "native constant buffer address"; return 255; } + emit(0x4c98000000000000ULL | predicate | scratch | (u64{15} << 39) | cbuf()); + reg = scratch; + } + if (absolute || negate) { + if (reg != scratch) emit(0x5c98000000000000ULL | predicate | scratch | (u64{reg} << 20) | (u64{15} << 39)); + if (absolute) emit(0x0400000000000000ULL | predicate | scratch | (u64{scratch} << 8) | (u64{0x7fffffff} << 20)); + if (negate) emit(0x0400000000000000ULL | predicate | scratch | (u64{scratch} << 8) | (u64{0x80000000} << 20) | (u64{2} << 53)); + reg = scratch; + } + return reg; + }; + const auto operand = [&](u32 which, bool floating = true) -> u32 { + const u32 form = insn.Bits(9, 3); + u32 reg{}, kind{}, abs{}, neg{}; + if (which == 0) { + reg = insn.Bits(24, 8); abs = insn.Bits(73, 1); neg = insn.Bits(72, 1); + } else { + if (form != 1 && form != 2 && form != 3 && form != 4 && form != 5 && form != 6 && form != 7) { + operand_error = "native ALU source form"; return 255; + } + const bool wide = which == 1 ? (form == 4 || form == 5 || form == 6) : (form == 2 || form == 3 || form == 7); + if (wide) { + kind = form == 6 || form == 7 ? 3 : (form == 4 || form == 2) ? 1 : 2; + if (kind == 3) reg = insn.Bits(32, 8); + if (kind == 2 || kind == 3) { abs = insn.Bits(62, 1); neg = insn.Bits(63, 1); } + } else { + const bool upper = which == 1 ? (form == 2 || form == 3 || form == 7) : true; + reg = insn.Bits(upper ? 64 : 32, 8); + abs = insn.Bits(upper ? 74 : 62, 1); neg = insn.Bits(upper ? 75 : 63, 1); + } + } + return materialize(kind, reg, 250 + which, floating && abs, floating && neg); + }; + struct HalfSource { u32 reg, swizzle; bool absolute, negate; }; + const auto half_source = [&](u32 which) -> HalfSource { + if (which == 0) return {insn.Bits(24, 8), insn.Bits(74, 2), + insn.Bits(73, 1) != 0, insn.Bits(72, 1) != 0}; + const u32 form = insn.Bits(9, 3); + const bool wide = which == 1 ? (form == 4 || form == 5) : (form == 2 || form == 3); + if (wide) { + const bool immediate = form == 2 || form == 4; + const u32 reg = materialize(immediate ? 1 : 2, 255, 250 + which, false, false); + return {reg, immediate ? 0U : insn.Bits(60, 2), + !immediate && insn.Bits(62, 1), !immediate && insn.Bits(63, 1)}; + } + const bool upper = which == 2 || form == 2 || form == 3; + return {insn.Bits(upper ? 64 : 32, 8), insn.Bits(upper ? 81 : 60, 2), + insn.Bits(upper ? 83 : 62, 1) != 0, insn.Bits(upper ? 84 : 63, 1) != 0}; + }; + const auto extract_half = [&](HalfSource source, u32 lane, u32 target) { + const bool high = source.swizzle == 3 || (source.swizzle == 0 && lane == 1); + emit(0x5ca8000000000000ULL | predicate | target | (u64{2} << 8) | + (u64{source.swizzle == 1 ? 2U : 1U} << 10) | (u64{source.reg} << 20) | + (u64{high} << 41) | (u64{source.negate} << 45) | (u64{source.absolute} << 49)); + }; + const auto xmad = [&](u32 target, u32 a, u32 b, u32 c, bool a_high, + bool b_high, u32 mode, bool immediate) { + emit((immediate ? 0x3600000000000000ULL : 0x5b00000000000000ULL) | + predicate | target | (u64{a} << 8) | + (u64{immediate ? ((b >> (b_high ? 16 : 0)) & 0xffff) : b} << 20) | + (u64{!immediate && b_high} << 35) | (u64{c} << 39) | + (u64{mode} << 50) | (u64{a_high} << 53)); + }; + const auto multiply_low = [&](u32 a, u32 b, u32 c, bool immediate) { + xmad(250, a, b, c, false, false, 0, immediate); + xmad(251, a, b, 255, true, false, 0, immediate); + xmad(251, a, b, 251, false, true, 0, immediate); + emit(base(0x5c18000000000000ULL) | (u64{251} << 8) | + (u64{250} << 20) | (u64{16} << 39)); + }; + switch (opcode) { + case 0x304: { // F2F.F16.F32, scalar round-to-nearest conversion. + if (insn.words[2] != 0x00200800 || insn.Bits(24, 8) || + (insn.words[1] & ~0xc00000ffU)) + return fail(index, "native scalar float conversion mode"); + emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(62, 1)} << 49) | + (u64{insn.Bits(63, 1)} << 45)); + break; + } + case 0x23e: case 0xa3e: { // F2FP.PACK_AB / MERGE_C, round-to-nearest + if (insn.Bits(62, 2) || insn.Bits(73, 5) || insn.Bits(79, 2) || + insn.Bits(81, 10) || insn.Bits(91, 1)) { + return fail(index, "native float packing modifiers"); + } + const bool merge = insn.Bits(78, 1); + if (insn.Bits(72, 1)) return fail(index, "native float packing selector"); + const u32 a = merge ? operand(2, false) : operand(0, false); + const u32 b = operand(1, false); + // Canonical F2F writes the converted half in its low 16 bits. + if (merge) { + emit(0x5c98000000000000ULL | predicate | 250 | (u64{a} << 20) | (u64{15} << 39)); + } else { + emit(0x5ca8000000000000ULL | predicate | 250 | (u64{1} << 8) | + (u64{2} << 10) | (u64{a} << 20)); + } + emit(0x5ca8000000000000ULL | predicate | 251 | (u64{1} << 8) | + (u64{2} << 10) | (u64{b} << 20)); + emit(base(0x5c18000000000000ULL) | (u64{250} << 8) | (u64{251} << 20) | (u64{16} << 39)); + break; + } + case 0x232: case 0x832: case 0xa32: { // HMUL2 + if (insn.Bits(78, 2) || insn.Bits(81, 10) || insn.Bits(91, 1)) + return fail(index, "native half multiply mode"); + const auto a = half_source(0), b = half_source(1); + const u32 precision = insn.Bits(76, 1) ? 2 : insn.Bits(80, 1); + emit(base(0x5d08000000000000ULL) | (u64{a.reg} << 8) | (u64{b.reg} << 20) | + (u64{b.swizzle} << 28) | (u64{b.absolute} << 30) | (u64{a.negate != b.negate} << 31) | + (u64{insn.Bits(77, 1)} << 32) | (u64{precision} << 39) | + (u64{a.absolute} << 44) | (u64{a.swizzle} << 47)); + break; + } + case 0x230: case 0x430: case 0x630: { // HADD2 + if (insn.Bits(76, 1) || insn.Bits(79, 1) || insn.Bits(81, 10) || insn.Bits(91, 1)) + return fail(index, "native half add mode"); + const auto a = half_source(0); + const auto b = half_source(opcode == 0x230 ? 1 : 2); + if (insn.Bits(78, 1)) { + if (a.reg != 255 || !a.negate || a.absolute || a.swizzle != 0 || + b.swizzle < 2 || insn.Bits(77, 1) || insn.Bits(80, 1)) + return fail(index, "native float-output half add mode"); + extract_half(b, 0, static_cast<u32>(dest)); + } else { + emit(base(0x5d10000000000000ULL) | (u64{a.reg} << 8) | (u64{b.reg} << 20) | + (u64{b.swizzle} << 28) | (u64{b.absolute} << 30) | (u64{b.negate} << 31) | + (u64{insn.Bits(77, 1)} << 32) | (u64{insn.Bits(80, 1)} << 39) | + (u64{a.negate} << 43) | (u64{a.absolute} << 44) | (u64{a.swizzle} << 47)); + } + break; + } + case 0x231: case 0x431: case 0x631: case 0x831: case 0xa31: { // HFMA2 + const bool relu = insn.Bits(79, 1) != 0; + if (insn.Bits(78, 1) || insn.Bits(85, 1) || insn.Bits(91, 1) || + (relu ? (insn.Bits(87, 4) != 7 || + insn.Bits(77, 1)) : insn.Bits(87, 4) != 0)) + return fail(index, "native half fused multiply-add mode"); + if (relu && native_predicate != 7) { + // Guard the whole expansion before its temporary P6 updates. + relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1}); + emit(0xe24000000000000fULL | (u64{native_predicate ^ 8U} << 16)); + predicate = u64{7} << 16; + } + auto a = half_source(0); + auto b = half_source(1); + const auto c = half_source(2); + if (insn.Bits(86, 1)) { // H0_NH1: negate only the upper packed component. + if (b.swizzle != 0 || b.absolute || b.reg >= 250) + return fail(index, "native half mixed-sign source mode"); + emit(0x0400000000000000ULL | predicate | 251 | (u64{b.reg} << 8) | + (u64{0x80000000} << 20) | (u64{2} << 53)); + b.reg = 251; + } + if (b.absolute || c.absolute || (a.absolute && a.reg >= 248)) + return fail(index, "native half fused multiply-add absolute source"); + if (a.absolute) { + emit(0x0400000000000000ULL | predicate | 250 | (u64{a.reg} << 8) | + (u64{a.swizzle == 1 ? 0x7fffffffU : 0x7fff7fffU} << 20)); + a.reg = 250; + a.absolute = false; + } + const u32 precision = insn.Bits(76, 1) ? 2 : insn.Bits(80, 1); + emit(0x5d00000000000000ULL | predicate | (relu ? 254U : dest) | (u64{a.reg} << 8) | (u64{b.reg} << 20) | + (u64{b.swizzle} << 28) | (u64{c.negate} << 30) | (u64{a.negate != b.negate} << 31) | + (u64{insn.Bits(77, 1)} << 32) | (u64{c.swizzle} << 35) | + (u64{precision} << 37) | (u64{c.reg} << 39) | (u64{a.swizzle} << 47)); + if (relu) { + // Clamp the rounded binary16 lanes, preserving positive infinity + // and canonicalizing either sign of NaN without saturating to one. + save_predicate(); + for (u32 lane = 0; lane < 2; ++lane) { + emit(0x3828000000070000ULL | 251 | (u64{254} << 8) | (u64{lane * 16} << 20)); + emit(0x0400000000070000ULL | 251 | (u64{251} << 8) | (u64{0xffff} << 20)); + emit(0x0400000000070000ULL | 252 | (u64{251} << 8) | (u64{0x7fff} << 20)); + mov_imm(250, 0x7c00); + emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{250} << 8) | + (u64{252} << 20) | (u64{7} << 39) | (u64{1} << 49)); + emit(0x3828000000070000ULL | 252 | (u64{251} << 8) | (u64{15} << 20)); + mov_imm(250, 1); + emit(0x5c10000000070000ULL | 252 | (u64{252} << 8) | (u64{250} << 20) | (u64{1} << 48)); + emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48)); + mov_imm(250, 0x7fff); + emit(0x5ca0000000070000ULL | 251 | (u64{250} << 8) | (u64{251} << 20) | (u64{6} << 39)); + if (lane == 0) { + emit(base(0x5c98000000000000ULL) | (u64{251} << 20) | (u64{15} << 39)); + } else { + emit(0x3848000000070000ULL | 251 | (u64{251} << 8) | (u64{16} << 20)); + emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{251} << 20) | + (u64{1} << 41) | (u64{7} << 48)); + } + } + restore_predicate(); + } + break; + } + case 0x240: case 0x840: { // HMNMX2, exact half promotion for min/max + if (insn.Bits(76, 11) || insn.Bits(91, 1)) + return fail(index, "native half min/max mode"); + const auto a = half_source(0); + const auto b = opcode == 0x840 ? HalfSource{255, 0, false, false} : half_source(1); + if (a.swizzle == 1 || b.swizzle == 1 || b.reg >= 250 && b.reg != 255) + return fail(index, "native half min/max source mode"); + for (u32 lane = 0; lane < 2; ++lane) { + extract_half(a, lane, 250); + if (opcode == 0x840) { + const u32 half = (insn.words[1] >> (lane * 16)) & 0xffff; + mov_imm(251, half); + extract_half(HalfSource{251, 2, false, false}, 0, 251); + } else { + extract_half(b, lane, 251); + } + const u32 target = lane == 0 ? 252 : 250; + emit(0x5c60000000000000ULL | predicate | target | (u64{250} << 8) | (u64{251} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(0x5ca8000000000000ULL | predicate | target | (u64{1} << 8) | + (u64{2} << 10) | (u64{target} << 20)); + } + emit(base(0x5c18000000000000ULL) | (u64{250} << 8) | (u64{252} << 20) | (u64{16} << 39)); + break; + } + case 0xc24: case 0xe24: { // IMAD low word with a uniform multiplier or addend. + // 0xe24 keeps the uniform addend in bits 32-37; bit 63 negates it ("RZ, RZ, -UR4"). + const bool negate_c = opcode == 0xe24 && insn.Bits(63, 1) != 0; + if ((insn.words[2] & ~0x080002ffU) != 0x078e0000 || + (insn.words[1] & ~(0xffU | (negate_c ? 0x80000000U : 0U)))) + return fail(index, "native uniform-addend multiply mode"); + const u32 a = operand(0, false), b = operand(1, false); + u32 c = operand(2, false); + if (negate_c) { + emit(0x5c10000000000000ULL | predicate | 252 | (u64{255} << 8) | (u64{c} << 20) | + (u64{1} << 48)); + c = 252; + } + multiply_low(a, b, c, false); + break; + } + case 0x224: { // IMAD register / zero-product move + if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 && insn.Bits(75, 1) && + (insn.words[2] & ~0xaffU) == 0x078e0000) { + emit(base(0x5c10000000000000ULL) | (u64{255} << 8) | + (u64{insn.Bits(64, 8)} << 20) | (u64{1} << 48)); + break; + } + if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 && + (insn.words[2] & ~0xffU) == 0x078e0000) { + emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.Bits(64, 8)} << 20)); + break; + } + if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (insn.words[1] & ~0xffU)) + return fail(index, "native integer register multiply mode"); + multiply_low(insn.Bits(24, 8), insn.Bits(32, 8), insn.Bits(64, 8), false); + break; + } + case 0x2a5: { // UIMAD.WIDE.U32 UR(d):UR(d+1) = URa * URb + URc:UR(c+1) + const u32 a = uniform_register(insn.Bits(24, 8)), b = uniform_register(insn.Bits(32, 8)); + const u32 c = insn.Bits(64, 8); + const u32 c_low = uniform_register(c), c_high = c == 63 ? 255U : uniform_register(c + 1); + const u32 d_low = uniform_register(insn.Bits(16, 8)), d_high = uniform_register(insn.Bits(16, 8) + 1); + if (native_predicate != 7 || (insn.words[1] & ~0xffU) || (insn.words[2] & ~0xffU) != 0x0f8e0000 || + (insn.Bits(16, 8) & 1) || (c != 63 && (c & 1)) || a == ~0U || b == ~0U || c_low == ~0U || + c_high == ~0U || d_low == ~0U || d_high == ~0U || d_low == 255) + return fail(index, "native uniform wide multiply mode"); + // High word in R253 (base-2^16 long multiplication, as IMAD.HI), low word in R250. + xmad(250, a, b, 255, false, false, 0, false); + xmad(251, a, b, 250, true, false, 2, false); + xmad(250, a, b, 251, false, true, 1, false); + xmad(252, a, b, 251, true, true, 2, false); + emit(0x3828000000070000ULL | 250 | (u64{250} << 8) | (u64{16} << 20)); + emit(0x5c10000000070000ULL | 253 | (u64{250} << 8) | (u64{252} << 20)); + xmad(250, a, b, 255, false, false, 0, false); + xmad(251, a, b, 255, true, false, 0, false); + xmad(251, a, b, 251, false, true, 0, false); + emit(0x5c18000000070000ULL | 250 | (u64{251} << 8) | (u64{250} << 20) | (u64{16} << 39)); + emit(0x5c10000000070000ULL | 250 | (u64{250} << 8) | (u64{c_low} << 20) | (u64{1} << 47)); + emit(0x5c10000000070000ULL | d_high | (u64{253} << 8) | (u64{c_high} << 20) | (u64{1} << 43)); + emit(0x5c10000000070000ULL | d_low | (u64{250} << 8) | (u64{255} << 20)); + break; + } + case 0x827: { // IMAD.HI.U32, captured zero addend + if (insn.words[2] != 0x078e00ff) return fail(index, "native high multiply mode"); + const u32 a = insn.Bits(24, 8), b = insn.words[1]; + // Base-2^16 long multiplication. Each intermediate sum fits U32. + xmad(250, a, b, 255, false, false, 0, true); + xmad(251, a, b, 250, true, false, 2, true); + xmad(250, a, b, 251, false, true, 1, true); + xmad(252, a, b, 251, true, true, 2, true); + emit(0x3828000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{16} << 20)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20)); + break; + } + case 0xa24: { // IMAD, constant-buffer multiplier + if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || !valid_cbuf() || insn.Bits(62, 2)) + return fail(index, "native integer constant-buffer multiply mode"); + emit(0x4c98000000000000ULL | predicate | 252 | (u64{15} << 39) | cbuf()); + multiply_low(insn.Bits(24, 8), 252, insn.Bits(64, 8), false); + break; + } + case 0x424: case 0x624: { // IMAD.MOV.U32: zero product plus immediate/cbuf + if (opcode == 0x624 && insn.Bits(24, 8) == 255 && + (insn.words[2] & ~0x07800000U) == 0x000e06ffU) { + const u32 source = operand(2, false); + mov_imm(250, 1); + emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(base(0x5c10000000000000ULL) | (u64{source} << 8) | (u64{250} << 20)); + break; + } + if (insn.Bits(24, 8) != 255 || insn.Bits(64, 8) != 255 || insn.words[2] != 0x078e00ff) { + if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (opcode == 0x624 && insn.Bits(62, 2))) + return fail(index, "native multiply wide addend mode"); + const u32 a = operand(0, false), b = operand(1, false), c = operand(2, false); + multiply_low(a, b, c, false); + break; + } + if (insn.Bits(24, 8) != 255 || insn.Bits(64, 8) != 255 || + insn.words[2] != 0x078e00ff) + return fail(index, "native integer move mode"); + if (opcode == 0x624) { + if (!valid_cbuf() || insn.Bits(62, 2)) + return fail(index, "native integer move constant buffer"); + emit(base(0x4c98000000000000ULL) | (u64{15} << 39) | cbuf()); + } else { + emit(base(0x0100000000000000ULL) | (u64{15} << 12) | + (u64{insn.words[1]} << 20)); + } + break; + } + case 0x824: { // Captured low-word IMAD forms; bit 75 negates the addend. + const bool negate_c = insn.Bits(75, 1) != 0; + if (insn.words[1] == 1 && (insn.words[2] & 0xfffff700U) == 0x078e0200) { + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(64, 8)} << 20) | (u64{negate_c} << 48)); + break; + } + const u32 multiplier = insn.words[1]; + if (insn.Bits(64, 8) == 255 && !negate_c && multiplier > 1 && !(multiplier & (multiplier - 1)) && + (insn.words[2] & ~0x2ffU) == 0x078e0000) { + // IMAD.SHL by a power of two: one shift (also the form Eden's jump-table tracking + // expects between the clamp and the table load). + u32 shift = 0; + while ((1U << shift) != multiplier) ++shift; + emit(base(0x3848000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{shift} << 20)); + break; + } + if (insn.words[1] != 48 || insn.words[2] != 0x078e02ff) { + if ((insn.words[2] & ~0xaffU) != 0x078e0000) + return fail(index, "native integer multiply mode"); + u32 c = insn.Bits(64, 8); + if (negate_c && c != 255) { + emit(0x5c10000000000000ULL | predicate | 252 | (u64{255} << 8) | (u64{c} << 20) | + (u64{1} << 48)); + c = 252; + } + multiply_low(insn.Bits(24, 8), insn.words[1], c, true); + break; + } + const u64 source = insn.Bits(24, 8); + emit(0x5c18000000000000ULL | predicate | 250 | (source << 8) | (source << 20) | (u64{1} << 39)); + emit(base(0x3848000000000000ULL) | (u64{250} << 8) | (u64{4} << 20)); + break; + } + case 0x806: { // VOTE: register ballot, ordinary predicate input and output. + if (insn.words[1] || insn.Bits(72, 2) > 2 || (insn.words[2] & ~0x078e0300U)) + return fail(index, "native vote mode"); + emit(base(0x50d8000000000000ULL) | (u64{insn.Bits(87, 4)} << 39) | + (u64{insn.Bits(81, 3)} << 45) | (u64{insn.Bits(72, 2)} << 48)); + break; + } + case 0x886: { // VOTEU ballot and independent uniform predicate output. + const u32 output = insn.Bits(81, 3), target = insn.Bits(16, 8); + if ((target >= 16 && target != 63) || insn.words[1] || + insn.Bits(72, 2) > 2 || native_predicate != 7) + return fail(index, "native uniform vote mode"); + const u32 temporary = output == 7 ? 7 : 6 - output; + const u32 mask = 1U << temporary; + if (output != 7) emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + emit(0x50d8000000070000ULL | uniform_register(target) | + (u64{insn.Bits(87, 4)} << 39) | (u64{temporary} << 45) | (u64{insn.Bits(72, 2)} << 48)); + if (output != 7) { + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20)); + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + } + break; + } + case 0x3c2: { // R2UR: broadcast the first lane satisfying the execution predicate. + const u32 target = uniform_register(static_cast<u32>(dest)); + if (target == ~0U || target == 255 || insn.words[1] || insn.words[2] != 0x000e0000) + return fail(index, "native register-to-uniform mode"); + save_predicate(); + emit(0x50d8000000070000ULL | 250 | (u64{native_predicate} << 39) | + (u64{6} << 45) | (u64{1} << 48)); + emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49)); + emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48)); + emit(0x5c30000000070000ULL | 250 | (u64{250} << 20)); + emit(0xef10000000070000ULL | 251 | (u64{insn.Bits(24, 8)} << 8) | + (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x5ca0000000070000ULL | target | (u64{251} << 8) | + (u64{target} << 20) | (u64{6} << 39)); + restore_predicate(); + break; + } + case 0x81c: { // PLOP3.LUT; bit 67 makes the third source a uniform predicate. + const u32 target = insn.Bits(81, 3); + const u32 lut = insn.Bits(64, 3) | (insn.Bits(72, 5) << 3); + if (target == 7 || insn.Bits(84, 3) != 7 || + insn.Bits(16, 8) || insn.words[1]) + return fail(index, "native predicate LUT mode"); + if (!insn.Bits(67, 1) && (lut == 0xfe || lut == 0xa8)) { + emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 | + (u64{insn.Bits(87, 4)} << 12) | (u64{1} << 24) | + (u64{insn.Bits(77, 4)} << 29) | (u64{insn.Bits(68, 4)} << 39) | + (u64{lut == 0xfe} << 45)); + break; + } + if (!insn.Bits(67, 1) && (lut == 0x80 || + (lut == 8 && insn.Bits(68, 4) == 7 && insn.Bits(77, 4) == 7))) { + emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 | + (u64{insn.Bits(87, 4) ^ (lut == 8 ? 8U : 0U)} << 12) | (u64{insn.Bits(77, 4)} << 29) | + (u64{insn.Bits(68, 4)} << 39)); + break; + } + predicate_lut(lut, insn.Bits(87, 4), insn.Bits(77, 4), insn.Bits(68, 4), target, + insn.Bits(67, 1) != 0); + break; + } + case 0x89c: { // UPLOP3.LUT: load the uniform predicates into P0-P6, combine, merge back. + const u32 target = insn.Bits(81, 3); + const u32 lut = insn.Bits(64, 3) | (insn.Bits(72, 5) << 3); + if (insn.Bits(84, 3) != 7 || insn.Bits(67, 1) || insn.Bits(16, 8) || insn.words[1]) + return fail(index, "native uniform predicate LUT mode"); + if (target == 7) break; + const auto slot = [](u32 operand) { return (operand & 7) == 7 ? operand : (operand & 8) | (6 - (operand & 7)); }; + predicate = u64{slot(native_predicate)} << 16; // a UPn guard is loaded as well + emit(0x38e8000000070000ULL | 254 | (u64{255} << 8) | (u64{0x7f} << 20)); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{0x7f} << 20)); + predicate_lut(lut, slot(insn.Bits(87, 4)), slot(insn.Bits(77, 4)), slot(insn.Bits(68, 4)), 6 - target); + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{1U << (6 - target)} << 20)); + emit(0x38f0000000070000ULL | (u64{254} << 8) | (u64{0x7f} << 20)); + break; + } + case 0x883: { // UP2UR UR, UPR, URZ, mask: bit k of the result is UPk (R239 bit 6-k). + const u32 target = uniform_register(insn.Bits(16, 8)), mask = insn.words[1]; + if (target == ~0U || insn.Bits(24, 8) != 63 || native_predicate != 7 || + insn.words[2] != 0x08000000 || mask > 0x7f) + return fail(index, "native uniform predicate move mode"); + if (target == 255) break; + mov_imm(target, 0); + for (u32 k = 0; k < 7; ++k) { + if (!((mask >> k) & 1)) continue; + emit(0x3800000000070000ULL | 250 | (u64{239} << 8) | (u64{(6 - k) | (1U << 8)} << 20)); + emit(0x36f0000000070000ULL | target | (u64{250} << 8) | (u64{k | (1U << 8)} << 20) | + (u64{target} << 39)); + } + break; + } + case 0x882: { // UMOV immediate. + if (dest >= 16 || insn.words[2]) return fail(index, "native uniform immediate move mode"); + mov_imm(static_cast<u32>(dest) + 216, insn.words[1]); + break; + } + case 0xc82: { // UMOV UR, UR (or URZ), including uniform execution predicates. + const u32 source = insn.words[1] < 64 ? uniform_register(insn.words[1]) : ~0U; + if (dest >= 16 || source == ~0U || insn.words[2] != 0x08000000) + return fail(index, "native uniform move mode"); + emit(0x5c98000000000000ULL | predicate | (dest + 216) | + (u64{source} << 20) | (u64{15} << 39)); + break; + } + case 0xab9: case 0xabb: { // ULDC U32/U64, direct or indexed. + const u32 uniform_dest = insn.Bits(16, 8), index_reg = insn.Bits(24, 8); + const u32 width = insn.Bits(73, 3); + const u32 target = uniform_register(uniform_dest); + const u32 index_target = opcode == 0xabb ? uniform_register(index_reg) : 255U; + if (target == ~0U || target == 255 || (width != 4 && width != 5) || + (width == 5 && ((uniform_dest & 1) || uniform_register(uniform_dest + 1) != target + 1)) || + bank() >= 18 || insn.Bits(38, 16) % 4 || insn.Bits(62, 2) || + (insn.words[2] & ~0xe00U) != (opcode == 0xab9 ? 0U : 0x08000000U) || + index_target == ~0U) + return fail(index, "native uniform constant load mode"); + if (opcode == 0xab9 && width == 4) { + emit(0x4c98000000000000ULL | predicate | target | (u64{15} << 39) | cbuf()); + } else { + emit(0xef90000000000000ULL | predicate | target | + (u64{index_target} << 8) | (u64{insn.Bits(38, 16)} << 20) | + (u64{bank()} << 36) | (u64{width} << 48)); + } + break; + } + case 0x899: { // USHF.L.U32; accept a zero word outside the selected half. + const u32 dst = insn.Bits(16, 8), low = insn.Bits(24, 8), high = insn.Bits(64, 8); + if (insn.words[2] == (0x08011400U | high) && low == 63 && high < 16 && + dst < 16 && native_predicate == 7 && insn.words[1] <= 32) { + emit(0x3828000000000000ULL | predicate | (dst + 216) | + (u64{high + 216} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48)); + break; + } + // Logical right shifts of one register (bit 76 right, bit 80 high word, bits 73-74 + // U32): USHF.R.U32 URd, URa, n, URZ and USHF.R.U32.HI URd, URZ, n, URb. + const bool right_low = insn.words[2] == 0x0800163f && low < 16; + const bool right_high = insn.words[2] == (0x08011600U | high) && low == 63 && high < 16; + if ((right_low || right_high) && dst < 16 && native_predicate == 7 && insn.words[1] <= 32) { + emit(0x3828000000000000ULL | predicate | (dst + 216) | + (u64{(right_high ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20)); + break; + } + const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high < 16; + const bool low_result = insn.words[2] == 0x0800063f && low < 16; + if (dst >= 16 || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result)) + return fail(index, "native uniform shift mode"); + emit(0x3848000000000000ULL | predicate | (dst + 216) | + (u64{(high_result ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20)); + break; + } + case 0xb82: { // LDC.64 indexed byte offset + const u32 width = insn.Bits(73, 3); + if ((width != 4 && width != 5) || insn.Bits(78, 2) || insn.Bits(91, 1) || + bank() >= 18 || + (width == 5 && ((dest & 1) || dest >= 254))) return fail(index, "native indexed constant load mode"); + emit(base(0xef90000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(38, 16)} << 20) | (u64{bank()} << 36) | (u64{width} << 48)); + break; + } + case 0xf89: { // SHFL with immediate lane and segmentation/clamp mask, any mode. + const u32 lane = insn.Bits(53, 5), mask = insn.Bits(40, 13), mode = insn.Bits(58, 2); + if (insn.words[2] != 0x000e0000 || insn.Bits(32, 8) || insn.Bits(60, 4)) + return fail(index, "native immediate shuffle mode"); + emit(base(0xef10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{lane} << 20) | (u64{1} << 28) | (u64{1} << 29) | + (u64{mode} << 30) | (u64{mask} << 34) | (u64{7} << 48)); + break; + } + case 0x822: { // FSWZADD: per lane 0 = a+b, 1 = a-b, 2 = b-a, 3 = b (nvdisasm PP/PN/NP/ZP). + const u32 swizzle = insn.words[1]; + if (swizzle > 0xff || insn.words[2] > 255) + return fail(index, "native swizzled add mode"); + // Maxwell's lane codes 1 and 2 are the other way round (1 = b-a, 2 = a-b). + const u32 maxwell = ((swizzle & 0x55) << 1) | ((swizzle & 0xaa) >> 1); + emit(base(0x50f8000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(64, 8)} << 20) | (u64{maxwell} << 28)); + break; + } + case 0xb6c: { // TXD ARRAY_2D, four channels and consecutive split destinations. + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const u32 coordinates = insn.Bits(24, 8), derivatives = insn.Bits(32, 8); + if ((insn.words[2] & ~0xffU) != 0x001e0f00 || insn.Bits(59, 2) || + insn.Bits(61, 3) != 5 || binding >= 18 || offset > 0x1fff || + (dest & 1) || dest >= 248 || insn.Bits(64, 8) != dest + 2 || + coordinates >= 248 || derivatives >= 247) + return fail(index, "native array gradient texture mode"); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native texture descriptor buffer conflict"); + result.required_texture_buffer = binding; + emit(base(0xde00000000000000ULL) | (u64{coordinates} << 8) | + (u64{derivatives} << 20) | (u64{3} << 28) | (u64{15} << 31) | + (u64{offset} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51)); + break; + } + case 0x989: { // SHFL, immediate lane and register clamp/segmentation mask, any mode. + if ((insn.words[1] & 0xf01fffffU) || (insn.words[2] & ~0x000e00ffU)) + return fail(index, "native register-mask shuffle mode"); + emit(base(0xef10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(53, 5)} << 20) | (u64{1} << 28) | (u64{insn.Bits(58, 2)} << 30) | + (u64{insn.Bits(64, 8)} << 39) | (u64{insn.Bits(81, 3)} << 48)); + break; + } + case 0x589: { // SHFL.IDX, register lane and immediate segmentation mask. + if ((insn.words[2] & ~0x000e0000U) || insn.Bits(53, 11)) + return fail(index, "native shuffle mode"); + emit(0xef10000000000000ULL | predicate | dest | + (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) | + (u64{1} << 29) | (u64{insn.Bits(40, 13)} << 34) | (u64{insn.Bits(81, 3)} << 48)); + break; + } + case 0x98e: { // RED.E.{ADD..XOR}.STRONG.GPU U32 (op in bits 87-89), register address. + if ((insn.words[2] & ~0x0380003fU) == 0x0c10e100 && insn.Bits(24, 8) == 255) { + // [URa + offset]: a uniform 64-bit address with a signed byte offset. + const u32 address = uniform_register(insn.Bits(64, 6)); + const s32 offset = static_cast<s32>(insn.Bits(40, 24) << 8) >> 8; + if (address == ~0U || (address & 1) || offset < -0x80000 || offset > 0x7ffff) + return fail(index, "native global reduction uniform address"); + emit(0xebf8000000000000ULL | predicate | insn.Bits(32, 8) | (u64{address} << 8) | + (u64{insn.Bits(87, 3)} << 23) | (u64{static_cast<u32>(offset) & 0xfffff} << 28) | + (u64{1} << 48)); + break; + } + if ((insn.words[2] & ~0x03800000U) != 0x0010e100 || insn.Bits(40, 24) || (insn.Bits(24, 8) & 1)) + return fail(index, "native global reduction mode"); + emit(0xebf8000000000000ULL | predicate | insn.Bits(32, 8) | + (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(87, 3)} << 23) | (u64{1} << 48)); + break; + } + case 0x2bd: // UFLO.U32, the highest set-bit index (.SH, bit 74: its shift amount). + if (dest >= 16 || insn.words[1] >= 16 || (insn.words[2] & ~0x400U) != 0x080e0000 || native_predicate != 7) + return fail(index, "native uniform leading bit mode"); + emit(0x5c30000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 20) | + (u64{insn.Bits(74, 1)} << 41)); + break; + case 0x2be: // UBREV, as a bit-reversing zero shift like BREV. + if (dest >= 16 || insn.words[1] >= 16 || insn.words[2] != 0x08000000 || native_predicate != 7) + return fail(index, "native uniform bit reverse mode"); + emit(0x3828000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 8) | + (u64{1} << 39) | (u64{1} << 40)); + break; + case 0x3a1: { // MATCH.ALL.U64 with only the all-equal predicate retained. + const u32 source = insn.Bits(24, 8), target = insn.Bits(81, 3); + if (dest != 255 || source >= 254 || (source & 1) || target >= 6 || + native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0000U) != 0x200) + return fail(index, "native warp match mode"); + save_predicate(); + emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); + emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49)); + emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48)); + emit(0x5c30000000070000ULL | 250 | (u64{250} << 20)); + for (u32 word = 0; word < 2; ++word) + emit(0xef10000000070000ULL | (251 + word) | (u64{source + word} << 8) | + (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source} << 8) | + (u64{251} << 20) | (u64{7} << 39) | (u64{2} << 49)); + emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source + 1} << 8) | + (u64{252} << 20) | (u64{6} << 39) | (u64{2} << 49)); + emit(0x50d8000000070000ULL | 255 | (u64{6} << 39) | (u64{target} << 45)); + restore_predicate(); + break; + } + case 0x3c4: { // REDUX.SUM (modulo 2^32) or .MAX (unsigned) over the actual active lanes. + const u32 source = insn.Bits(24, 8), target = uniform_register(static_cast<u32>(dest)); + const bool maximum = insn.words[2] == 0x14000; + if (target == ~0U || target == 255 || insn.words[1] || + (insn.words[2] != 0xc000 && !maximum) || native_predicate != 7) + return fail(index, "native warp sum mode"); + emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); + mov_imm(target, 0); + for (u32 lane = 0; lane < 32; ++lane) { + emit(0xef10000000070000ULL | 251 | (u64{source} << 8) | (u64{lane} << 20) | + (u64{1} << 28) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x3828000000070000ULL | 252 | (u64{250} << 8) | (u64{lane} << 20)); + emit(0x0400000000070000ULL | 252 | (u64{252} << 8) | (u64{1} << 20)); + emit(0x5c10000000070000ULL | 252 | (u64{255} << 8) | (u64{252} << 20) | (u64{1} << 48)); + emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48)); + if (maximum) { + emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | (u64{15} << 39)); + } else { + emit(0x5c10000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20)); + } + } + break; + } + case 0xf8c: { // ATOMS.POPC.INC.32 with no returned value: one increment per active lane. + const u32 stride = insn.Bits(78, 2); + if (dest != 255 || insn.words[1] || (insn.words[2] & ~0xc000U) != 0x0d00003f) + return fail(index, "native shared increment mode"); + u32 address = insn.Bits(24, 8); + if (stride && address != 255) { + emit(0x3848000000000000ULL | predicate | 251 | (u64{address} << 8) | + (u64{stride + 1} << 20)); + address = 251; + } + mov_imm(250, 1); + emit(0xec00000000000000ULL | predicate | 255 | (u64{address} << 8) | (u64{250} << 20)); + break; + } + case 0x38c: { // ATOMS.ADD.U32, naturally aligned shared byte offset. + const u32 offset = insn.Bits(40, 24); + if (insn.words[2] || (offset & 3)) return fail(index, "native shared atomic mode"); + emit(0xec00000000000000ULL | predicate | dest | + (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) | + (u64{offset >> 2} << 30)); + break; + } + case 0x3a8: { // ATOMG.E.ADD.STRONG.GPU U32 at a 64-bit register address plus an offset. + const u32 address = insn.Bits(24, 8); + const s32 offset = static_cast<s32>(insn.Bits(40, 24) << 8) >> 8; + if (insn.words[2] != 0x001ee100 || (address & 1) || address >= 246 || + offset < -0x80000 || offset > 0x7ffff) + return fail(index, "native global atomic register mode"); + emit(0xed00000000000000ULL | predicate | dest | (u64{address} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{static_cast<u32>(offset) & 0xfffff} << 28) | + (u64{1} << 48)); + break; + } + case 0x9a8: { // ATOMG.E.ADD.STRONG.GPU U32 through a uniform pointer plus an offset. + const u32 address = uniform_register(insn.Bits(64, 6)); + const s32 offset = static_cast<s32>(insn.Bits(40, 24) << 8) >> 8; + if (insn.Bits(24, 8) != 255 || offset < -0x80000 || offset > 0x7ffff || + (insn.words[2] & ~0x3fU) != 0x081ee140 || address == ~0U || (address & 1)) + return fail(index, "native global atomic mode"); + emit(0xed00000000000000ULL | predicate | dest | (u64{address} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{static_cast<u32>(offset) & 0xfffff} << 28) | + (u64{1} << 48)); + break; + } + case 0xb97: { // SULD.P.2D/2D_ARRAY.IGN, direct descriptor, channel mask at 72-75. + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const u32 coordinate = insn.Bits(24, 8); + const u32 channels = insn.Bits(72, 4); + const u32 dimension = insn.Bits(61, 3); // Maxwell's codes: 3 = 2D, 4 = 2D_ARRAY + if ((insn.words[2] & ~0xf00U) != 0x001e0000 || channels == 0 || (dimension != 3 && dimension != 4) || + binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || dest == 255 || + coordinate > (dimension == 4 ? 252U : 253U)) + return fail(index, "native formatted surface load mode"); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native surface descriptor buffer conflict"); + result.required_texture_buffer = binding; + emit(base(0xeb00000000000000ULL) | (u64{coordinate} << 8) | + (u64{channels} << 20) | (u64{dimension} << 33) | (u64{offset} << 36) | (u64{1} << 51)); + break; + } + case 0xb9b: { // SUST.P.2D/3D.IGN, direct descriptor and all four channels. + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const u32 data = insn.Bits(32, 8), coordinate = insn.Bits(24, 8); + const u32 dimension = insn.Bits(61, 3); + if (insn.words[2] != 0x00100f00 || dimension < 3 || dimension > 5 || + binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || data > 252 || coordinate > (dimension >= 4 ? 252U : 253U)) + return fail(index, "native formatted surface store mode"); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native surface descriptor buffer conflict"); + result.required_texture_buffer = binding; + emit(0xeb20000000000000ULL | predicate | data | (u64{coordinate} << 8) | + (u64{15} << 20) | (u64{dimension} << 33) | (u64{offset} << 36) | (u64{1} << 51)); + break; + } + case 0x301: // BREV register. + if (insn.words[2] || insn.words[1] > 255) return fail(index, "native bit reverse mode"); + emit(base(0x3828000000000000ULL) | (u64{insn.words[1]} << 8) | + (u64{1} << 39) | (u64{1} << 40)); + break; + case 0x219: case 0xc19: { // U32 register (or uniform, 0xc19) shifts with a zero word outside the selected result. + const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8); + const u32 shift = opcode == 0xc19 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + const bool right = insn.Bits(76, 1) != 0, high_result = insn.Bits(80, 1) != 0; + // SHF.R.U32 of the low word with a zero high word: a plain logical right shift. + if (right && !high_result && high == 255 && insn.Bits(73, 2) == 3 && shift != ~0U && + insn.words[1] <= 255 && + (insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) == 0x00000600U) { + emit(base(0x5c28000000000000ULL) | (u64{low} << 8) | (u64{shift} << 20)); + break; + } + if (insn.words[1] > 255 || insn.Bits(73, 2) != 3 || shift == ~0U || + (insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) != 0x00000600U || + (right ? (!high_result || low != 255) : (high_result || high != 255))) + return fail(index, "native register shift mode"); + emit(base(right ? 0x5cf8000000000000ULL : 0x5bf8000000000000ULL) | + (u64{right ? high : 255U} << 8) | (u64{shift} << 20) | + (u64{right ? 255U : low} << 39) | (u64{insn.Bits(75, 1)} << 50)); + break; + } + case 0x300: case 0xd00: { // FLO, including a uniform source. + if ((insn.words[2] & ~(0x600U | (opcode == 0xd00 ? 0x08000000U : 0U))) != 0x000e0000U) + return fail(index, "native leading bit mode"); + const u32 source = operand(1, false); + emit(base(0x5c30000000000000ULL) | (u64{source} << 20) | + (u64{insn.Bits(63, 1)} << 40) | (u64{insn.Bits(74, 1)} << 41) | + (u64{insn.Bits(73, 1)} << 48)); + break; + } + case 0x309: case 0xd09: { // POPC, including a uniform source. + if (insn.words[2] != (opcode == 0xd09 ? 0x08000000U : 0U)) + return fail(index, "native population count mode"); + const u32 source = operand(1, false); + emit(base(0x5c08000000000000ULL) | (u64{source} << 20) | (u64{insn.Bits(63, 1)} << 40)); + break; + } + case 0x918: + if (insn.words[1] || insn.words[2]) return fail(index, "native NOP mode"); + emit(0x50b0000000000f00ULL | predicate); + break; + case 0x992: { // MEMBAR.SC: preserve workgroup/device visibility. + const u32 scope = insn.Bits(76, 3); + if (insn.words[1] || insn.words[2] != (scope << 12) || + (scope != 0 && scope != 2 && scope != 3 && scope != 5)) + return fail(index, "native memory barrier scope"); + const u32 legacy_scope = scope == 0 ? 0 : scope == 2 ? 1 : scope == 3 ? 2 : 3; + emit(0xef98000000000000ULL | predicate | (u64{legacy_scope} << 8)); + break; + } + case 0x9ab: // ERRBAR: wait for prior memory operations on the host backend. + if (insn.words[1] || insn.words[2]) return fail(index, "native error barrier mode"); + emit(0xef98000000000000ULL | predicate | (u64{1} << 8)); + break; + case 0xb1d: // BAR.SYNC 0, full workgroup. + if (insn.words[1] || insn.words[2] || native_predicate != 7) + return fail(index, "native workgroup barrier mode"); + emit(0xf0a8000000000000ULL | predicate | (u64{1} << 43) | + (u64{1} << 44) | (u64{1} << 39) | (u64{7} << 39)); + break; + case 0x388: case 0x984: { // Shared memory load/store with scaled register offsets. + const u32 width = insn.Bits(73, 3), stride = insn.Bits(78, 2); + // LDS [Ra.Xn + URb + offset]: bit 91 adds a uniform register (bits 32-39). + const bool uniform_base = opcode == 0x984 && insn.Bits(91, 1); + const u32 base_register = uniform_base ? uniform_register(insn.Bits(32, 8)) : 255U; + if (width > 6 || (insn.words[2] & ~(uniform_base ? 0x08000000U : 0U)) != ((width << 9) | (stride << 14)) || + (opcode == 0x984 && !uniform_base && insn.Bits(32, 8)) || base_register == ~0U) + return fail(index, "native shared memory addressing mode"); + u32 address = insn.Bits(24, 8); + if (stride && address != 255) { + emit(0x3848000000000000ULL | predicate | 250 | + (u64{address} << 8) | (u64{stride + 1} << 20)); + address = 250; + } + if (base_register != 255) { + emit(0x5c10000000000000ULL | predicate | 250 | (u64{address} << 8) | (u64{base_register} << 20)); + address = 250; + } + const u32 data = opcode == 0x388 ? insn.Bits(32, 8) : static_cast<u32>(dest); + emit((opcode == 0x388 ? 0xef58000000000000ULL : 0xef48000000000000ULL) | + predicate | data | (u64{address} << 8) | (u64{insn.Bits(40, 24)} << 20) | + (u64{width} << 48)); + break; + } + case 0x23a: { // MOVM.16.MT88: transpose an 8x8 16-bit matrix held one row pair per lane. + // Lane t holds row t/4, columns 2(t%4)..+1; the result is the transposed matrix in the + // same layout, so each half comes from lane 8(t%4) + t/8 (+4) at half (t/4)%2. + const u32 source = insn.Bits(24, 8); + if (insn.words[1] || insn.words[2] || source >= 246 || dest >= 246) + return fail(index, "native matrix move mode"); + emit(0xf0c8000000000000ULL | predicate | 246); // laneid + emit(0x0400000000000000ULL | predicate | 249 | (u64{246} << 8) | (u64{3} << 20)); + emit(0x3848000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{3} << 20)); // 8(t%4) + emit(0x3828000000000000ULL | predicate | 247 | (u64{246} << 8) | (u64{3} << 20)); // t/8 + emit(0x5c10000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{247} << 20)); + emit(0xef10000000000000ULL | predicate | 247 | (u64{source} << 8) | (u64{249} << 20) | + (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x1c00000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{4} << 20)); + emit(0xef10000000000000ULL | predicate | 248 | (u64{source} << 8) | (u64{249} << 20) | + (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x3828000000000000ULL | predicate | 249 | (u64{246} << 8) | (u64{2} << 20)); + emit(0x0400000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{1} << 20)); + emit(0x3848000000000000ULL | predicate | 249 | (u64{249} << 8) | (u64{4} << 20)); // 0 or 16 + emit(0x5c28000000000000ULL | predicate | 247 | (u64{247} << 8) | (u64{249} << 20)); + emit(0x0400000000000000ULL | predicate | 247 | (u64{247} << 8) | (u64{0xffff} << 20)); + emit(0x5c28000000000000ULL | predicate | 248 | (u64{248} << 8) | (u64{249} << 20)); + emit(0x5c18000000000000ULL | predicate | dest | (u64{248} << 8) | (u64{247} << 20) | (u64{16} << 39)); + break; + } + case 0x23c: { // HMMA.16816.F16: D(16x8) = A(16x16) * B(16x8) + C, f16 fragments. + // Lane t (group g = t/4, i = t%4): A regs hold rows g / g+8 at k 2i..2i+1 (a0, a1) and + // 8+2i.. (a2, a3); B regs hold k 2i.. (b0) / 8+2i.. (b1) of column g; D holds rows + // g / g+8 at columns 2i, 2i+1. Rebuilt from shuffles with HFMA2, accumulating in R252/R253. + // The expansion is correct but large: title 4's 372-HMMA network takes the host driver + // minutes to compile, so it stays opt-in (the QMD still completes when rejected). + // EDEN_NX2_TENSOR=stub keeps the rest of the program but makes every HMMA D = C. + static const char* const mode = std::getenv("EDEN_NX2_TENSOR"); + if (!mode) return fail(index, "native matrix multiply (set EDEN_NX2_TENSOR to emulate)"); + const u32 a = insn.Bits(24, 8), b = insn.Bits(32, 8), c = insn.Bits(64, 8); + if ((insn.words[1] & ~0xffU) || (insn.words[2] & ~0xffU) != 0x800 || dest + 2 > 246 || + a + 4 > 246 || b + 2 > 246 || (c != 255 && c + 2 > 246)) + return fail(index, "native matrix multiply mode"); + if (std::string_view{mode} == "stub") { + for (u32 row = 0; row < 2; ++row) { + if (c == 255) mov_imm(static_cast<u32>(dest) + row, 0); + else if (c != dest) emit(0x5c98000000000000ULL | predicate | (dest + row) | (u64{15} << 39) | (u64{c + row} << 20)); + } + break; + } + const auto shuffle = [&](u32 target, u32 source_reg) { + emit(0xef10000000000000ULL | predicate | target | (u64{source_reg} << 8) | (u64{246} << 20) | + (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + }; + const auto lane_a = [&](u32 i) { // 4g + i + emit(0xf0c8000000000000ULL | predicate | 246); + emit(0x0400000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{0x1c} << 20)); + if (i) emit(0x1c00000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{i} << 20)); + }; + const auto lane_b = [&](u32 add) { // 8(t%4) + add: column 2(t%4) (+1 when add >= 4) + emit(0xf0c8000000000000ULL | predicate | 246); + emit(0x0400000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{3} << 20)); + emit(0x3848000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{3} << 20)); + if (add) emit(0x1c00000000000000ULL | predicate | 246 | (u64{246} << 8) | (u64{add} << 20)); + }; + const auto hfma2 = [&](u32 accumulator, u32 swizzle_a, u32 pair) { + emit(0x5d00000000000000ULL | predicate | accumulator | (u64{251} << 8) | (u64{pair} << 20) | + (u64{accumulator} << 39) | (u64{swizzle_a} << 47)); + }; + for (u32 row = 0; row < 2; ++row) { + if (c == 255) mov_imm(252 + row, 0); + else emit(0x5c98000000000000ULL | predicate | (252 + row) | (u64{15} << 39) | (u64{c + row} << 20)); + } + for (u32 i = 0; i < 4; ++i) { + for (u32 k_block = 0; k_block < 2; ++k_block) { + lane_b(i); + shuffle(247, b + k_block); + lane_b(4 + i); + shuffle(248, b + k_block); + // P0 = (B[2i][2t], B[2i][2t+1]), P1 = (B[2i+1][2t], B[2i+1][2t+1]) + emit(0x0400000000000000ULL | predicate | 249 | (u64{247} << 8) | (u64{0xffff} << 20)); + emit(0x5c18000000000000ULL | predicate | 249 | (u64{248} << 8) | (u64{249} << 20) | (u64{16} << 39)); + emit(0x3828000000000000ULL | predicate | 250 | (u64{247} << 8) | (u64{16} << 20)); + emit(0x0400000000000000ULL | predicate | 248 | (u64{248} << 8) | (u64{0xffff0000} << 20)); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{248} << 20)); + for (u32 row = 0; row < 2; ++row) { + lane_a(i); + shuffle(251, a + k_block * 2 + row); + hfma2(252 + row, 2, 249); // A.H0 broadcast * P0 + hfma2(252 + row, 3, 250); // A.H1 broadcast * P1 + } + } + } + for (u32 row = 0; row < 2; ++row) + emit(0x5c98000000000000ULL | predicate | (dest + row) | (u64{15} << 39) | (u64{252 + row} << 20)); + break; + } + case 0x83b: { // LDSM.16.M88.{1,2,4} (ldmatrix): 8x8 16-bit matrices from shared memory. + // Lanes 8m..8m+7 give the row addresses of matrix m; lane t receives the 32 bits at + // row t/4, byte (t%4)*4 of each matrix in Rd+m. Emulated with SHFL.IDX + LDS.32. + const u32 count = 1U << insn.Bits(72, 2); + const bool transpose = insn.Bits(78, 1) != 0; + if (insn.Bits(72, 2) == 3 || (insn.words[2] & ~0x4300U) || insn.Bits(32, 8) || + dest + count > 246) + return fail(index, "native matrix load mode"); + u32 address = insn.Bits(24, 8); + const u64 offset = insn.Bits(40, 24); + if (address != 255 && address >= dest && address < dest + count) { + emit(0x5c10000000000000ULL | predicate | 253 | (u64{address} << 8) | (u64{255} << 20)); + address = 253; + } + if (transpose) { + // .MT88: lane t receives stored rows 2(t%4) and 2(t%4)+1 at column t/4 (16 bits each). + emit(0xf0c8000000000000ULL | predicate | 246); // laneid + emit(0x0400000000000000ULL | predicate | 247 | (u64{246} << 8) | (u64{3} << 20)); + emit(0x3848000000000000ULL | predicate | 247 | (u64{247} << 8) | (u64{1} << 20)); // row 2(t%4) + emit(0x3828000000000000ULL | predicate | 248 | (u64{246} << 8) | (u64{2} << 20)); + emit(0x3848000000000000ULL | predicate | 248 | (u64{248} << 8) | (u64{1} << 20)); // byte 2(t/4) + for (u32 matrix = 0; matrix < count; ++matrix) { + for (u32 half = 0; half < 2; ++half) { + emit(0x1c00000000000000ULL | predicate | 249 | (u64{247} << 8) | + (u64{matrix * 8 + half} << 20)); + emit(0xef10000000000000ULL | predicate | 250 | (u64{address} << 8) | (u64{249} << 20) | + (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{248} << 20)); + emit(0xef48000000000000ULL | predicate | (251 + half) | (u64{250} << 8) | + (offset << 20) | (u64{2} << 48)); // LDS.U16 + } + emit(0x5c18000000000000ULL | predicate | (dest + matrix) | (u64{252} << 8) | + (u64{251} << 20) | (u64{16} << 39)); + } + break; + } + emit(0xf0c8000000000000ULL | predicate | 250); // S2R laneid + emit(0x3828000000000000ULL | predicate | 251 | (u64{250} << 8) | (u64{2} << 20)); // row + emit(0x0400000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{3} << 20)); + emit(0x3848000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{2} << 20)); // column bytes + for (u32 matrix = 0; matrix < count; ++matrix) { + u32 lane = 251; + if (matrix) { + emit(0x1c00000000000000ULL | predicate | 252 | (u64{251} << 8) | (u64{matrix * 8} << 20)); + lane = 252; + } + emit(0xef10000000000000ULL | predicate | 253 | (u64{address} << 8) | (u64{lane} << 20) | + (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{250} << 20)); + emit(0xef48000000000000ULL | predicate | (dest + matrix) | (u64{253} << 8) | + (offset << 20) | (u64{4} << 48)); + } + break; + } + case 0x919: case 0x9c3: { // S2R / S2UR thread and workgroup identities. + const u32 special = insn.Bits(72, 8); + if (insn.words[1] || insn.words[2] != (special << 8) || + (special != 0 && special != 33 && special != 34 && special != 35 && + special != 37 && special != 38 && special != 39 && + special != 56 && special != 57 && special != 58 && special != 59 && special != 60)) + return fail(index, "native special register"); + const u32 target = opcode == 0x9c3 ? uniform_register(static_cast<u32>(dest)) : static_cast<u32>(dest); + if (target == ~0U) return fail(index, "native uniform special destination"); + emit(0xf0c8000000000000ULL | predicate | target | (u64{special} << 20)); + break; + } + case 0x28c: case 0x88c: { // UISETP, signed (bit 73) or unsigned, with UPT accumulation. + const u32 output = insn.Bits(81, 3); + const u32 a = uniform_register(insn.Bits(24, 8)), b = opcode == 0x88c ? 252U : uniform_register(insn.Bits(32, 8)); + if (output > 6 || insn.Bits(84, 3) != 7 || insn.Bits(87, 4) != 7 || + (insn.Bits(72, 4) & ~2U) || insn.Bits(12, 4) != 7 || a == ~0U || b == ~0U) + return fail(index, "native uniform comparison mode"); + const u32 temporary = 6 - output, mask = 1U << temporary; + if (opcode == 0x88c) mov_imm(252, insn.words[1]); + emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + emit(0x5b60000000070000ULL | 7 | (u64{temporary} << 3) | + (u64{a} << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{insn.Bits(73, 1)} << 48) | + (u64{insn.Bits(76, 3)} << 49)); + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20)); + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + break; + } + case 0x287: case 0x887: { // USEL, UP0/UP1; preserve ordinary predicates. + const u32 condition = insn.Bits(87, 3); + const u32 target = uniform_register(static_cast<u32>(dest)); + const u32 a = uniform_register(insn.Bits(24, 8)), b = opcode == 0x887 ? 252U : uniform_register(insn.Bits(32, 8)); + if (condition > 6 || target == ~0U || a == ~0U || b == ~0U || + insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x07800000U) != 0x08000000U) + return fail(index, "native uniform select mode"); + if (opcode == 0x887 && index > 0 && condition == 0 && insn.words[1] <= 0x7ffff && + code[index - 1].Bits(0, 12) == 0x88c && + code[index - 1].Bits(24, 8) == insn.Bits(24, 8) && + code[index - 1].words[1] == insn.words[1] && code[index - 1].words[2] == 0x0bf01070 && + insn.words[2] == 0x08000000) { + // Preserve the unsigned clamp idiom for indirect-table analysis. + emit(0x3820000000070000ULL | target | (u64{a} << 8) | + (u64{insn.words[1]} << 20) | (u64{7} << 39)); + break; + } + if (opcode == 0x887) mov_imm(252, insn.words[1]); + const u32 temporary = 6 - condition, mask = 1U << temporary; + emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{mask} << 20)); + emit(0x5ca0000000070000ULL | target | (u64{a} << 8) | (u64{b} << 20) | + (u64{temporary} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + break; + } + case 0x381: case 0x981: { // LDG.E of 8 to 128 bits, register or uniform 64-bit address. + // Sizes as Maxwell's: U8, S8, U16, S16, 32, 64, 128. + const u32 size = insn.Bits(73, 3); + u32 address = insn.Bits(24, 8); + if (size > 6 || (opcode == 0x381 && insn.Bits(32, 8)) || + (size == 5 && (dest & 1)) || (size == 6 && (dest & 3))) + return fail(index, "native global load mode"); + if (opcode == 0x381) { + // Bit 77 (strong/scoped ordering) needs nothing extra from a Maxwell LDG. + if ((insn.words[2] & ~0x2e00U) != 0x001e8100 || (address & 1) || address >= 246) + return fail(index, "native global load mode"); + } else { + // The uniform base sits where a register source would (bits 32-37). + const u32 uniform_address = + insn.Bits(38, 2) ? ~0U : uniform_register(insn.Bits(32, 6)); + if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U | 0x2000U)) != 0x081e8100 || + uniform_address == ~0U || (uniform_address & 1)) + return fail(index, "native global load uniform addressing mode"); + if (address == 255) { + address = uniform_address; + } else { + if (insn.Bits(90, 1) && ((address & 1) || address >= 246)) + return fail(index, "native global load register address pair"); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{uniform_address} << 8) | + (u64{address} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 251 | (u64{uniform_address + 1} << 8) | + (u64{insn.Bits(90, 1) ? address + 1 : 255} << 20) | (u64{1} << 43)); + address = 250; + } + } + emit(base(0xeed0000000000000ULL) | (u64{address} << 8) | + (u64{insn.Bits(40, 24)} << 20) | (u64{1} << 45) | (u64{size} << 48)); + break; + } + case 0x386: case 0x986: { // Global store of 32/64/128 bits, 64-bit address. + u32 data = insn.Bits(32, 8); + const u32 size = insn.Bits(73, 3); + if (size > 6 || + (data != 255 && ((size == 5 && (data & 1)) || (size == 6 && (data & 3)) || + data + (size < 4 ? 1U : 1U << (size - 4)) > 246))) + return fail(index, "native global store size"); + if (data == 255 && size > 4) { + // Zeros wider than one register: store from zeroed scratch registers (aligned to the + // access: R248-R251 for 128 bits, R250-R251 for 64). + data = size == 6 ? 248 : 250; + for (u32 i = 0; i < (1U << (size - 4)); ++i) mov_imm(data + i, 0); + } + u32 address = insn.Bits(24, 8); + bool extended = insn.Bits(72, 1) != 0; + if (opcode == 0x986) { + const u32 uniform_address = uniform_register(insn.Bits(64, 6)); + if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U)) != 0x0810a100 || + uniform_address == ~0U || (uniform_address & 1) || !extended) + return fail(index, "native global store uniform addressing mode"); + if (address == 255) address = uniform_address; + else { + if (insn.Bits(90, 1) && ((address & 1) || address >= 254)) + return fail(index, "native global store register address pair"); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{uniform_address} << 8) | + (u64{address} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 251 | (u64{uniform_address + 1} << 8) | + (u64{insn.Bits(90, 1) ? address + 1 : 255} << 20) | (u64{1} << 43)); + address = 250; + } + } else if ((insn.words[2] & ~0xe00U) != 0x0010a100) { + return fail(index, "native global store register addressing mode"); + } + emit(0xeed8000000000000ULL | predicate | data | + (u64{address} << 8) | (u64{insn.Bits(40, 24)} << 20) | + (u64{extended} << 45) | (u64{size} << 48)); + break; + } + case 0x95b: // Fragment discard; execution predicate is preserved. + if (insn.words[1] || insn.words[2] != 0x03800000) + return fail(index, "native fragment discard mode"); + emit(0xe33000000000000fULL | predicate); + break; + case 0x305: case 0xb05: case 0xd05: { // F2I NTZ, register/constant-buffer/uniform source + const u32 mode = insn.words[2] & ~(opcode == 0xd05 ? 0x08000000U : 0U); + const bool half = opcode == 0x305 && mode == 0x0010f000; + const bool small = mode == 0x00212800; + const bool floor = (mode & ~0x10100U) == 0x00207000; + const bool nearest = (mode & ~0x10100U) == 0x00203000; + if ((!half && !small && !floor && !nearest && (mode & ~0x100U) != 0x0021f000) || + (opcode != 0xb05 && (insn.words[1] & 0xffffff00U)) || + (opcode == 0xd05 && uniform_register(insn.Bits(32, 8)) == ~0U)) { + return fail(index, "native float-to-integer conversion mode"); + } + const bool is_signed = insn.Bits(72, 1); + u32 source = opcode == 0xb05 ? materialize(2, 0, 252, false, false) + : opcode == 0xd05 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + if (half) { + extract_half({source, 2, false, false}, 0, 252); + source = 252; + } + if (!is_signed) { + // Clear NaNs without consuming a guest predicate. Clamp the + // conversion below 2^32, retaining an overflow mask for saturation. + emit(0x5800000000000000ULL | predicate | 250 | (u64{source} << 8) | + (u64{source} << 20) | (u64{7} << 39) | (u64{7} << 48)); + emit(0x5c40000000000000ULL | predicate | 250 | (u64{source} << 8) | + (u64{250} << 20) | (u64{7} << 48)); + if (!small) { + mov_imm(252, 0x4f800000); + emit(0x5800000000000000ULL | predicate | 251 | (u64{250} << 8) | + (u64{252} << 20) | (u64{7} << 39) | (u64{6} << 48)); + mov_imm(252, 0x4f7fffff); + emit(0x5c60000000000000ULL | predicate | 250 | (u64{250} << 8) | + (u64{252} << 20) | (u64{7} << 39)); + } + source = 250; + } + emit(base(0x5cb0000000000000ULL) | (u64{small ? 1U : 2U} << 8) | (u64{2} << 10) | + (u64{is_signed} << 12) | (u64{source} << 20) | + (u64{small || nearest ? 0U : floor ? 1U : 3U} << 39) | (u64{insn.Bits(80, 1)} << 44)); + if (!is_signed && !small) { + emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{251} << 20) | + (u64{1} << 41) | (u64{7} << 48)); + } + break; + } + case 0xb6f: { // TXQ texture dimensions, two packed scalar outputs. + if (insn.Bits(59, 5) || insn.Bits(64, 8) != 255 || insn.Bits(72, 4) == 0 || insn.Bits(72, 4) > 3 || + insn.Bits(76, 14) || insn.Bits(91, 1) || insn.Bits(54, 5) != 2 || + insn.Bits(40, 14) > 0x1fff || (insn.Bits(72, 4) == 3 && (dest & 1)) || dest >= 254) + return fail(index, "native texture dimension query mode"); + result.required_texture_buffer = 2; + emit(0xdf48000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | + (u64{1} << 22) | (u64{insn.Bits(72, 4)} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49)); + break; + } + case 0xb66: { // TLD.SCR.LZ, direct texel fetch + const u32 mask = insn.Bits(72, 4); + const u32 dimension = insn.Bits(61, 3); + const bool array = dimension == 5; + const bool volume = dimension == 2; + u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + const u32 second_dest = insn.Bits(64, 8); + const bool split = second_dest != 255; + constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12}; + constexpr std::array<u32, 5> rgba_masks{7, 11, 13, 14, 15}; + u32 swizzle = ~0U; + if (split) { + for (u32 i = 0; i < rgba_masks.size(); ++i) if (rgba_masks[i] == mask) swizzle = i; + } else { + for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i; + } + if (!insn.Bits(60, 1) || (dimension != 0 && dimension != 1 && !array && !volume) || + swizzle == ~0U || insn.Bits(76, 5) || + insn.Bits(81, 3) != 7 || insn.Bits(87, 3) != 1 || + insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 || + dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) || + (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) || + (array && ((source_a & 1) || source_a >= 249)) || + (volume && ((source_a & 1) || source_a >= 254))) { + return fail(index, "native texel fetch mode or descriptor binding"); + } + const u32 binding = insn.Bits(54, 5); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native texture descriptor buffer conflict"); + result.required_texture_buffer = binding; + if (array) { + emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a + 1} << 20)); + emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{source_b} << 20)); + source_b = 250; + } + emit(0xda00000000000000ULL | predicate | dest | (u64{source_a} << 8) | + (u64{source_b} << 20) | (u64{insn.Bits(64, 8)} << 28) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | + (u64{swizzle} << 50) | (u64{array ? 8U : volume ? 7U : dimension == 0 ? 0U : 2U} << 53)); + break; + } + case 0x91a: // DEPBAR.LE SB5, 1: scoreboard wait, represented by IR data dependencies. + if (insn.words[0] != 0x791a || (insn.words[1] & ~0xfc0U) != 0xd000 || insn.words[2]) { + return fail(index, "native dependency barrier mode"); + } + break; + case 0xb69: { // TMML (LOD query), 2D. Results are 8.8 fixed point, as on Maxwell. + const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8); + const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + const u32 binding = insn.Bits(54, 5); + if (insn.Bits(61, 3) != 1 || insn.Bits(76, 4) || insn.Bits(80, 10) || insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || binding >= 18 || mask == 0 || source_a >= 246 || + (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)) + return fail(index, "native texture LOD query mode"); + result.required_texture_buffer = binding; + // Coordinates: (Ra, Ra+1) when Rb is RZ, else (Ra, Rb). + emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a} << 20)); + emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | + (u64{source_b == 255 ? source_a + 1 : source_b} << 20)); + emit(0xdf58000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49)); + const u32 components = static_cast<u32>(std::popcount(mask)); + const bool split = second_dest != 255; + for (u32 i = 0; i < components; ++i) { + u32 target = 255; + if (!split || i < 2) target = dest == 255 ? 255U : static_cast<u32>(dest) + i; + else target = second_dest + (i - 2); + if (target != 255) + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{250 + i} << 20)); + } + break; + } + case 0xb63: { // TLD4 (gather), 2D without offsets or depth compare; component at bits 87-88. + const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8); + const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + const u32 binding = insn.Bits(54, 5); + if (insn.Bits(61, 3) != 1 || insn.Bits(76, 4) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || binding >= 18 || insn.Bits(89, 1) || mask == 0 || + (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)) + return fail(index, "native texture gather mode"); + result.required_texture_buffer = binding; + // Two coordinates: x in Ra, y in Rb (the coordinate list splits evenly across them). + emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a} << 20)); + emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | + (u64{source_b == 255 ? 255U : source_b} << 20)); + emit(0xc800000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | (u64{insn.Bits(87, 2)} << 56)); + const u32 components = static_cast<u32>(std::popcount(mask)); + const bool split = second_dest != 255; + for (u32 i = 0; i < components; ++i) { + u32 target = 255; + if (!split || i < 2) target = dest == 255 ? 255U : static_cast<u32>(dest) + i; + else target = second_dest + (i - 2); + if (target != 255) + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{250 + i} << 20)); + } + break; + } + case 0xb60: { // TEX, scalar 2D implicit-LOD sample + const u32 mask = insn.Bits(72, 4); + const u32 second_dest = insn.Bits(64, 8); + constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12}; + constexpr std::array<u32, 5> rgba_masks{7, 11, 13, 14, 15}; + u32 swizzle = ~0U; + const bool split = second_dest != 255; + const bool depth_compare = insn.Bits(78, 1); + const u32 dimension = insn.Bits(61, 3), lod = insn.Bits(87, 3); + const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + if (dimension == 5 && depth_compare && lod == 0 && !split && mask == 1) { + if (!insn.Bits(60, 1) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 || + insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || + (source_a & 1) || (source_b & 1) || source_a >= 248 || source_b >= 248 || dest >= 248) + return fail(index, "native array depth comparison sample mode"); + result.required_texture_buffer = 2; + for (u32 i = 0; i < 3; ++i) { + const u32 source = i < 2 ? source_a + i : source_b; + emit(0x5c98000000000000ULL | predicate | (250 + i) | + (u64{15} << 39) | (u64{source} << 20)); + } + emit(0xc000000000000000ULL | predicate | dest | (u64{250} << 8) | + (u64{source_b + 1} << 20) | (u64{3} << 28) | (u64{1} << 31) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | + (u64{1} << 50) | (u64{7} << 51)); + break; + } + // General form: gather the native operand list (array index first, coordinates, + // then LOD/bias and the depth reference) into Maxwell TEX's coordinate and meta + // registers, sample into scratch and copy the packed components out. Native packs + // up to four values as halves of Ra and Rb (1+1, 2+1, 2+2), five or more as 4+n. + const auto general_tex = [&]() -> bool { + constexpr std::array<u32, 8> coordinate_count{1, 2, 3, 3, 2, 3, 0, 4}; + constexpr std::array<u32, 8> maxwell_type{0, 2, 4, 6, 1, 3, 0, 7}; + const bool offsets = insn.Bits(76, 1) != 0; + if (dimension == 6 || insn.Bits(77, 1) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 || (lod > 3 && lod != 6 && lod != 7) || + (depth_compare && mask != 1) || mask == 0) + return false; + const u32 coordinates = coordinate_count[dimension]; + const bool has_lod = lod == 2 || lod == 3 || lod == 6 || lod == 7; + const u32 count = coordinates + (has_lod ? 1 : 0) + (offsets ? 1 : 0) + (depth_compare ? 1 : 0); + const u32 in_a = count <= 4 ? (count + 1) / 2 : 4; + const auto value = [&](u32 i) -> u32 { + if (i < in_a) return source_a == 255 ? 255U : source_a + i; + return source_b == 255 ? 255U : source_b + (i - in_a); + }; + if ((source_a != 255 && source_a + in_a > 246) || + (count > in_a && source_b != 255 && source_b + (count - in_a) > 246)) + return false; + const u32 binding = insn.Bits(54, 5); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return false; + result.required_texture_buffer = binding; + const auto copy = [&](u32 target, u32 source) { + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20)); + }; + // Coordinates to R250.., then LOD/bias, offsets and reference to R246... + for (u32 i = 0; i < coordinates; ++i) copy(250 + i, value(i)); + u32 meta = 246; + u32 next = coordinates; + if (has_lod) copy(meta++, value(next++)); + if (offsets) copy(meta++, value(next++)); + if (depth_compare) copy(meta++, value(next++)); + emit(0xc000000000000000ULL | predicate | 250 | (u64{250} << 8) | + (u64{meta == 246 ? 255U : 246U} << 20) | (u64{maxwell_type[dimension]} << 28) | + (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) | (u64{7} << 51) | + (u64{offsets} << 54) | (u64{lod} << 55)); + const u32 components = static_cast<u32>(std::popcount(mask)); + if (insn.Bits(79, 1)) { + // FP16 results: (c0, c1) packed into the first destination, (c2, c3) into + // the second (F2F writes the half in the low 16 bits; ISCADD merges). + const auto pack = [&](u32 target, u32 low, u32 high) { + if (target == 255) return; + emit(0x5ca8000000000000ULL | predicate | 246 | (u64{1} << 8) | (u64{2} << 10) | (u64{low} << 20)); + emit(0x5ca8000000000000ULL | predicate | 247 | (u64{1} << 8) | (u64{2} << 10) | (u64{high} << 20)); + emit(0x5c18000000000000ULL | predicate | target | (u64{246} << 8) | (u64{247} << 20) | (u64{16} << 39)); + }; + pack(static_cast<u32>(dest), 250, components > 1 ? 251U : 255U); + if (components > 2) { + pack(split ? second_dest : (dest == 255 ? 255U : static_cast<u32>(dest) + 1), + 252, components > 3 ? 253U : 255U); + } + return true; + } + for (u32 i = 0; i < components; ++i) { + u32 target = 255; + if (!split) target = dest == 255 ? 255U : static_cast<u32>(dest) + i; + else if (i < 2) target = dest == 255 ? 255U : static_cast<u32>(dest) + i; + else target = second_dest + (i - 2); + if (target != 255) copy(target, 250 + i); + } + return true; + }; + if (split) { + for (u32 i = 0; i < rgba_masks.size(); ++i) if (rgba_masks[i] == mask) swizzle = i; + } else { + for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i; + } + // SM70 scalar coordinates and split RG/BA destinations match TEXS. + // Descriptor offsets are words in both encodings, but only native + // explicitly encodes the constant-buffer binding. + if (!insn.Bits(60, 1) || (dimension != 1 && dimension != 5) || + swizzle == ~0U || insn.Bits(76, 2) || + insn.Bits(81, 3) != 7 || (lod > 1 && (lod != 3 || dimension != 1 || depth_compare)) || + (lod == 3 && ((source_a & 1) || source_a >= 249)) || + insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || + dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) || + (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) || + (depth_compare && (dimension != 1 || split || mask != 1 || (source_a & 1) || source_a >= 249)) || + (dimension == 5 && ((source_a & 1) || source_a >= 249))) { + if (!general_tex()) return fail(index, "native texture mode or descriptor binding"); + break; + } + result.required_texture_buffer = 2; + emit((0xd800000000000000ULL & ~(u64{insn.Bits(79, 1)} << 59)) | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(64, 8)} << 28) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | + (u64{swizzle} << 50) | (u64{dimension == 5 ? (lod ? 8U : 7U) : depth_compare ? (lod ? 6U : 4U) : (lod == 3 ? 3U : lod ? 2U : 1U)} << 53)); + break; + } + case 0x816: { // Captured PRMT half-word concatenation + if (insn.words[1] > 0xffff || (insn.words[2] & ~0xffU)) return fail(index, "native permutation mode"); + if (insn.words[1] != 0x5410 && insn.words[1] != 0x5432 && insn.words[1] != 0x7610 && insn.words[1] != 0x7632) { + // Byte i of the result is byte (s & 7) of c:a, or its sign bit repeated when s & 8. + const u32 a = insn.Bits(24, 8), c = insn.Bits(64, 8); + if (a >= 250 && a != 255) return fail(index, "native permutation source"); + mov_imm(250, 0); + for (u32 i = 0; i < 4; ++i) { + const u32 select = (insn.words[1] >> (i * 4)) & 0xf; + const u32 source = select & 4 ? c : a, position = (select & 3) * 8; + emit(0x3800000000000000ULL | predicate | 251 | (u64{source} << 8) | + (u64{select & 8 ? (position + 7) | (1U << 8) : position | (8U << 8)} << 20) | + (u64{(select >> 3) & 1} << 48)); + emit(0x36f0000000000000ULL | predicate | 250 | (u64{251} << 8) | + (u64{(i * 8) | (8U << 8)} << 20) | (u64{250} << 39)); + } + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{255} << 20)); + break; + } + const u32 a = insn.Bits(24, 8), b = insn.Bits(64, 8); + if (insn.words[1] != 0x5432 && insn.words[1] != 0x7632) { + emit(0x0400000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{0xffff} << 20)); + } else { + emit(0x3828000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{16} << 20)); + } + if (insn.words[1] == 0x7610 || insn.words[1] == 0x7632) { + emit(0x0400000000000000ULL | predicate | 251 | (u64{b} << 8) | (u64{0xffff0000} << 20)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{251} << 20)); + } else { + emit(base(0x5c18000000000000ULL) | (u64{b} << 8) | (u64{250} << 20) | (u64{16} << 39)); + } + break; + } + case 0x307: { // FRND.F32, round to an integral floating-point value + if (insn.Bits(75, 2) != 2 || insn.Bits(84, 2) != 2 || insn.Bits(81, 3) || + insn.Bits(86, 6)) return fail(index, "native floating round format"); + emit(base(0x5ca8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | + (u64{insn.Bits(32, 8)} << 20) | (u64{8 + insn.Bits(78, 2)} << 39) | + (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(63, 1)} << 45) | + (u64{insn.Bits(62, 1)} << 49)); + break; + } + case 0x321: // ALD + case 0x322: { // AST + if (insn.Bits(76, 2) || insn.Bits(79, 1)) return fail(index, "native indexed/patch attribute access"); + const u32 address = insn.Bits(40, 10); + const u32 components = insn.Bits(74, 2) + 1; + if (address % 4 || address + components * 4 > 0x400 || insn.Bits(24, 8) != 255) { + return fail(index, "native attribute address or indexing"); + } + // a[0x5c] has no Maxwell meaning (Ampere uses it for a per-primitive hint such as the + // shading rate). Nothing reads it back; dropping the store keeps full-rate shading. + if (opcode == 0x322 && address == 0x5c && components == 1) break; + u32 value_reg = opcode == 0x321 ? static_cast<u32>(dest) : insn.Bits(32, 8); + if (value_reg == 255) { + if (opcode == 0x321) break; // load into RZ: nothing to do + // AST of RZ writes zeros: store them from zeroed scratch registers. + for (u32 i = 0; i < components; ++i) mov_imm(250 + i, 0); + value_reg = 250; + } + if (value_reg + components > 255) return fail(index, "native attribute register range"); + const u32 vertex = opcode == 0x321 ? insn.Bits(32, 8) : insn.Bits(64, 8); + const u64 translated = (opcode == 0x321 ? 0xefd8000000000000ULL : 0xeff0000000000000ULL) | + predicate | value_reg | (u64{255} << 8) | (u64{address} << 20) | + (u64{vertex} << 39) | (u64{components - 1} << 47); + result.instructions.push_back(translated); + break; + } + case 0x387: case 0x983: { // Captured scalar local spill at an immediate byte offset. + const bool store = opcode == 0x387; + const u32 data = insn.Bits(32, 8), offset = insn.Bits(40, 24); + if (insn.Bits(24, 8) != 255 || (store && (insn.Bits(16, 8) || data > 247)) || + (!store && (data || dest >= 248)) || (offset & 3) || offset > 0x7fffff || + insn.words[2] != (store ? 0x00100800U : 0x00300800U)) + return fail(index, "native local scalar spill mode"); + emit((store ? 0xef50000000000000ULL : 0xef40000000000000ULL) | predicate | + (store ? data : dest) | (u64{255} << 8) | (u64{offset} << 20) | (u64{4} << 48)); + break; + } + case 0x803: { // P2R PR, register merge source, immediate mask, byte select (76-77) + if (insn.words[1] > 0x7f || (insn.words[2] & ~0x3000U)) + return fail(index, "native predicate-to-register mode"); + emit(base(0x38e8000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.words[1]} << 20) | (u64{insn.Bits(76, 2)} << 41)); + break; + } + case 0x804: { // R2P PR, register, immediate mask, byte select (76-77) + if (insn.Bits(16, 8) || insn.words[1] > 0x7f || (insn.words[2] & ~0x3000U)) + return fail(index, "native register-to-predicate mode"); + emit(0x38f0000000000000ULL | predicate | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.words[1]} << 20) | (u64{insn.Bits(76, 2)} << 41)); + break; + } + case 0xc02: // MOV uniform register + case 0x202: // MOV register + case 0x802: // MOV immediate + case 0xa02: { // MOV constant buffer + if (insn.Bits(72, 4) != 15 || (insn.Bits(76, 16) & ~(opcode == 0xc02 ? 0x8000U : 0U))) return fail(index, "native MOV modifiers"); + if (opcode == 0x802) { + result.instructions.push_back(base(0x0100000000000000ULL) | (u64{15} << 12) | + (u64{insn.words[1]} << 20)); + } else if (opcode == 0xc02) { + if (insn.words[1] >= 16 || insn.words[2] != 0x08000f00) return fail(index, "native uniform move source"); + emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.words[1] + 216} << 20)); + } else if (opcode == 0x202) { + if (insn.Bits(40, 24)) return fail(index, "native MOV register modifiers"); + result.instructions.push_back(base(0x5c98000000000000ULL) | (u64{15} << 39) | + (u64{insn.Bits(32, 8)} << 20)); + } else { + if (!valid_cbuf() || insn.Bits(62, 2)) return fail(index, "native MOV constant buffer"); + result.instructions.push_back(base(0x4c98000000000000ULL) | (u64{15} << 39) | cbuf()); + } + break; + } + case 0x223: case 0x423: case 0x623: case 0x823: case 0xa23: case 0xc23: case 0xe23: { // FFMA + if (insn.Bits(76, 1) || (insn.Bits(81, 11) & ~(opcode == 0xe23 || opcode == 0xc23 ? 0x400U : 0U))) return fail(index, "native FFMA denormal/source mode"); + const u32 a = operand(0), b = operand(1), c = operand(2); + emit(base(0x5980000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | (u64{c} << 39) | + (u64{insn.Bits(77, 1)} << 50) | (u64{insn.Bits(78, 2)} << 51) | (u64{insn.Bits(80, 1)} << 53)); + break; + } + case 0x220: case 0x820: case 0xa20: case 0xc20: { // FMUL + const u32 scale = insn.Bits(84, 3); + if (insn.Bits(76, 1) || scale == 0 || insn.Bits(81, 3) || + insn.Bits(87, 4) || insn.Bits(91, 1) != (opcode == 0xc20) || + (scale != 4 && (!insn.Bits(80, 1) || insn.Bits(78, 2)))) + return fail(index, "native FMUL scale/mode"); + const u32 a = operand(0), b = operand(1); + emit(base(0x5c68000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(78, 2)} << 39) | (u64{std::array<u32, 8>{0, 3, 2, 1, 0, 6, 5, 4}[scale]} << 41) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(77, 1)} << 50)); + break; + } + case 0x221: case 0x421: case 0x621: case 0x821: case 0xa21: case 0xe21: { // FADD + if (insn.Bits(81, 10) || insn.Bits(91, 1) != (opcode == 0xe21)) return fail(index, "native FADD mode"); + const u32 a = operand(0), b = operand(insn.Bits(9, 3) == 2 || insn.Bits(9, 3) == 3 || insn.Bits(9, 3) == 7 ? 2 : 1); + emit(base(0x5c58000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(78, 2)} << 39) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(77, 1)} << 50)); + break; + } + case 0x209: case 0x809: case 0xa09: case 0xc09: { // FMNMX, including a uniform source + if (insn.Bits(74, 6) || insn.Bits(81, 6) || (opcode != 0xc09 && insn.Bits(91, 1))) + return fail(index, "native floating min/max mode"); + const u32 a = operand(0), b = operand(1); + emit(base(0x5c60000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | + (u64{insn.Bits(80, 1)} << 44)); + break; + } + case 0x308: case 0xb08: { // MUFU + const u32 operation = insn.Bits(74, 6); + if (operation > 8 || operation == 6 || operation == 7 || insn.Bits(80, 12)) return fail(index, "native MUFU mode"); + // SM70+ MUFU.COS/SIN take the angle in turns (compilers multiply by 1/2pi first); + // Eden's Maxwell MUFU takes radians (its RRO is a plain move), so scale by 2pi. + const bool turns = operation == 0 || operation == 1; + const auto to_radians = [&](u32 source) { + emit(0x1e00000000000000ULL | predicate | 250 | (u64{source} << 8) | + (u64{0x40c90fdbU} << 20)); + return 250U; + }; + if (insn.Bits(73, 1)) { + if (opcode != 0x308 || insn.Bits(60, 2) > 1) + return fail(index, "native half special-function source"); + emit(0x5ca8000000000000ULL | predicate | 250 | (u64{2} << 8) | (u64{1} << 10) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(60, 1)} << 41) | + (u64{insn.Bits(63, 1)} << 45) | (u64{insn.Bits(62, 1)} << 49)); + if (turns) to_radians(250); + emit(0x5080000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{operation} << 20)); + emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) | (u64{250} << 20)); + } else { + u32 src = operand(1); + if (turns) src = to_radians(src); + emit(base(0x5080000000000000ULL) | (u64{src} << 8) | (u64{operation} << 20)); + } + break; + } + case 0x526: { // IPA.PASS / CONSTANT with the attribute indexed by a uniform register + const u32 index_register = uniform_register(insn.Bits(24, 8)); + if (index_register == ~0U || insn.Bits(32, 8) != 255 || + (insn.Bits(76, 4) != 0 && insn.Bits(76, 4) != 4) || insn.Bits(81, 3) != 7) + return fail(index, "native indexed IPA mode"); + if (index_register == 255) { + emit(base(0xe000000000000000ULL) | (u64{255} << 8) | (u64{255} << 20) | + (u64{insn.Bits(64, 8)} << 30) | (u64{insn.Bits(78, 1)} << 55)); + break; + } + // Maxwell IPA.IDX reads the whole byte address from the index register. + emit(0x1c00000000000000ULL | predicate | 250 | (u64{index_register} << 8) | + (u64{insn.Bits(64, 8) * 4} << 20)); + emit(base(0xe000000000000000ULL) | (u64{250} << 8) | (u64{255} << 20) | + (u64{insn.Bits(64, 8)} << 30) | (u64{1} << 38) | (u64{insn.Bits(78, 1)} << 55)); + break; + } + case 0x326: { // IPA.PASS / CONSTANT, default or centroid sample location + // Bits 76-77 are the sample mode (title 1 interpolates most varyings at the centroid), + // bit 78 the constant mode; Maxwell keeps them at bits 52-53 and 55. + if (insn.Bits(32, 8) != 255 || insn.Bits(76, 2) > 1 || insn.Bits(79, 1) || + insn.Bits(81, 3) != 7) + return fail(index, "native IPA interpolation mode"); + emit(base(0xe000000000000000ULL) | (u64{255} << 8) | (u64{255} << 20) | (u64{insn.Bits(64, 8)} << 30) | + (u64{insn.Bits(76, 2)} << 52) | (u64{insn.Bits(78, 1)} << 55)); + break; + } + case 0x81a: { // SGXT.U32 Rd, Ra, n: keep the low n bits (BFE.U32 at position 0) + if (insn.words[2] || insn.words[1] > 32 || dest >= 255) + return fail(index, "native sign extend mode"); + emit(base(0x3800000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.words[1] << 8} << 20)); + break; + } + case 0x299: { // USHF by a uniform amount, one register shifted (SHL/SHR, bit 75 wraps). + const u32 dst = uniform_register(insn.Bits(16, 8)), low = uniform_register(insn.Bits(24, 8)); + const u32 shift = uniform_register(insn.Bits(32, 8)), high = uniform_register(insn.Bits(64, 8)); + const bool right = insn.Bits(76, 1) != 0, high_result = insn.Bits(80, 1) != 0; + if (dst == ~0U || dst == 255 || low == ~0U || shift == ~0U || high == ~0U || + native_predicate != 7 || insn.Bits(73, 2) != 3 || + (insn.words[2] & ~0x000118ffU) != 0x08000600U) + return fail(index, "native uniform register shift mode"); + u32 source; + if (high_result && low == 255) source = high; // (high:0) >> or << s, high word + else if (!high_result && high == 255) source = low; // (0:low) >> or << s, low word + else return fail(index, "native uniform funnel shift"); + emit((right ? 0x5c28000000000000ULL : 0x5c48000000000000ULL) | predicate | dst | + (u64{source} << 8) | (u64{shift} << 20) | (u64{insn.Bits(75, 1)} << 39)); + break; + } + case 0x419: { // SHF.L.W.U32.HI Rd, RZ, Rb, imm (high word of imm:0 << Rb) = imm << (Rb & 31) + if (insn.Bits(24, 8) != 255 || (insn.words[2] & ~0xffU) != 0x00010e00) + return fail(index, "native funnel shift immediate mode"); + mov_imm(250, insn.words[1]); + emit(base(0x5c48000000000000ULL) | (u64{250} << 8) | (u64{insn.Bits(64, 8)} << 20) | + (u64{1} << 39)); + break; + } + case 0x805: { // CS2R SRZ + if (insn.Bits(72, 8) != 255 || !insn.Bits(80, 1) || dest >= 249) return fail(index, "native special-register read"); + mov_imm(static_cast<u32>(dest), 0); + mov_imm(static_cast<u32>(dest) + 1, 0); + break; + } + case 0x306: case 0xd06: { // I2F.F32.U8, low byte, round-to-nearest (0xd06: uniform source) + const u32 source = opcode == 0xd06 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + if (opcode == 0x306 && insn.words[2] == 0x00101000 && !(insn.words[1] & 0xcfffff00U)) { + // I2F.U16 Rd, Rs(.H1): Maxwell's 16-bit source format, byte selector 0 or 2. + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{1} << 10) | (u64{source} << 20) | + (u64{insn.Bits(60, 2) * 2} << 41)); + break; + } + if (insn.words[2] != (opcode == 0xd06 ? 0x08001000U : 0x1000U) || + (insn.words[1] & 0xffffff00U) || source == ~0U) + return fail(index, "native byte-to-float conversion mode"); + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{source} << 20)); + break; + } + case 0x245: case 0xa45: case 0xc45: { // I2FP.F32.U32 + // Bits 78-79: rounding (RN/RM/RP/RZ), as Maxwell I2F's bits 39-40. + if ((insn.words[2] & ~(0x400U | 0xc000U | (opcode == 0xc45 ? 0x08000000U : 0U))) != 0x00201000) return fail(index, "native packed integer-to-float format"); + const u32 src = operand(1, false); + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | (u64{insn.Bits(74, 1)} << 13) | (u64{src} << 20) | + (u64{insn.Bits(78, 2)} << 39)); + break; + } + case 0x207: case 0x807: case 0xa07: + case 0x208: case 0x808: case 0xa08: { // Bitwise SEL / FSEL + const u32 a = operand(0, false), b = operand(1, false); + emit(base(0x5ca0000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + break; + } + case 0x20a: case 0x80a: case 0xa0a: { // FSET.BF, combined with Pc (bits 87-90) by bits 74-75 + if (insn.Bits(64, 8) || insn.Bits(74, 2) == 3 || insn.Bits(81, 6) || insn.Bits(91, 1)) + return fail(index, "native floating comparison output mode"); + const u32 a = operand(0), b = operand(1); + emit(base(0x5800000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(87, 4)} << 39) | (u64{insn.Bits(74, 2)} << 45) | + (u64{insn.Bits(76, 4)} << 48) | (u64{1} << 52) | (u64{insn.Bits(80, 1)} << 55)); + break; + } + case 0x434: { // HSETP2 with an immediate half pair: A swizzle 74-75 (neg 72, abs 73), + // combine 69-70, H_AND 71, compare 76-79, FTZ 80, P/Q 81-86, input 87-90. + mov_imm(250, insn.words[1]); + emit(0x5d20000000000000ULL | predicate | insn.Bits(84, 3) | (u64{insn.Bits(81, 3)} << 3) | + (u64{insn.Bits(80, 1)} << 6) | (u64{insn.Bits(24, 8)} << 8) | (u64{250} << 20) | + (u64{insn.Bits(76, 4)} << 35) | (u64{insn.Bits(87, 3)} << 39) | + (u64{insn.Bits(90, 1)} << 42) | (u64{insn.Bits(72, 1)} << 43) | + (u64{insn.Bits(73, 1)} << 44) | (u64{insn.Bits(69, 2)} << 45) | + (u64{insn.Bits(74, 2)} << 47) | (u64{insn.Bits(71, 1)} << 49)); + break; + } + case 0x20b: case 0x80b: case 0xa0b: // FSETP + case 0x20c: case 0x80c: case 0xa0c: case 0xc0c: { // ISETP + const bool floating = (opcode & 0x1ff) == 0xb; + if (!floating && insn.Bits(72, 1)) return fail(index, "native extended integer comparison"); + const u32 a = operand(0, floating), b = operand(1, floating); + const u32 combine = insn.Bits(74, 2); + if (combine > 2) return fail(index, "native predicate combination"); + u64 out = (floating ? 0x5bb0000000000000ULL : 0x5b60000000000000ULL) | predicate | + insn.Bits(84, 3) | (u64{insn.Bits(81, 3)} << 3) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | (u64{combine} << 45); + if (floating) out |= (u64{insn.Bits(80, 1)} << 47) | (u64{insn.Bits(76, 4)} << 48); + else out |= (u64{insn.Bits(73, 1)} << 48) | (u64{insn.Bits(76, 3)} << 49); + emit(out); + break; + } + case 0x212: case 0x812: case 0xa12: case 0xc12: { // LOP3.LUT, any table, optional P = (result != 0) + const u32 lut = insn.Bits(72, 8); + const u32 pred_target = insn.Bits(81, 3); + if (insn.Bits(80, 1) || insn.Bits(87, 4) != 15 || + (opcode == 0x212 && insn.Bits(40, 24)) || (opcode == 0xa12 && insn.Bits(62, 2))) + return fail(index, "native LUT/predicate mode"); + u32 b = 255; + if (opcode == 0x212) { + b = insn.Bits(32, 8); + } else if (opcode == 0x812) { + mov_imm(251, insn.words[1]); + b = 251; + } else if (opcode == 0xc12) { + b = uniform_register(insn.Bits(32, 8)); + if (b == ~0U) return fail(index, "native uniform logic source"); + } else { + b = operand(1, false); + } + const u32 a = insn.Bits(24, 8), c = insn.Bits(64, 8); + emit(0x5be0000000000000ULL | predicate | dest | (u64{a} << 8) | (u64{b} << 20) | + (u64{lut} << 28) | (u64{pred_target == 7 ? 0U : 3U} << 36) | (u64{c} << 39) | + (u64{pred_target} << 48)); + break; + } + case 0x213: case 0xa13: case 0xc13: { // IABS: max(x, 0 - x), signed. + if (insn.words[2] & ~(opcode == 0xc13 ? 0x08000000U : 0U)) + return fail(index, "native integer absolute mode"); + const u32 source = operand(1, false); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{255} << 8) | (u64{source} << 20) | + (u64{1} << 48)); + emit(base(0x5c20000000000000ULL) | (u64{source} << 8) | (u64{250} << 20) | + (u64{15} << 39) | (u64{1} << 48)); + break; + } + case 0x217: case 0x817: case 0xa17: case 0xc17: { // IMNMX register, immediate, cbuf or uniform operand. + if ((insn.words[2] & ~(0x04000200U | (opcode == 0xc17 ? 0x08000000U : 0U))) != 0x03800000 || + (opcode != 0xa17 && insn.words[1] > (opcode == 0x817 ? 0x7ffffU : 255U)) || + (opcode == 0xa17 && (!valid_cbuf() || insn.Bits(62, 2)))) + return fail(index, "native integer min/max mode"); + const u32 source = opcode == 0xc17 ? uniform_register(insn.words[1]) + : opcode == 0xa17 ? operand(1, false) + : insn.words[1]; + if (source == ~0U) return fail(index, "native uniform min/max operand"); + emit(base(opcode == 0x817 ? 0x3820000000000000ULL : 0x5c20000000000000ULL) | + (u64{insn.Bits(24, 8)} << 8) | (u64{source} << 20) | + (u64{insn.Bits(87, 4)} << 39) | (u64{insn.Bits(73, 1)} << 48)); + break; + } + case 0x819: { // SHF.HI, immediate shift + const bool is_signed = insn.Bits(73, 2) == 2; + if (!insn.Bits(80, 1) && !insn.Bits(76, 1) && insn.Bits(64, 8) == 255 && + insn.Bits(73, 2) == 3 && !insn.Bits(75, 1) && insn.words[1] <= 32) { + emit(base(0x3848000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20)); + break; + } + if (!insn.Bits(80, 1) && insn.Bits(76, 1) && insn.Bits(64, 8) == 255 && + (insn.Bits(73, 2) == 3 || is_signed) && !insn.Bits(75, 1) && insn.words[1] <= 32) { + // SHF.R low word with a zero high word: a logical right shift. + emit(base(0x3828000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20)); + break; + } + if ((insn.Bits(73, 2) != 3 && !is_signed) || insn.Bits(75, 1) || !insn.Bits(80, 1) || insn.words[1] > 32) return fail(index, "native funnel-shift mode"); + const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8), shift = insn.words[1]; + if (insn.Bits(76, 1)) { + if (low != 255) return fail(index, "native nonzero low right-shift word"); + emit(base(0x3828000000000000ULL) | (u64{high} << 8) | (u64{shift} << 20) | + (u64{is_signed} << 48)); + } else { + if (is_signed) return fail(index, "native signed left funnel shift"); + emit(base(0x36f8000000000000ULL) | (u64{low} << 8) | (u64{high} << 39) | (u64{shift} << 20)); + } + break; + } + case 0xa11: { // LEA with constant base and carry predicate. + if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || + insn.Bits(81, 3) == 7 || insn.Bits(62, 2)) + return fail(index, "native constant LEA carry mode"); + const u32 b = operand(1, false); + emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) | + (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); + break; + } + case 0x211: case 0x811: { // LEA low word, no carry result + if (insn.Bits(80, 1)) { + // LEA.HI Rd, Ra, Rb (or an immediate, 0x811), RZ, s: Rb + high word of (Ra << s); + // bit 73 sign-extends Ra. + if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(81, 3) != 7 || + insn.Bits(64, 8) != 255 || (opcode == 0x211 && insn.Bits(62, 2))) + return fail(index, "native high effective address mode"); + u32 b = insn.Bits(32, 8); + if (opcode == 0x811) { + b = insn.words[1] ? 251U : 255U; + if (b == 251) mov_imm(251, insn.words[1]); + } + const u64 shift = (u64{32 - insn.Bits(75, 5)} << 20) | (u64{insn.Bits(73, 1)} << 48); + if (b == 255) { + emit(base(0x3828000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | shift); + } else { + emit(0x3828000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | shift); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{b} << 20)); + } + break; + } + if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || insn.Bits(81, 3) != 7 || + (opcode == 0x211 && insn.Bits(62, 1))) return fail(index, "native LEA carry/high mode"); + // 0x211 bit 63 negates Rb (LEA Rd, Ra, -Rb, s), as Maxwell ISCADD's bit 48. + u32 b = insn.Bits(32, 8); + if (opcode == 0x811) { mov_imm(251, insn.words[1]); b = 251; } + emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39) | + (u64{opcode == 0x211 ? insn.Bits(63, 1) : 0U} << 48)); + break; + } + case 0x210: case 0x810: case 0xa10: case 0xc10: { // IADD3 with zero third source and no carry outputs + // 64-bit adds: IADD3 with a carry-out predicate (bits 81-83), then IADD3.X (bit 74) with + // a carry-in predicate (bits 87-89, negate 90). Uniform forms keep UPn in R239 bit 6-n. + if ((opcode == 0x210 || opcode == 0x810) && insn.Bits(64, 8) == 255 && !insn.Bits(73, 1) && + !insn.Bits(75, 2) && insn.Bits(84, 3) == 7 && insn.Bits(77, 4) == 15 && !insn.Bits(72, 1) && + (insn.Bits(81, 3) != 7 || insn.Bits(74, 1))) { + const bool carry_in = insn.Bits(74, 1) != 0; + const u32 carry_out = insn.Bits(81, 3); + const u32 a = insn.Bits(24, 8); + if ((carry_in && carry_out != 7) || (opcode == 0x210 && (insn.words[1] & ~0xffU)) || + (rewritten_uniform && carry_out != 7 && carry_out > 1) || + (rewritten_uniform && carry_in && insn.Bits(87, 3) > 1 && insn.Bits(87, 3) != 7)) + return fail(index, "native integer carry mode"); + if (opcode == 0x810) { + emit(0x1c00000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{insn.words[1]} << 20)); + } else { + emit(0x5c10000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{insn.Bits(32, 8)} << 20)); + } + const auto save = [&](u32 mask) { emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); }; + const auto restore = [&](u32 mask) { emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); }; + if (!carry_in) { + // carry = (a + b) < a, unsigned + const u32 target = rewritten_uniform ? 6 - carry_out : carry_out; + const u32 mask = 1U << target; + if (rewritten_uniform) save(mask); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{target} << 3) | (u64{250} << 8) | + (u64{a} << 20) | (u64{7} << 39) | (u64{1} << 49)); + if (rewritten_uniform) { + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20)); + restore(mask); + } + emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{250} << 20)); + break; + } + const u32 condition = insn.Bits(87, 3); + const u32 source = rewritten_uniform && condition != 7 ? 6 - condition : condition; + const u32 mask = 1U << source; + mov_imm(252, 1); + if (rewritten_uniform && condition != 7) { + save(mask); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{mask} << 20)); + } + emit(0x5ca0000000000000ULL | predicate | 251 | (u64{252} << 8) | (u64{255} << 20) | + (u64{source} << 39) | (u64{insn.Bits(90, 1)} << 42)); + if (rewritten_uniform && condition != 7) restore(mask); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{251} << 20)); + break; + } + if (opcode == 0xa10 && insn.Bits(81, 3) != 7) { + if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(84, 7) != 0x7f || + insn.Bits(77, 4) != 15 || insn.Bits(63, 1) || insn.Bits(72, 1)) + return fail(index, "native constant add carry mode"); + const u32 b = operand(1, false); + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) | + (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); + break; + } + if (opcode == 0xa10 && insn.Bits(74, 1)) { + if (insn.Bits(24, 8) != 255 || (insn.words[2] & ~0x07800000U) != 0x007fe4ffU) + return fail(index, "native extended constant add mode"); + const u32 b = operand(1, false); + mov_imm(250, 1); + emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(base(0x5c10000000000000ULL) | (u64{b} << 8) | (u64{250} << 20)); + break; + } + // Third source, optionally negated (bit 75; 63 and 72 negate b and a). + if (insn.Bits(64, 8) != 255 && (insn.Bits(73, 4) & ~4U) == 0 && insn.Bits(77, 7) == 0x7f && insn.Bits(84, 7) == 0x7f) { + const u32 b = operand(1, false); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | + (u64{b} << 20) | (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{insn.Bits(64, 8)} << 20) | + (u64{insn.Bits(75, 1)} << 48)); + break; + } + if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(77, 7) != 0x7f || insn.Bits(84, 7) != 0x7f) return fail(index, "native integer carry/third source"); + if (opcode == 0xa10 || opcode == 0xc10) { + if (opcode == 0xc10 && (insn.words[1] & ~0x8000003fU)) return fail(index, "native uniform add operand"); + const u32 b = operand(1, false); + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20) | + (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49)); + } else if (opcode == 0x210) { + if (insn.words[1] & ~0x800000ffU) return fail(index, "native integer add register modifiers"); + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49)); + } else { + emit(base(0x1c00000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20) | (u64{insn.Bits(72, 1)} << 56)); + } + break; + } + case 0x949: case 0x958: { // Captured BRX / BRXU with absolute native-PC table values + if (insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x3ffffU) != (opcode == 0x958 ? 0x0b800000U : 0x03800000U)) + return fail(index, "native computed branch mode"); + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + const s64 relative = static_cast<s64>(raw << 14) >> 14; + if (s64{index} * 16 + 16 + relative != 0) + return fail(index, "native computed branch relative base"); + const s64 offset = -s64{address_of(static_cast<u32>(result.instructions.size()))} - 8; + if (offset < -0x800000) return fail(index, "native computed branch range"); + const u32 source = opcode == 0x958 ? uniform_register(insn.Bits(24, 8)) : insn.Bits(24, 8); + if (source == ~0U) return fail(index, "native uniform computed branch source"); + emit(0xe25000000000000fULL | predicate | (u64{source} << 8) | + ((static_cast<u64>(offset) & 0xffffff) << 20)); + computed_branch = true; + break; + } + case 0x944: { // Verified CALL.REL.NOINC with R55 holding the return PC + // A predicated NOINC forward transfer can terminate a loop at its + // reconvergence point without a return-link setup or matching RET. + if (native_predicate != 7 && insn.words[1] == 16 && insn.words[2] == 0x03c00000 && + index + 2 < code.size() && code[index + 1].Bits(0, 12) == 0x947) { + relocations.push_back({static_cast<u32>(result.instructions.size()), index + 2}); + emit(0xe24000000000000fULL | predicate); + break; + } + if (main_exited || index == 0 || insn.words[0] != 0x7944 || + (insn.words[2] & ~0x3ffffU) != 0x03c00000 || + (code[index - 1].words[0] & ~0x00ff0000U) != 0x00007802 || + code[index - 1].Bits(16, 8) >= 250 || + code[index - 1].words[1] != (index + 1) * 16 || + code[index - 1].words[2] != 0xf00) + return fail(index, "native call linkage mode"); + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + const s64 relative = static_cast<s64>(raw << 14) >> 14; + const s64 target = s64{index} * 16 + 16 + relative; + if (target < 0 || target % 16) return fail(index, "native call destination alignment"); + const u32 link = code[index - 1].Bits(16, 8); + for (size_t call = 0; call < call_targets.size(); ++call) { + if (call_targets[call] == target / 16 && call_links[call] != link) + return fail(index, "native inconsistent call linkage"); + } + call_targets.push_back(static_cast<u32>(target / 16)); + call_links.push_back(link); + relocations.push_back({static_cast<u32>(result.instructions.size()), target / 16}); + emit(0xe26000000007000fULL); + break; + } + case 0x950: { // Verified RET.REL.NODEC epilogue restoring the call link + const u32 return_reg = insn.Bits(24, 8); + // The return register is the link itself, or the last write to it in this function + // is a copy of the link (MOV or IMAD.MOV.U32 Rd, RZ, RZ, Rlink). + const auto copies_link = [&](const Instruction& prior) { + if (prior.Bits(16, 8) != return_reg || prior.Bits(12, 4) != 7) return false; + if ((prior.words[0] & 0xff00ffffU) == 0x00007202U) + return prior.words[1] == *active_link && prior.words[2] == 0xf00; + if ((prior.words[0] & 0xff00ffffU) == 0xff007224U) + return prior.words[1] == 0xff && prior.words[2] == (0x078e0000U | *active_link); + return false; + }; + bool linked = active_link && return_reg == *active_link; + for (u32 prior = index; active_link && !linked && prior-- > active_entry;) { + if (copies_link(code[prior])) linked = true; + else if (code[prior].Bits(16, 8) == return_reg) break; + } + if (!main_exited || !linked || + (insn.words[0] & 0x00ffffffU) != 0x00007950 || return_reg >= 249 || + (insn.words[2] & ~0x3ffffU) != 0x03c00000 || !barrier_tokens.empty()) + return fail(index, "native return linkage mode"); + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + const s64 relative = static_cast<s64>(raw << 14) >> 14; + if (s64{index} * 16 + 16 + relative != 0) + return fail(index, "native return relative base"); + emit(0xe32000000007000fULL); + bool complete = !computed_branch; + for (u32 target : call_targets) complete &= target <= index; + if (complete) return finish(index); + break; + } + case 0x945: case 0x941: case 0x942: case 0x948: // (+ WARPSYNC) + // BSSY / BSYNC / BREAK only manage warp reconvergence: BSYNC waits for the other + // threads and falls through, BREAK drops the thread from that wait. Eden runs threads + // independently, so all three are no-ops (a SYNC would wrongly jump, and a branch + // out of a lowered SSY region would leave its token on Eden's flow stack). + break; + case 0x947: { // BRA + u64 branch_predicate = predicate; + if (opcode == 0x947 && insn.Bits(87, 4) != 7) { + if (insn.Bits(12, 4) != 7) { + relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1}); + emit(0xe24000000000000fULL | (u64{insn.Bits(12, 4) ^ 8U} << 16)); + } + branch_predicate = u64{insn.Bits(87, 4)} << 16; + } else if (insn.Bits(87, 4) != 7) { + return fail(index, "native extra branch condition"); + } + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + const s64 relative = static_cast<s64>(raw << 14) >> 14; + const s64 target = s64{index} * 16 + 16 + relative; + if (target < 0 || target % 16) return fail(index, "native branch destination alignment"); + relocations.push_back({static_cast<u32>(result.instructions.size()), target / 16}); + emit(0xe24000000000000fULL | branch_predicate); + if (computed_branch && target == s64{index} * 16 && branch_predicate == (u64{7} << 16)) + return finish(index); // the closing self-branch + break; + } + case 0x94d: { // EXIT + if (insn.Bits(84, 2)) { + return fail(index, "native conditional/special EXIT"); + } + if (insn.Bits(87, 4) != 7) { + // @Pg EXIT Pc: exits when both hold; skip over a Pc-predicated EXIT unless Pg. + if (native_predicate != 7) { + relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1}); + emit(0xe24000000000000fULL | (u64{native_predicate ^ 8U} << 16)); + } + emit(0xe30000000000000fULL | (u64{insn.Bits(87, 4)} << 16)); + break; + } + result.instructions.push_back(0xe30000000000000fULL | predicate); + if (insn.Bits(12, 4) != 7) break; + if (!barrier_tokens.empty()) return fail(index, "native unclosed reconvergence scope"); + if (call_targets.empty()) { + const bool pending_branch = std::any_of(relocations.begin(), relocations.end(), + [&](const Relocation& relocation) { return relocation.target > index; }); + if (!pending_branch && !computed_branch) return finish(index); + break; + } + if (main_exited) { + // A library function may end the shader instead of returning (Eden inlines + // calls, so this is an ordinary exit at that point). + bool complete = std::none_of(relocations.begin(), relocations.end(), + [&](const Relocation& relocation) { return relocation.target > index; }); + for (u32 target : call_targets) complete &= target <= index; + if (complete && !computed_branch) return finish(index); + break; + } + for (u32 target : call_targets) { + if (target <= index) return fail(index, "native call target overlaps main program"); + } + main_exited = true; + break; + } + default: + return fail(index, "unsupported native opcode"); + } + if (!operand_error.empty()) return fail(index, operand_error); + if (output_mask) { + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{output_mask} << 20)); + emit(0x38f0000000070000ULL | (u64{254} << 8) | (u64{output_mask} << 20)); + } + if (guarded_uniform) restore_predicate(guard_mask); + } + return fail(static_cast<u32>(code.size()), main_exited ? "native program lacks captured library return" : "native program has no unconditional EXIT"); +} + +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ampere/program_address_map.h b/src/shader_recompiler/frontend/ampere/program_address_map.h new file mode 100644 index 0000000000..bee32a821c --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/program_address_map.h @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once +#include <optional> +#include <span> +#include <vector> +#include "common/common_types.h" + +namespace Shader::Ampere { +constexpr u64 NativeAddressMapMagic = 0x50414d4e4544454eULL; +inline bool HasNativeAddressMap(std::span<const u64> code) { + return code.size() >= 2 && code.back() == NativeAddressMapMagic; +} +inline void AppendNativeAddressMap(std::vector<u64>& code, std::span<const u32> offsets) { + if (offsets.empty()) return; + for (u32 index : offsets) code.push_back(0x50U + index * 8 + ((index + 1) / 3) * 8); + code.push_back(offsets.size()); + code.push_back(NativeAddressMapMagic); +} +inline std::optional<u32> NativeProgramAddress(std::span<const u64> code, u32 address) { + if (!HasNativeAddressMap(code) || address % 16) return {}; + const u64 count = code[code.size() - 2]; + if (!count || count > 65536 || count + 2 > code.size() || address / 16 >= count) return {}; + const size_t start = code.size() - static_cast<size_t>(count) - 2; + const u64 target = code[start + address / 16]; + if (target < 0x50 || target >= start * 8 || target % 8 || target % 32 == 0) return {}; + return static_cast<u32>(target); +} +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ampere/program_capture.h b/src/shader_recompiler/frontend/ampere/program_capture.h new file mode 100644 index 0000000000..7e308de4ab --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/program_capture.h @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h" + +namespace Shader::Ampere { +class ProgramCaptureState { +public: + enum class Result { Continue, Complete, InvalidTarget }; + static constexpr u32 MaximumInstructions = 65536; + + Result Observe(const Instruction& instruction, u32 index) { + const u32 opcode = instruction.Bits(0, 12); + if (opcode == 0x944 || opcode == 0x947 || opcode == 0x945) { + const u64 raw = instruction.words[1] | (u64{instruction.words[2] & 0x3ffff} << 32); + const s64 relative = opcode == 0x945 ? static_cast<s32>(instruction.words[1]) + : static_cast<s64>(raw << 14) >> 14; + const s64 target = s64{index} * 16 + 16 + relative; + if (target < 0 || target % 16 || target >= MaximumInstructions * 16) + return Result::InvalidTarget; + // CALL.REL.NOINC (bit 86) pushes no return address: the compiler uses it as a jump + // (the lowering treats it as one), so it never pairs with a RET. + const bool noinc_call = opcode == 0x944 && (instruction.words[2] & 0x400000) != 0; + if (opcode == 0x944 && !noinc_call) { + last_call_target = std::max(last_call_target, static_cast<u32>(target / 16)); + } else { + last_forward_target = std::max(last_forward_target, static_cast<u32>(target / 16)); + } + // The compiler ends every program with an unconditional branch to itself; with a jump + // table (whose targets live in a constant buffer) that is the only reliable end. + if (opcode == 0x947 && computed_branch && instruction.Bits(12, 4) == 7 && + target == s64{index} * 16) + return Result::Complete; + } + if (opcode == 0x949 || opcode == 0x958) computed_branch = true; + if (computed_branch) return Result::Continue; + if (opcode == 0x94d && instruction.Bits(12, 4) == 7 && index >= last_forward_target) { + if (!last_call_target) return Result::Complete; + main_exited = true; + } + if (opcode == 0x950 && instruction.Bits(12, 4) == 7 && main_exited && + index >= last_call_target && index >= last_forward_target) + return Result::Complete; + return Result::Continue; + } + +private: + u32 last_call_target{}; + u32 last_forward_target{}; + bool main_exited{}; + bool computed_branch{}; +}; +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ampere/program_override.h b/src/shader_recompiler/frontend/ampere/program_override.h new file mode 100644 index 0000000000..0943028b0c --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/program_override.h @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include <cstring> +#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h" + +namespace Shader::Ampere { + +inline u64 NativeProgramKey(std::span<const Instruction> source) { + u64 hash = 14695981039346656037ULL; + for (const auto& instruction : source) { + for (u32 word : instruction.words) { + for (u32 byte = 0; byte < 4; ++byte) hash = (hash ^ ((word >> (byte * 8)) & 255)) * 1099511628211ULL; + } + } + return hash; +} + +// Private iteration format: version, lengths, texture bank, exact original +// instructions, then the newly generated canonical instructions. Source bytes +// are compared directly; the filename hash is only an index. +inline std::vector<u8> EncodeProgramOverride(std::span<const Instruction> source, + const LoweredProgram& program) { + if (!program || program.source_size != source.size_bytes() || program.source_instruction_offsets.size() != source.size()) return {}; + const std::array<u32, 6> header{0x4e454445, 0x3230564f, + static_cast<u32>(source.size_bytes()), static_cast<u32>(program.instructions.size()), + program.required_texture_buffer, static_cast<u32>(program.source_instruction_offsets.size())}; + std::vector<u8> bytes(sizeof(header) + source.size_bytes() + program.instructions.size() * 8 + program.source_instruction_offsets.size() * 4); + std::memcpy(bytes.data(), header.data(), sizeof(header)); + std::memcpy(bytes.data() + sizeof(header), source.data(), source.size_bytes()); + std::memcpy(bytes.data() + sizeof(header) + source.size_bytes(), program.instructions.data(), program.instructions.size() * 8); + std::memcpy(bytes.data() + sizeof(header) + source.size_bytes() + program.instructions.size() * 8, + program.source_instruction_offsets.data(), program.source_instruction_offsets.size() * 4); + return bytes; +} + +inline u32 ProgramOverrideSourceSize(std::span<const u8> bytes) { + std::array<u32, 6> header{}; + if (bytes.size() < sizeof(header)) return 0; + std::memcpy(header.data(), bytes.data(), sizeof(header)); + const bool mapped = header[1] == 0x3230564f; + if (header[0] != 0x4e454445 || (!mapped && header[1] != 0x3130564f) || + (mapped ? header[5] != header[2] / 16 : header[5] != 0) || + header[2] == 0 || header[2] % sizeof(Instruction) || header[2] > 1048576 || + header[3] == 0 || header[3] > 1048576 || (header[4] != ~0U && header[4] >= 18)) return 0; + if (bytes.size() != sizeof(header) + size_t{header[2]} + size_t{header[3]} * 8 + size_t{header[5]} * 4) return 0; + return header[2]; +} + +inline LoweredProgram DecodeProgramOverride(std::span<const u8> bytes, + std::span<const Instruction> source) { + LoweredProgram result; + const auto fail = [&result](std::string_view message) { + result.instructions.clear(); + result.source_instruction_offsets.clear(); + result.error = message; + return result; + }; + std::array<u32, 6> header{}; + if (!ProgramOverrideSourceSize(bytes)) return fail("invalid native override header or length"); + std::memcpy(header.data(), bytes.data(), sizeof(header)); + if (header[2] != source.size_bytes()) return fail("native override source length mismatch"); + if (std::memcmp(bytes.data() + sizeof(header), source.data(), source.size_bytes())) + return fail("native override source mismatch"); + result.instructions.resize(header[3]); + std::memcpy(result.instructions.data(), bytes.data() + sizeof(header) + source.size_bytes(), header[3] * 8); + if (header[5]) { + result.source_instruction_offsets.resize(header[5]); + std::memcpy(result.source_instruction_offsets.data(), + bytes.data() + sizeof(header) + source.size_bytes() + size_t{header[3]} * 8, + size_t{header[5]} * 4); + u32 previous = 0; + for (u32 offset : result.source_instruction_offsets) { + if (offset >= result.instructions.size() || offset < previous) + return fail("native override address map mismatch"); + previous = offset; + } + } + result.source_size = header[2]; + result.required_texture_buffer = header[4]; + return result; +} + +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ir/breadth_first_search.h b/src/shader_recompiler/frontend/ir/breadth_first_search.h index afe153f180..ad19917781 100644 --- a/src/shader_recompiler/frontend/ir/breadth_first_search.h +++ b/src/shader_recompiler/frontend/ir/breadth_first_search.h @@ -6,10 +6,12 @@ #pragma once +#include <algorithm> #include <functional> #include <optional> #include <type_traits> #include <queue> +#include <unordered_set> #include <boost/container/small_vector.hpp> @@ -28,6 +30,23 @@ auto BreadthFirstSearch(Value value, Pred&& pred, Resolve&& resolve = {}) // Breadth-first search visiting the right most arguments first // Small vector has been determined from shaders in Super Smash Bros. Ultimate boost::container::small_vector<const Inst*, 2> visited; + // Once a search grows, a linear scan of `visited` per argument made it quadratic: a 28,000 + // instruction Switch 2 compute shader took 3.4 s to translate, most of it here. Same visit + // order, so the same result. + std::unordered_set<const Inst*> visited_set; + const auto first_visit = [&](const Inst* inst) { + if (!visited_set.empty()) { + return visited_set.insert(inst).second; + } + if (std::ranges::find(visited, inst) != visited.end()) { + return false; + } + visited.push_back(inst); + if (visited.size() > 64) { + visited_set.insert(visited.begin(), visited.end()); + } + return true; + }; std::queue<const Inst*> queue; queue.push(value.InstRecursive()); @@ -47,8 +66,7 @@ auto BreadthFirstSearch(Value value, Pred&& pred, Resolve&& resolve = {}) } // Queue instruction if it hasn't been visited const Inst* const arg_inst{arg_value.InstRecursive()}; - if (std::ranges::find(visited, arg_inst) == visited.end()) { - visited.push_back(arg_inst); + if (first_visit(arg_inst)) { queue.push(arg_inst); } } diff --git a/src/shader_recompiler/frontend/maxwell/control_flow.cpp b/src/shader_recompiler/frontend/maxwell/control_flow.cpp index 7dabfbf16e..9fb30df664 100644 --- a/src/shader_recompiler/frontend/maxwell/control_flow.cpp +++ b/src/shader_recompiler/frontend/maxwell/control_flow.cpp @@ -457,7 +457,7 @@ CFG::AnalysisState CFG::AnalyzeBRX(Block* block, Location pc, Instruction inst, block->indirect_branches.reserve(targets.size()); for (const u32 target : targets) { - Block* const branch{AddLabel(block, block->stack, target, function_id)}; + Block* const branch{AddLabel(block, block->stack, env.TranslateProgramAddress(target), function_id)}; block->indirect_branches.push_back({ .block = branch, .address = target, diff --git a/src/shader_recompiler/frontend/maxwell/translate_program.cpp b/src/shader_recompiler/frontend/maxwell/translate_program.cpp index 0cc4b3b844..cd4678cef6 100644 --- a/src/shader_recompiler/frontend/maxwell/translate_program.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate_program.cpp @@ -391,6 +391,7 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo Optimization::ConstantPropagationPass(env, program); Optimization::PositionPass(env, program); + Optimization::BlitConstantsPass(env, program); Optimization::GlobalMemoryToStorageBufferPass(program, normalized_host_info); Optimization::GeometryCompactionPass(program, normalized_host_info); diff --git a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp index f49d54e7ab..bdb4494a15 100644 --- a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp +++ b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <optional> +#include <unordered_map> #include <boost/container/flat_set.hpp> #include <boost/container/small_vector.hpp> @@ -54,6 +55,11 @@ struct StorageInfo { StorageBufferSet set; StorageInstVector to_replace; StorageWritesSet writes; + /// Track results by the instruction the search starts from, with and without the NVN bias: + /// accesses through the same address share them, and a search can walk thousands of + /// instructions (the IR does not change while storage buffers are collected). + std::unordered_map<const IR::Inst*, std::optional<StorageBufferAddr>> biased_tracks; + std::unordered_map<const IR::Inst*, std::optional<StorageBufferAddr>> unbiased_tracks; }; /// Returns true when the instruction is a global memory instruction @@ -387,10 +393,22 @@ void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info) } // First try to find storage buffers in the NVN address const IR::U32 low_addr{low_addr_info->value}; - std::optional<StorageBufferAddr> storage_buffer{Track(low_addr, &nvn_bias)}; + const auto cached_track = [&](const Bias* bias, auto& cache) { + if (low_addr.IsImmediate()) { + return Track(low_addr, bias); + } + const IR::Inst* const start{low_addr.InstRecursive()}; + if (const auto it = cache.find(start); it != cache.end()) { + return it->second; + } + const std::optional<StorageBufferAddr> result{Track(low_addr, bias)}; + cache.emplace(start, result); + return result; + }; + std::optional<StorageBufferAddr> storage_buffer{cached_track(&nvn_bias, info.biased_tracks)}; if (!storage_buffer) { // If it fails, track without a bias - storage_buffer = Track(low_addr, nullptr); + storage_buffer = cached_track(nullptr, info.unbiased_tracks); if (!storage_buffer) { // If that also fails, use NVN fallbacks LOG_WARNING(Shader, "Storage buffer failed to track, using global memory fallbacks"); diff --git a/src/shader_recompiler/ir_opt/passes.h b/src/shader_recompiler/ir_opt/passes.h index bf325a4cd9..e3422deb9d 100644 --- a/src/shader_recompiler/ir_opt/passes.h +++ b/src/shader_recompiler/ir_opt/passes.h @@ -28,6 +28,7 @@ void LowerInt64ToInt32(IR::Program& program); void RescalingPass(IR::Program& program); void SsaRewritePass(IR::Program& program); void PositionPass(Environment& env, IR::Program& program); +void BlitConstantsPass(Environment& env, IR::Program& program); void TexturePass(Environment& env, IR::Program& program, const HostTranslateInfo& host_info); void LayerPass(IR::Program& program, const HostTranslateInfo& host_info); void VendorWorkaroundPass(IR::Program& program); diff --git a/src/shader_recompiler/ir_opt/position_pass.cpp b/src/shader_recompiler/ir_opt/position_pass.cpp index 3c20b71897..4890346944 100644 --- a/src/shader_recompiler/ir_opt/position_pass.cpp +++ b/src/shader_recompiler/ir_opt/position_pass.cpp @@ -74,4 +74,70 @@ void PositionPass(Environment& env, IR::Program& program) { } } } +// Switch 2 fullscreen blit vertex shaders read a parameter block from constant bank 24 (lowered to +// bank 17, which games never bind) that the console provides itself. For a blit covering the whole +// render target it holds: texel size (0x0/0x4, again at 0x10/0x14), size (0x8/0xc), a UV scale of +// 1 (0x18/0x1c), the UV clamp rectangle 0,0..1,1 (0x30-0x3c, read by the fragment shader), a zero +// dither bias (0x40-0x48) and the integer source rectangle 0,0..W-1,H-1 (0x50-0x5c). +void BlitConstantsPass(Environment& env, IR::Program& program) { + if (env.ShaderStage() != Stage::VertexB && env.ShaderStage() != Stage::Fragment) { + return; + } + bool used{}; + for (IR::Block* const block : program.post_order_blocks) { + for (IR::Inst& inst : block->Instructions()) { + const IR::Opcode opcode{inst.GetOpcode()}; + if (opcode != IR::Opcode::GetCbufU32 && opcode != IR::Opcode::GetCbufF32 && + opcode != IR::Opcode::GetCbufU32x2) { + continue; + } + if (!inst.Arg(0).IsImmediate() || inst.Arg(0).U32() != 17 || !inst.Arg(1).IsImmediate()) { + continue; + } + IR::IREmitter ir{*block, IR::Block::InstructionList::s_iterator_to(inst)}; + const IR::F32 width{ir.RenderAreaWidth()}; + const IR::F32 height{ir.RenderAreaHeight()}; + const auto word = [&](u32 offset) -> IR::U32 { + switch (offset) { + case 0x0: + case 0x10: + return ir.BitCast<IR::U32>(IR::F32{ir.FPRecip(width)}); + case 0x4: + case 0x14: + return ir.BitCast<IR::U32>(IR::F32{ir.FPRecip(height)}); + case 0x8: + return ir.BitCast<IR::U32>(width); + case 0xc: + return ir.BitCast<IR::U32>(height); + case 0x18: + case 0x1c: + case 0x38: + case 0x3c: + return ir.Imm32(0x3f800000U); + case 0x58: + return IR::U32{ir.ConvertFToU(32, ir.FPAdd(width, ir.Imm32(-1.0f)))}; + case 0x5c: + return IR::U32{ir.ConvertFToU(32, ir.FPAdd(height, ir.Imm32(-1.0f)))}; + default: + return ir.Imm32(0U); + } + }; + const u32 offset{inst.Arg(1).U32()}; + IR::Value value; + if (opcode == IR::Opcode::GetCbufU32x2) { + value = ir.CompositeConstruct(word(offset), word(offset + 4)); + } else if (opcode == IR::Opcode::GetCbufF32) { + value = ir.BitCast<IR::F32>(word(offset)); + } else { + value = word(offset); + } + inst.ReplaceUsesWith(value); + used = true; + } + } + if (used) { + program.info.uses_render_area = true; + } +} + } // namespace Shader::Optimization diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 64d14ab2a6..e97b425412 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -90,6 +90,8 @@ add_library(video_core STATIC host1x/vic.h macro.cpp macro.h + macro_memo.h + macro_turing.h fence_manager.h gpu.cpp gpu.h @@ -99,6 +101,10 @@ add_library(video_core STATIC invalidation_accumulator.h memory_manager.cpp memory_manager.h + native_qmd.h + native_query_trace.h + nx2.h + nx2_profile.h present.h pte_kind.h query_cache/bank_base.h diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index f30c85911f..ad2e200438 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -9,6 +9,9 @@ #include <algorithm> #include <memory> #include <numeric> +#include <stdexcept> +#include <vector> +#include <fmt/ranges.h> #include "common/range_sets.inc" #include "video_core/buffer_cache/buffer_cache_base.h" @@ -78,6 +81,20 @@ void BufferCache<P>::TickFrame() { return; } runtime.TickFrame(slot_buffers); + if (static u64 frames{}; ++frames % 600 == 0) { + u64 total{}; + size_t count{}; + std::vector<std::pair<u64, VAddr>> largest; + for (const auto [buffer_id, buffer] : slot_buffers) { + total += buffer->SizeBytes(); + ++count; + largest.emplace_back(buffer->SizeBytes(), buffer->CpuAddr()); + } + std::partial_sort(largest.begin(), largest.begin() + std::min<size_t>(6, largest.size()), + largest.end(), std::greater<>{}); + LOG_INFO(HW_GPU, "Buffer cache: {} buffers {} MiB; largest {}", count, total >> 20, + fmt::join(largest.begin(), largest.begin() + std::min<size_t>(6, largest.size()), ", ")); + } // Calculate hits and shots and move hit bits to the right const u32 hits = std::reduce(channel_state->uniform_cache_hits.begin(), @@ -1194,8 +1211,15 @@ void BufferCache<P>::BindHostComputeUniformBuffers() { return alignment > 1 && (offset % alignment) != 0; } }(); + // Small buffers the GPU has not written are streamed, as graphics ones are: synchronizing + // them made every dispatch re-protect pages an inline upload had just unprotected (two + // VirtualProtect calls per dispatch, ~4,000 dispatches a second in title 1). + const bool use_fast_buffer = + needs_alignment_stream || + (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size && + !memory_tracker.IsRegionGpuModified(binding.device_addr, size)); if constexpr (!IS_OPENGL) { - if (needs_alignment_stream) { + if (use_fast_buffer) { const std::span<u8> span = runtime.BindMappedUniformBuffer(0, binding_index, size); device_memory.ReadBlockUnsafe(binding.device_addr, span.data(), size); @@ -1372,7 +1396,12 @@ void BufferCache<P>::UpdateVertexBuffer(u32 index) { const auto& array = maxwell3d->regs.vertex_streams[index]; const auto& limit = maxwell3d->regs.vertex_stream_limits[index]; const GPUVAddr gpu_addr_begin = array.Address(); - const GPUVAddr gpu_addr_end = limit.Address() + 1; + const bool native = maxwell3d->GetNativeMacroEngine().native_turing; + const u64 native_size = native ? maxwell3d->regs.NativeVertexStreamSize(index) : 0; + if (native && array.enable != 0 && native_size > (std::numeric_limits<u32>::max)()) { + throw std::runtime_error("Native vertex stream exceeds supported 32-bit buffer size"); + } + const GPUVAddr gpu_addr_end = native ? gpu_addr_begin + native_size : limit.Address() + 1; const std::optional<DAddr> device_addr = gpu_memory->GpuToCpuAddress(gpu_addr_begin); const u32 address_size = static_cast<u32>(gpu_addr_end - gpu_addr_begin); u32 size = address_size; // TODO: Analyze stride and number of vertices diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 26bf3dd9ed..57011752cc 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -21,6 +21,7 @@ #include <boost/container/small_vector.hpp> #include "common/common_types.h" +#include "common/spin_recursive_mutex.h" #include "common/div_ceil.h" #include "common/literals.h" #include "common/lru_cache.h" @@ -339,7 +340,7 @@ public: } while (channel_state->has_deleted_buffers); } - std::recursive_mutex mutex; + Common::SpinRecursiveMutex mutex; Runtime& runtime; bool any_buffer_uploaded = false; @@ -544,7 +545,8 @@ private: bool immediately_free = false; #endif - std::array<BufferId, ((1ULL << 34) >> CACHING_PAGEBITS)> page_table; + // Covers the device address space (MaxwellDeviceTraits::device_virtual_bits). + std::array<BufferId, ((1ULL << 36) >> CACHING_PAGEBITS)> page_table; Common::ScratchBuffer<u8> tmp_buffer; }; diff --git a/src/video_core/buffer_cache/memory_tracker_base.h b/src/video_core/buffer_cache/memory_tracker_base.h index b8547f3149..8936da023f 100644 --- a/src/video_core/buffer_cache/memory_tracker_base.h +++ b/src/video_core/buffer_cache/memory_tracker_base.h @@ -22,7 +22,7 @@ namespace VideoCommon { template <typename DeviceTracker> class MemoryTrackerBase { - static constexpr size_t MAX_CPU_PAGE_BITS = 34; + static constexpr size_t MAX_CPU_PAGE_BITS = 36; // device addresses (MaxwellDeviceTraits) static constexpr size_t HIGHER_PAGE_BITS = 22; static constexpr size_t HIGHER_PAGE_SIZE = 1ULL << HIGHER_PAGE_BITS; static constexpr size_t HIGHER_PAGE_MASK = HIGHER_PAGE_SIZE - 1ULL; diff --git a/src/video_core/buffer_cache/usage_tracker.h b/src/video_core/buffer_cache/usage_tracker.h index 6f84e463e3..3e2da8bc0b 100644 --- a/src/video_core/buffer_cache/usage_tracker.h +++ b/src/video_core/buffer_cache/usage_tracker.h @@ -6,6 +6,10 @@ #pragma once +#include <array> +#include <utility> +#include <vector> + #include "common/alignment.h" #include "common/common_types.h" @@ -24,9 +28,19 @@ public: void Reset() noexcept { std::ranges::fill(pages, 0ULL); + recent.fill({~u64{0}, 0}); } void Track(u64 offset, u64 size) noexcept { + // Tracking is idempotent until Reset: skip ranges tracked just before. Draws bind the + // same (often multi-megabyte) storage buffers again and again, and each Track wrote a + // word per 4 KiB of the range. + for (const auto& [recent_offset, recent_size] : recent) { + if (recent_offset == offset && recent_size == size) { + return; + } + } + recent[recent_cursor++ % recent.size()] = {offset, size}; const size_t page = offset >> BUFFER_PAGE_SHIFT; const size_t page_end = (offset + size) >> BUFFER_PAGE_SHIFT; if (page_end < page || page_end >= pages.size()) { @@ -83,6 +97,9 @@ private: private: std::vector<u64> pages; + /// Ranges tracked last, (offset, size). + std::array<std::pair<u64, u64>, 4> recent{{{~u64{0}, 0}, {~u64{0}, 0}, {~u64{0}, 0}, {~u64{0}, 0}}}; + size_t recent_cursor{}; }; } // namespace VideoCommon diff --git a/src/video_core/control/channel_state.cpp b/src/video_core/control/channel_state.cpp index 12a452dfc0..8e0b6c1b9e 100644 --- a/src/video_core/control/channel_state.cpp +++ b/src/video_core/control/channel_state.cpp @@ -13,7 +13,9 @@ #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/maxwell_dma.h" #include "video_core/engines/puller.h" +#include "video_core/macro.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" namespace Tegra::Control { namespace { @@ -21,15 +23,30 @@ namespace { // Match NVK/Nouveau's initial pushbuffer subchannel layout. constexpr u32 Nvk3DSubchannel = 0; constexpr u32 NvkComputeSubchannel = 1; +constexpr u32 NvkM2mfSubchannel = 2; constexpr u32 Nvk2DSubchannel = 3; constexpr u32 NvkCopySubchannel = 4; +// Switch 2 channels of a process share one TSG, whose graphics context holds the macro RAM. +std::shared_ptr<NativeMacroContext> Nx2MacroContext() { + static const auto context = [] { + auto created = std::make_shared<NativeMacroContext>(); + created->engine.native_turing = true; + created->compute_engine.native_turing = true; + return created; + }(); + return context; +} + void BindNvkDefaultSubchannels(ChannelState::Payload& payload) { auto& dma_pusher = payload.dma_pusher; dma_pusher.BindSubchannel(&payload.maxwell_3d, Nvk3DSubchannel, Engines::EngineTypes::Maxwell3D); dma_pusher.BindSubchannel(&payload.kepler_compute, NvkComputeSubchannel, Engines::EngineTypes::KeplerCompute); - // Subchannel 2 is M2MF there; Eden does not expose a 0x9039 engine yet. + // Subchannel 2 is M2MF/inline-to-memory there (NVN uses it without binding a class); + // KeplerMemory has the same LINE_LENGTH_IN/OFFSET_OUT/LAUNCH_DMA/LOAD_INLINE_DATA layout. + dma_pusher.BindSubchannel(&payload.kepler_memory, NvkM2mfSubchannel, + Engines::EngineTypes::KeplerMemory); dma_pusher.BindSubchannel(&payload.fermi_2d, Nvk2DSubchannel, Engines::EngineTypes::Fermi2D); dma_pusher.BindSubchannel(&payload.maxwell_dma, NvkCopySubchannel, Engines::EngineTypes::MaxwellDMA); @@ -56,6 +73,17 @@ void ChannelState::Init(Core::System& system, u64 program_id_) { program_id = program_id_; payload.emplace(system, *memory_manager, *this); BindNvkDefaultSubchannels(*payload); + if (g_nx2_ampere) { + // Switch 2 command streams use Ampere classes with Turing+ (MME64) macros, and its + // block-linear surfaces the console's GOB swizzle. + memory_manager->EnableNativeTiling(); + const auto macros = Nx2MacroContext(); + payload->maxwell_3d.SetNativeMacroContext(macros); + payload->maxwell_3d.SetNativeTuringMacroMode(true); + payload->kepler_compute.SetNativeMacroMode(¯os->engine, macros->positions, + macros.get()); + payload->kepler_compute.SetNativeDataRam(&payload->maxwell_3d.GetNativeDataRam()); + } initialized = true; } diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp index 8af13e415c..24f1db1f7c 100644 --- a/src/video_core/control/scheduler.cpp +++ b/src/video_core/control/scheduler.cpp @@ -4,7 +4,9 @@ // SPDX-FileCopyrightText: 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include <algorithm> #include <memory> +#include <vector> #include "common/assert.h" #include "video_core/control/channel_state.h" @@ -26,6 +28,36 @@ void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) { // Multiple channels can prepare their commands in parallel. channel_state->payload->dma_pusher.Push(std::move(entries)); channel_state->payload->dma_pusher.DispatchCalls(); + if (channel_state->payload->dma_pusher.HasPendingAcquire() && + std::find(pending_channels.begin(), pending_channels.end(), channel_state) == + pending_channels.end()) { + pending_channels.push_back(channel_state); + } + if (!pending_channels.empty()) { + ResumePending(gpu); + } +} + +void Scheduler::ResumePending(GPU& gpu) { + // Running one channel can release the semaphore another waits for: retry until none moves. + for (bool progress = true; progress;) { + progress = false; + for (const auto& state : pending_channels) { + auto& dma_pusher = state->payload->dma_pusher; + if (!dma_pusher.HasPendingAcquire() || !dma_pusher.CanResumeAcquire()) { + continue; + } + { + std::scoped_lock lk{scheduling_guard}; + gpu.BindChannel(state->bind_id); + } + dma_pusher.DispatchCalls(); + progress = true; + } + } + std::erase_if(pending_channels, [](const auto& state) { + return !state->payload->dma_pusher.HasPendingAcquire(); + }); } void Scheduler::DeclareChannel(std::shared_ptr<ChannelState> new_channel) { diff --git a/src/video_core/control/scheduler.h b/src/video_core/control/scheduler.h index 342bea5025..3a3dcc58cf 100644 --- a/src/video_core/control/scheduler.h +++ b/src/video_core/control/scheduler.h @@ -8,6 +8,7 @@ #include <memory> #include <mutex> +#include <vector> #include "common/container/unordered_map.h" #include "video_core/dma_pusher.h" @@ -24,11 +25,18 @@ class Scheduler { public: void Push(GPU& gpu, s32 channel, CommandList&& entries); + /// Runs channels whose pending semaphore acquire is now met. + void ResumePending(GPU& gpu); + [[nodiscard]] bool HasPendingChannels() const { + return !pending_channels.empty(); + } + void DeclareChannel(std::shared_ptr<ChannelState> new_channel); private: ::Common::unordered_map<s32, std::shared_ptr<ChannelState>> channels; std::mutex scheduling_guard; + std::vector<std::shared_ptr<ChannelState>> pending_channels; ///< Suspended on an acquire }; } // namespace Control diff --git a/src/video_core/dirty_flags.cpp b/src/video_core/dirty_flags.cpp index c2ecc12f56..e6856aaca3 100644 --- a/src/video_core/dirty_flags.cpp +++ b/src/video_core/dirty_flags.cpp @@ -22,6 +22,7 @@ void SetupDirtyVertexBuffers(Maxwell3D::DirtyState::Tables& tables) { OFF(vertex_stream_limits) + i * NUM(vertex_stream_limits[0]); FillBlock(tables, array_offset, num_array, VertexBuffer0 + i, VertexBuffers); + FillBlock(tables, 0x600 / sizeof(u32) + i * 2, 2, VertexBuffer0 + i, VertexBuffers); FillBlock(tables, limit_offset, NUM(vertex_stream_limits), VertexBuffer0 + i, VertexBuffers); } diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 2c0ba007ef..3cc5afe967 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -4,13 +4,21 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <algorithm> +#include <atomic> +#include <cstdlib> +#include <cstdio> + #include "common/settings.h" #include "core/core.h" +#include "core/memory.h" #include "video_core/dma_pusher.h" +#include "video_core/control/channel_state.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/gpu.h" #include "video_core/guest_memory.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" #include "video_core/rasterizer_interface.h" namespace Tegra { @@ -29,7 +37,7 @@ DmaPusher::DmaPusher(Core::System& system_, MemoryManager& memory_manager_, Cont DmaPusher::~DmaPusher() = default; void DmaPusher::DispatchCalls() { - dma_pushbuffer_subindex = 0; + // The sub-index is kept: a channel suspended on a semaphore resumes inside its command list. dma_state.is_last_call = true; while (system.IsPoweredOn()) { if (!Step()) { @@ -44,6 +52,9 @@ bool DmaPusher::Step() { if (!ib_enable || dma_pushbuffer.empty()) { return false; } + if (!puller.ResumeAcquire(*this)) { + return false; + } CommandList& command_list = dma_pushbuffer.front(); @@ -57,7 +68,13 @@ bool DmaPusher::Step() { } if (prefetch_size > 0) { - ProcessCommands(command_list.prefetch_command_list); + const std::span<const CommandHeader> commands{command_list.prefetch_command_list}; + command_resume_offset += + ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); + if (puller.HasPendingAcquire()) { + return false; + } + command_resume_offset = 0; dma_pushbuffer.pop(); return true; } @@ -83,17 +100,74 @@ bool DmaPusher::Step() { subchannels[dma_state.subchannel]->current_dirty = memory_manager.IsMemoryDirty(dma_state.dma_get, words * sizeof(u32)); } } + if (command_resume_offset == 0) { + // A method's data continues across GP entries, whatever their level: title 4 ends a + // main entry after a macro's first word and supplies the remaining parameters in a + // 5-word subroutine entry. EDEN_NX2_PARK_SPLIT_METHODS restores the old guess (an + // entry elsewhere is a call that starts clean; resume where the split one ended). + static const bool park_split_methods = std::getenv("EDEN_NX2_PARK_SPLIT_METHODS") != nullptr; + if (g_nx2_ampere) { + if (static u32 split_logs{}; dma_state.method_count != 0 && split_logs < 64) { + ++split_logs; + LOG_DEBUG(HW_GPU, "Split method {:#x} ({} left) on ch {}: next entry {:#x} size {} raw {:#x} (level bit {}, priv bit {}), expected {:#x}, parked {}", + dma_state.method, dma_state.method_count, channel_state.bind_id, + u64{header.addr}, u64{header.size}, header.raw, u64{header.is_push_buffer}, + u64{header.allow_flush}, split_method_end, parked_methods.size()); + std::array<u32, 8> next_words{}; + std::array<u32, 4> prev_words{}; + memory_manager.ReadBlock(header.addr, next_words.data(), + std::min<size_t>(header.size, 8) * sizeof(u32)); + memory_manager.ReadBlock(split_method_end - sizeof(prev_words), prev_words.data(), + sizeof(prev_words)); + LOG_DEBUG(HW_GPU, " previous entry ends {:08x}; next entry starts {:08x}", + fmt::join(prev_words, " "), fmt::join(next_words, " ")); + } + if (park_split_methods) { + ResolveSplitMethod(header.addr); + } + } + if (dma_pushbuffer_subindex < command_list.submit_hashes.size()) { + CheckSubmitHash(header, command_list.submit_hashes[dma_pushbuffer_subindex]); + } + } + // A resumed entry is read again: the words after the acquire may be new by now. const bool use_safe = Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe(); if (use_safe) { Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::SafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers); - ProcessCommands(headers); + const std::span<const CommandHeader> commands{headers.data(), headers.size()}; + RecordParsedEntry(header, commands); + command_resume_offset += + ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); } else { Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::UnsafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers); - ProcessCommands(headers); + const std::span<const CommandHeader> commands{headers.data(), headers.size()}; + RecordParsedEntry(header, commands); + command_resume_offset += + ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); + } + if (puller.HasPendingAcquire()) { + return false; + } + command_resume_offset = 0; + if (dma_state.method_count != 0) { + split_method_end = header.addr + header.size * sizeof(u32); } } if (++dma_pushbuffer_subindex >= command_list_size) { + // A split method resumes within the submission that split it; ring addresses repeat + // every frame, so a stale one could otherwise swallow the headers of a later entry. + for (const auto& parked : parked_methods) { + static std::atomic<u32> reported{}; + if (reported.fetch_add(1) < 16) { + LOG_WARNING(HW_GPU, + "Submission ended with parked method {:#x} (sub {}, {} words left, " + "resume {:#x})", + parked.method, parked.subchannel, parked.method_count, + parked.resume_address); + } + } + parked_methods.clear(); dma_pushbuffer.pop(); dma_pushbuffer_subindex = 0; } else { @@ -111,20 +185,23 @@ bool DmaPusher::Step() { return true; } -void DmaPusher::ProcessCommands(std::span<const CommandHeader> commands) { +size_t DmaPusher::ProcessCommands(std::span<const CommandHeader> commands, size_t word_offset) { for (size_t index = 0; index < commands.size();) { // Data word of methods command if (dma_state.method_count && dma_state.non_incrementing) { auto const& command_header = commands[index]; //must ref (MUltiMethod re) - dma_state.dma_word_offset = u32(index * sizeof(u32)); - const u32 max_write = u32(std::min<std::size_t>(index + dma_state.method_count, commands.size()) - index); + dma_state.dma_word_offset = u64((word_offset + index) * sizeof(u32)); + u32 max_write = u32(std::min<std::size_t>(index + dma_state.method_count, commands.size()) - index); + if (g_nx2_ampere && dma_state.method < non_puller_methods) { + max_write = 1; // a semaphore acquire may suspend the channel after any word + } CallMultiMethod(&command_header.argument, max_write); dma_state.method_count -= max_write; dma_state.is_last_call = true; index += max_write; } else if (dma_state.method_count) { auto const command_header = commands[index]; //can copy - dma_state.dma_word_offset = u32(index * sizeof(u32)); + dma_state.dma_word_offset = u64((word_offset + index) * sizeof(u32)); dma_state.is_last_call = dma_state.method_count <= 1; CallMethod(command_header.argument); dma_state.method += !dma_state.non_incrementing ? 1 : 0; @@ -163,6 +240,54 @@ void DmaPusher::ProcessCommands(std::span<const CommandHeader> commands) { } index++; } + if (puller.HasPendingAcquire()) { + return index; + } + } + return commands.size(); +} + +// NVN on Switch 2 ends GPFIFO entries in the middle of a method's data and calls another command +// buffer (e.g. a compute dispatch) before the entry that carries the rest, which starts right +// where the first one ended. An entry starting anywhere else is such a call: run it with a clean +// method state and resume the unfinished method when the stream continues at that address. +void DmaPusher::ResolveSplitMethod(GPUVAddr entry_address) { + const auto continued = std::ranges::find(parked_methods, entry_address, + &ParkedMethod::resume_address); + std::optional<ParkedMethod> resume; + if (continued != parked_methods.end()) { + resume = *continued; + parked_methods.erase(continued); + } + if (dma_state.method_count != 0 && entry_address != split_method_end) { + if (parked_methods.size() == MaxParkedMethods) { + static std::atomic<u32> reported{}; + if (reported.fetch_add(1) < 8) { + const auto& oldest = parked_methods.front(); + LOG_WARNING(HW_GPU, + "Dropping parked method {:#x} (sub {}, {} words left, resume {:#x})", + oldest.method, oldest.subchannel, oldest.method_count, + oldest.resume_address); + } + parked_methods.erase(parked_methods.begin()); + } + parked_methods.push_back(ParkedMethod{ + .method = dma_state.method, + .subchannel = dma_state.subchannel, + .method_count = dma_state.method_count, + .non_incrementing = dma_state.non_incrementing, + .increment_once = dma_increment_once, + .resume_address = split_method_end, + }); + dma_state.method_count = 0; + dma_increment_once = false; + } + if (resume && dma_state.method_count == 0) { + dma_state.method = resume->method; + dma_state.subchannel = resume->subchannel; + dma_state.method_count = resume->method_count; + dma_state.non_incrementing = resume->non_incrementing; + dma_increment_once = resume->increment_once; } } @@ -172,7 +297,28 @@ void DmaPusher::SetState(const CommandHeader& command_header) { dma_state.method_count = command_header.method_count; } +// EDEN_NX2_SEM_WATCH=<gpu address>: log every method whose argument is the low word of an +// address in the 16 bytes there (finds the engine and method that release a semaphore). +static const u32 g_semaphore_watch = [] { + const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); + return value ? static_cast<u32>(std::strtoull(value, nullptr, 16)) : 0U; +}(); + +// EDEN_NX2_PB_CHECK also keeps the last parsed entries, for DumpCurrentEntry. +static const bool g_record_parsed_entries = std::getenv("EDEN_NX2_PB_CHECK") != nullptr; + +void DmaPusher::WatchArgument(u32 method, u32 argument) const { + const u32 watch = g_semaphore_watch; + if ((argument & ~0xfU) == watch) { + LOG_INFO(HW_GPU, "Semaphore watch: channel {} subchannel {} method {:#x} argument {:#x}", + channel_state.bind_id, dma_state.subchannel, method, argument); + } +} + void DmaPusher::CallMethod(u32 argument) { + if (g_semaphore_watch != 0) [[unlikely]] { + WatchArgument(dma_state.method, argument); + } if (dma_state.method < non_puller_methods) { puller.CallPullerMethod(*this, Engines::Puller::MethodCall{ dma_state.method, @@ -182,6 +328,10 @@ void DmaPusher::CallMethod(u32 argument) { }); } else { auto subchannel = subchannels[dma_state.subchannel]; + if (subchannel == nullptr) { + ReportUnboundSubchannel(); + return; + } if (!subchannel->execution_mask[dma_state.method]) { subchannel->method_sink.emplace_back(dma_state.method, argument); } else { @@ -193,16 +343,128 @@ void DmaPusher::CallMethod(u32 argument) { } void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { + if (g_semaphore_watch != 0) [[unlikely]] { + for (u32 i = 0; i < num_methods; ++i) { + WatchArgument(dma_state.method + (dma_state.non_incrementing ? 0 : i), base_start[i]); + } + } if (dma_state.method < non_puller_methods) { puller.CallMultiMethod(*this, dma_state.method, dma_state.subchannel, base_start, num_methods, dma_state.method_count); } else { auto subchannel = subchannels[dma_state.subchannel]; + if (subchannel == nullptr) { + ReportUnboundSubchannel(); + return; + } subchannel->ConsumeSink(system); subchannel->current_dma_segment = dma_state.dma_get + dma_state.dma_word_offset; subchannel->CallMultiMethod(system, dma_state.method, base_start, num_methods, dma_state.method_count); } } +void DmaPusher::ReportUnboundSubchannel() { + // Methods sent to a subchannel with no engine (no SetObject yet) are dropped. + static std::atomic<u32> reported{}; + if (reported.fetch_add(1) < 8) { + LOG_WARNING(HW_GPU, "Dropping method {:#x} on unbound subchannel {}", dma_state.method, + dma_state.subchannel); + } +} + +void DmaPusher::CheckSubmitHash(const CommandListHeader& header, u64 submit_hash) { + std::vector<u32> words(header.size); + memory_manager.ReadBlockUnsafe(header.addr, words.data(), words.size() * sizeof(u32)); + static std::atomic<u64> checked{}; + static std::atomic<u64> changed{}; + ++checked; + if (HashPushbufferWords(words) != submit_hash) { + if (changed.fetch_add(1) < 16) { + LOG_WARNING(HW_GPU, "pushbuffer entry {:#x}+{:#x} changed between submit and execution", + header.addr.Value(), header.size.Value()); + } + } + if ((checked % 20000) == 0) { + LOG_WARNING(HW_GPU, "pushbuffer check: {} of {} entries changed after submit", + changed.load(), checked.load()); + } +} + +void DmaPusher::RecordParsedEntry(const CommandListHeader& header, + std::span<const CommandHeader> commands) { + if (!g_record_parsed_entries) { + return; + } + u64 hash = 0xcbf29ce484222325ULL; + for (const CommandHeader& command : commands) { + hash = (hash ^ command.argument) * 0x100000001b3ULL; + } + parsed_entries[parsed_entry_count++ % parsed_entries.size()] = {header.addr, static_cast<u32>(header.size), hash}; +} + +void DmaPusher::DumpCurrentEntry(const char* reason) { + // Entries whose memory changed after they were parsed (the CPU rewrote them meanwhile). + for (size_t i = 0; i < std::min(parsed_entry_count, parsed_entries.size()); ++i) { + const auto& entry = parsed_entries[(parsed_entry_count - 1 - i) % parsed_entries.size()]; + std::vector<u32> words(entry.size); + memory_manager.ReadBlockUnsafe(entry.address, words.data(), words.size() * sizeof(u32)); + u64 hash = 0xcbf29ce484222325ULL; + for (const u32 word : words) { + hash = (hash ^ word) * 0x100000001b3ULL; + } + LOG_WARNING(HW_GPU, " parsed -{}: {:#x}+{:#x} {}", i, entry.address, entry.size, + hash == entry.hash ? "unchanged" : "CHANGED since parsed"); + } + // EDEN_NX2_PB_DUMP=<file>: also write the whole command list (u64 address, u32 size and the + // current words of every entry, then the current entry index) for offline replay. + if (static const char* const path = std::getenv("EDEN_NX2_PB_DUMP"); + path != nullptr && !dma_pushbuffer.empty()) { + if (std::FILE* file = std::fopen(path, "wb")) { + const auto& lists = dma_pushbuffer.front().command_lists; + const u64 current = dma_pushbuffer_subindex; + const u64 count = lists.size(); + std::fwrite(&count, sizeof(count), 1, file); + std::fwrite(¤t, sizeof(current), 1, file); + for (const CommandListHeader& header : lists) { + const u64 address = header.addr; + const u32 size = static_cast<u32>(header.size); + std::vector<u32> words(size); + memory_manager.ReadBlockUnsafe(address, words.data(), words.size() * sizeof(u32)); + std::fwrite(&address, sizeof(address), 1, file); + std::fwrite(&size, sizeof(size), 1, file); + std::fwrite(words.data(), sizeof(u32), words.size(), file); + } + std::fclose(file); + } + } + LOG_WARNING(HW_GPU, "pushbuffer dump ({}): method={:#x} sub={} count={} non_inc={} get={:#x}", + reason, dma_state.method, dma_state.subchannel, dma_state.method_count, + dma_state.non_incrementing, dma_state.dma_get); + if (dma_pushbuffer.empty()) { + return; + } + const auto& lists = dma_pushbuffer.front().command_lists; + const size_t first = dma_pushbuffer_subindex >= 2 ? dma_pushbuffer_subindex - 2 : 0; + for (size_t i = first; i <= dma_pushbuffer_subindex && i < lists.size(); ++i) { + const CommandListHeader& header = lists[i]; + LOG_WARNING(HW_GPU, " entry {} of {}: addr={:#x} size={:#x}", i, lists.size(), + header.addr.Value(), header.size.Value()); + for (u64 word = 0; word < std::min<u64>(header.size, 0x80); word += 8) { + std::string gpu_line; + std::string cpu_line; + for (u64 j = word; j < std::min<u64>(header.size, word + 8); ++j) { + const GPUVAddr address = header.addr + j * sizeof(u32); + const u32 gpu_value = memory_manager.Read<u32>(address); + const u32 cpu_value = system.ApplicationMemory().Read32(address); + gpu_line += fmt::format(" {:08x}", gpu_value); + cpu_line += gpu_value == cpu_value ? std::string(" ........") + : fmt::format(" {:08x}", cpu_value); + } + LOG_WARNING(HW_GPU, " +{:03x}:{}", word * 4, gpu_line); + LOG_WARNING(HW_GPU, " cpu :{}", cpu_line); + } + } +} + void DmaPusher::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { rasterizer = rasterizer_; } diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 62af628511..bbcc345c84 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -8,6 +8,7 @@ #include <array> #include <condition_variable> +#include <optional> #include <span> #include <vector> #include <boost/container/small_vector.hpp> @@ -114,8 +115,19 @@ struct CommandList final { boost::container::small_vector<CommandListHeader, 512> command_lists; boost::container::small_vector<CommandHeader, 512> prefetch_command_list; + /// EDEN_NX2_PB_CHECK: hash of each entry's words when it was submitted. + std::vector<u64> submit_hashes; }; +/// FNV-1a over pushbuffer words (EDEN_NX2_PB_CHECK). +inline u64 HashPushbufferWords(std::span<const u32> words) { + u64 hash = 0xcbf29ce484222325ULL; + for (const u32 word : words) { + hash = (hash ^ word) * 0x100000001b3ULL; + } + return hash; +} + /// @brief The DmaPusher class implements DMA submission to FIFOs, providing an area of memory that the /// emulated app fills with commands and tells PFIFO to process. The pushbuffers are then assembled /// into a "command stream" consisting of 32-bit words that make up "commands". @@ -132,6 +144,14 @@ public: void DispatchCalls(); + /// Switch 2: true while the channel is suspended on a semaphore acquire. + [[nodiscard]] bool HasPendingAcquire() const { + return puller.HasPendingAcquire(); + } + [[nodiscard]] bool CanResumeAcquire() { + return puller.CanResumeAcquire(*this); + } + void BindSubchannel(Engines::EngineInterface* engine, u32 subchannel_id, Engines::EngineTypes engine_type) { subchannels[subchannel_id] = engine; subchannel_type[subchannel_id] = engine_type; @@ -143,12 +163,26 @@ private: static constexpr u32 non_puller_methods = 0x40; static constexpr u32 max_subchannels = 8; bool Step(); - void ProcessCommands(std::span<const CommandHeader> commands); + /// Returns how many words were processed: fewer than given when a semaphore acquire + /// suspended the channel. + size_t ProcessCommands(std::span<const CommandHeader> commands, size_t word_offset = 0); + size_t command_resume_offset{}; ///< Words of the current entry done before a suspension void SetState(const CommandHeader& command_header); void CallMethod(u32 argument); void CallMultiMethod(const u32* base_start, u32 num_methods); + void ReportUnboundSubchannel(); + void WatchArgument(u32 method, u32 argument) const; + +public: + /// Logs the GPFIFO entry being processed (and the one before it), read through the GPU MMU + /// and straight from guest memory, to find pushbuffer desyncs. Debug aid for Switch 2. + void DumpCurrentEntry(const char* reason); + void CheckSubmitHash(const CommandListHeader& header, u64 submit_hash); + void RecordParsedEntry(const CommandListHeader& header, std::span<const CommandHeader> commands); + +private: public: Common::ScratchBuffer<CommandHeader> command_headers; ///< Buffer for list of commands fetched at once @@ -172,6 +206,31 @@ public: Control::ChannelState& channel_state; DmaState dma_state{}; + + // Switch 2: methods whose data continues in a later GPFIFO entry, set aside while other + // command buffers run (NVN interleaves several, so more than one can be pending). + struct ParkedMethod { + u32 method; + u32 subchannel; + u32 method_count; + bool non_incrementing; + bool increment_once; + GPUVAddr resume_address; ///< Where the entry that continues the method starts + }; + void ResolveSplitMethod(GPUVAddr entry_address); + static constexpr size_t MaxParkedMethods = 16; + + // Debug aid: hashes of the words of the last entries as they were parsed. + struct ParsedEntry { + GPUVAddr address; + u32 size; + u64 hash; + }; + std::array<ParsedEntry, 64> parsed_entries{}; + size_t parsed_entry_count{}; + std::vector<ParkedMethod> parked_methods; + GPUVAddr split_method_end{}; ///< End of the last entry that ended inside a method + std::array<Engines::EngineInterface*, max_subchannels> subchannels{}; std::array<Engines::EngineTypes, max_subchannels> subchannel_type; diff --git a/src/video_core/engines/engine_upload.cpp b/src/video_core/engines/engine_upload.cpp index ba21eec4c4..0ffdae8036 100644 --- a/src/video_core/engines/engine_upload.cpp +++ b/src/video_core/engines/engine_upload.cpp @@ -27,6 +27,7 @@ void State::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { void State::ProcessExec(const bool is_linear_) { write_offset = 0; + flushed_offset = 0; copy_size = regs.line_length_in * regs.line_count; inner_buffer.resize_destructive(copy_size); is_linear = is_linear_; @@ -39,10 +40,29 @@ void State::ProcessData(const u32 data, const bool is_last_call) { if (!is_last_call) { return; } + if (native_partial && is_linear && regs.line_count == 1) { + const u32 count = write_offset - flushed_offset; + if (count) { + rasterizer->AccelerateInlineToMemory(regs.dest.Address() + flushed_offset, count, + std::span<const u8>{inner_buffer.data() + flushed_offset, count}); + flushed_offset = write_offset; + } + return; + } ProcessData(inner_buffer); } void State::ProcessData(const u32* data, size_t num_data) { + if (native_partial && is_linear && regs.line_count == 1) { + const auto count = std::min<size_t>(num_data * sizeof(u32), copy_size - write_offset); + if (count) { + rasterizer->AccelerateInlineToMemory(regs.dest.Address() + write_offset, count, + std::span<const u8>{reinterpret_cast<const u8*>(data), count}); + write_offset += static_cast<u32>(count); + flushed_offset = write_offset; + } + return; + } std::span<const u8> read_buffer(reinterpret_cast<const u8*>(data), num_data * sizeof(u32)); ProcessData(read_buffer); } diff --git a/src/video_core/engines/engine_upload.h b/src/video_core/engines/engine_upload.h index 21bf8aeb46..55a82f938b 100644 --- a/src/video_core/engines/engine_upload.h +++ b/src/video_core/engines/engine_upload.h @@ -62,6 +62,7 @@ public: explicit State(MemoryManager& memory_manager_, Registers& regs_); ~State(); + void SetNativePartialTransfers(bool enabled) { native_partial = enabled; } void ProcessExec(bool is_linear_); void ProcessData(u32 data, bool is_last_call); void ProcessData(const u32* data, size_t num_data); @@ -80,6 +81,8 @@ public: private: void ProcessData(std::span<const u8> read_buffer); + bool native_partial{}; + u32 flushed_offset{}; u32 write_offset = 0; u32 copy_size = 0; Common::ScratchBuffer<u8> inner_buffer; diff --git a/src/video_core/engines/fermi_2d.cpp b/src/video_core/engines/fermi_2d.cpp index aeff5e156f..35b2c69801 100644 --- a/src/video_core/engines/fermi_2d.cpp +++ b/src/video_core/engines/fermi_2d.cpp @@ -4,6 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> #include "common/assert.h" #include "common/logging.h" #include "video_core/engines/fermi_2d.h" @@ -59,6 +60,27 @@ void Fermi2D::ConsumeSinkImpl(Core::System& system) { } void Fermi2D::Blit() { + if (static const u64 trace_destination = [] { + const char* value = std::getenv("EDEN_NX2_TRACE_DST"); + return value != nullptr ? std::strtoull(value, nullptr, 16) : 0ULL; + }(); + trace_destination != 0 && regs.dst.Address() >= trace_destination && + regs.dst.Address() < trace_destination + (32ULL << 20)) { + static u32 traced_destination{}; + if (traced_destination++ < 40) { + LOG_INFO(HW_GPU, "nx2 2D blit to traced dst: {:#x} {}x{} -> {:#x} {}x{}", + regs.src.Address(), regs.src.width, regs.src.height, regs.dst.Address(), + regs.dst.width, regs.dst.height); + } + } + if (static const bool trace = std::getenv("EDEN_NX2_RT_TRACE") != nullptr; trace) { + static u32 traced{}; + if (traced++ < 64) { + LOG_INFO(HW_GPU, "nx2 2D blit {:#x} {}x{} -> {:#x} {}x{}", regs.src.Address(), + regs.src.width, regs.src.height, regs.dst.Address(), regs.dst.width, + regs.dst.height); + } + } LOG_DEBUG(HW_GPU, "called. source address={:#x}, destination address={:#x}", regs.src.Address(), regs.dst.Address()); UNIMPLEMENTED_IF_MSG(regs.operation != Operation::SrcCopy, "Operation is not copy"); diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 96a12539a2..7792bc9b5d 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -4,13 +4,25 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <algorithm> +#include <atomic> #include <bitset> +#include <cstdlib> +#include <fstream> +#include <optional> +#include <string> +#include <unordered_map> +#include <unordered_set> #include "common/assert.h" #include "common/logging.h" +#include "shader_recompiler/exception.h" #include "core/core.h" +#include "video_core/nx2.h" +#include "video_core/engines/fermi_2d.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/memory_manager.h" +#include "video_core/native_query_trace.h" #include "video_core/rasterizer_interface.h" #include "video_core/textures/decoders.h" @@ -20,6 +32,13 @@ KeplerCompute::KeplerCompute(MemoryManager& memory_manager_) : memory_manager{memory_manager_} , upload_state{memory_manager, regs.upload} { + // EDEN_NX2_QMD_TRACE: log the QMD scheduler's loads, dependency counts and launches. + if (std::getenv("EDEN_NX2_QMD_TRACE")) { + native_qmd.trace = [this](std::string_view event, u64 address, s32 value) { + LOG_INFO(HW_GPU, "QMD trace {} {:#x} {} (engine {} as {})", event, address, value, + fmt::ptr(this), memory_manager.GetID()); + }; + } execution_mask.reset(); execution_mask[KEPLER_COMPUTE_REG_INDEX(exec_upload)] = true; execution_mask[KEPLER_COMPUTE_REG_INDEX(data_upload)] = true; @@ -35,13 +54,347 @@ void KeplerCompute::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) void KeplerCompute::ConsumeSinkImpl(Core::System& system) { for (auto [method, value] : method_sink) { - regs.reg_array[method] = value; + if (method < Regs::NUM_REGS) { + regs.reg_array[method] = value; + } } method_sink.clear(); } +// SET_REPORT_SEMAPHORE_A-D (0x6c0-0x6c3): address high/low, payload, then operation in bits 1:0 +// (release / acquire / report only / trap), reduction enable bit 3 with the op in bits 11:9, and +// a one-word (no timestamp) structure in bit 28. NVN2 releases queue syncs from compute this way. +void KeplerCompute::ProcessNativeReportSemaphore(u32 argument) { + const GPUVAddr address = (GPUVAddr{native_registers[0x6c0] & 0x1ffffff} << 32) | + native_registers[0x6c1]; + const u32 payload = native_registers[0x6c2]; + const u32 operation = argument & 3; + const bool one_word = (argument >> 28) & 1; + static u32 logged{}; + if (logged++ < 32) { + LOG_INFO(HW_GPU, "Native compute report semaphore {:#x} = {:#x}, control {:#x}", address, + payload, argument); + } + if (operation != 0 && operation != 2) { + LOG_WARNING(HW_GPU, "Native compute report semaphore operation {} not supported", operation); + return; + } + if ((argument >> 3) & 1) { + const u32 reduction = (argument >> 9) & 7; + rasterizer->SignalFence([this, address, payload, reduction] { + const u32 old_value = memory_manager.Read<u32>(address); + const u32 value = reduction == 0 ? old_value + payload + : reduction == 1 ? std::min(old_value, payload) + : reduction == 2 ? std::max(old_value, payload) + : reduction == 3 ? (old_value >= payload ? 0 : old_value + 1) + : reduction == 4 ? (old_value == 0 || old_value > payload ? payload : old_value - 1) + : reduction == 5 ? (old_value & payload) + : reduction == 6 ? (old_value | payload) + : (old_value ^ payload); + memory_manager.Write<u32>(address, value); + }); + return; + } + rasterizer->Query(address, VideoCommon::QueryType::Payload, + one_word ? VideoCommon::QueryPropertiesFlags::IsAFence + : VideoCommon::QueryPropertiesFlags::HasTimeout, + payload, 0); +} + +bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call) { + if (native_context) { + if (method == 0x45 || method == 0x47) native_compute_uploaded = true; + native_macro = native_compute_uploaded ? &native_context->compute_engine : &native_context->engine; + native_macro_positions = native_compute_uploaded ? std::span<u32>{native_context->compute_positions} : std::span<u32>{native_context->positions}; + } + if (method >= native_registers.size()) { + LOG_ERROR(HW_GPU, "Native compute method out of range: {:#x}", method); + return true; + } + if (method >= 0xe00) { + if (!native_executing_macro) { + if (method & 1) { LOG_ERROR(HW_GPU, "Native compute macro starts on argument method"); return true; } + native_executing_macro = method; + } + native_parameters.push_back(argument); + if (!is_last_call) return true; + const u32 entry = ((*native_executing_macro - 0xe00) / 2) % native_macro_positions.size(); + native_executing_macro_logged = native_executing_macro; + native_executing_macro.reset(); + static const u32 watch = [] { + const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); + return value ? static_cast<u32>(std::strtoull(value, nullptr, 16)) : 0U; + }(); + const bool watched = watch != 0 && std::ranges::any_of(native_parameters, + [](u32 parameter) { return (parameter & ~0xfU) == watch; }); + if (watched) { + LOG_INFO(HW_GPU, "Semaphore watch: compute macro start={:#x} parameters={:08x}", + native_macro_positions[entry], fmt::join(native_parameters, ",")); + } + // EDEN_NX2_MME_DUMP: write the compute macro code once per engine, and log each macro + // call with its entry and parameters (diagnostics). + if (static const bool mme_dump = std::getenv("EDEN_NX2_MME_DUMP") != nullptr; mme_dump) { + if (!native_mme_dumped && native_macro_positions[entry] == 0xda) { + native_mme_dumped = true; + std::ofstream dump(fmt::format("native-compute-mme-{}.bin", fmt::ptr(this)), + std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_macro->native_code.data()), + sizeof(native_macro->native_code)); + } + LOG_INFO(HW_GPU, "MME call engine {} method {:#x} start={:#x} parameters={:08x}", + fmt::ptr(this), *native_executing_macro_logged, native_macro_positions[entry], + fmt::join(native_parameters, ",")); + } + if (native_data_ram) native_macro->native_interpreter.active_dram = native_data_ram; + const u32 dram0_before = (*native_macro->native_interpreter.active_dram)[0x27]; + const u32 start = native_macro_positions[entry]; + const auto emit = [&](u32 address, u32 value) { + static u32 diagnostic_writes{}; + static const bool log_emits = std::getenv("EDEN_NX2_SEM_WATCH_EMITS") != nullptr; + if (watched && log_emits) LOG_INFO(HW_GPU, "Semaphore watch: compute macro emit {:#x} = {:#x}", address, value); + if (diagnostic_writes++ < 4096) LOG_DEBUG(HW_GPU, "Native compute MME emit method={:#x}, value={:#x}", address, value); + if (address >= 0xe00) return false; + CallMethod(system, address, value, true); + return true; + }; + // Repeated calls replay their writes, as for 3D macros (macro_memo.h). + static const bool use_memo = [] { + const char* value = std::getenv("EDEN_NX2_MME_MEMO"); + return value == nullptr || value[0] != '0'; + }(); + auto& memo = native_macro->native_memo; + if (use_memo && !watched) { + memo.Synchronize(native_macro->native_code_generation); + if (std::remove_reference_t<decltype(memo)>::Effects effects; + memo.Find(start, native_parameters, *native_macro->native_interpreter.active_dram, + effects)) { + auto& data_ram = *native_macro->native_interpreter.active_dram; + for (size_t i = 0; i < effects.data_ram_writes.size(); i += 2) { + data_ram[effects.data_ram_writes[i]] = effects.data_ram_writes[i + 1]; + } + for (size_t i = 0; i < effects.writes.size(); i += 2) { + emit(effects.writes[i], effects.writes[i + 1]); + } + native_parameters.clear(); + return true; + } + native_macro->native_writes.clear(); + } + // The interpreter consumes native_parameters as its FIFO: keep the call's own parameters. + native_memo_parameters.assign(native_parameters.begin(), native_parameters.end()); + const u64 code_generation = native_macro->native_code_generation; + const auto result = native_macro->native_interpreter.Execute(native_macro->native_code, + start, native_parameters, + [&](u32 address) { return address < native_registers.size() ? native_registers[address] : 0U; }, + [&](u32 address, u32 value) { + if (use_memo) { + native_macro->native_writes.emplace_back(address, value); + } + return emit(address, value); + }); + const auto& interpreter = native_macro->native_interpreter; + if (use_memo && !watched && result.success && !interpreter.read_registers && + !interpreter.reread_data_ram && code_generation == native_macro->native_code_generation) { + memo.Store(start, native_memo_parameters, interpreter.data_ram_reads, + interpreter.data_ram_writes, native_macro->native_writes); + } + if (static const bool mme_dump = std::getenv("EDEN_NX2_MME_DUMP") != nullptr; + mme_dump && (*native_macro->native_interpreter.active_dram)[0x27] != dram0_before) { + LOG_INFO(HW_GPU, "MME data RAM[0x27] {:#x} -> {:#x} by engine {} macro start={:#x}", + dram0_before, (*native_macro->native_interpreter.active_dram)[0x27], fmt::ptr(this), + native_macro_positions[entry]); + } + if (!result.success) { + native_macro->native_execution_failed = true; + if (static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr; + diagnostics) { + std::ofstream dump("native-compute-mme.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_macro->native_code.data()), sizeof(native_macro->native_code)); + } + LOG_ERROR(HW_GPU, "Native MME failed: compute start={:#x}, pc={:#x}, consumed={}/{}, error={}, parameters={:08x}", + native_macro_positions[entry], result.pc, result.consumed, native_parameters.size(), result.error, + fmt::join(native_parameters, ",")); + } + native_parameters.clear(); + return true; + } + native_registers[method] = argument; + if (method == 0x6c3) { // SET_REPORT_SEMAPHORE_D: the compute class's semaphore release + ProcessNativeReportSemaphore(argument); + return true; + } + if (method == 0x5a || (method >= 0xc6 && method <= 0x108)) { + VideoCommon::TraceNativeQuery(12, memory_manager.GetID(), method, 0, 0, argument, 0); + } + if (method == 0x107) { + const GPUVAddr address = ((GPUVAddr{native_registers[0xc6]} << 32) | + native_registers[0xc7]) << 8; + const GPUVAddr program = (GPUVAddr{native_registers[0xc8 + 49] & 0x1ffff} << 32) | + native_registers[0xc8 + 48]; + const u32 control = native_registers[0xa7]; + LOG_DEBUG(HW_GPU, "Native inline QMD: address={:#x}, program={:#x}, control={:#x}, grid={}/{}/{}", + address, program, control, native_registers[0xc8 + 12], + native_registers[0xc8 + 13] & 0xffff, native_registers[0xc8 + 14] & 0xffff); + if (static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr; + diagnostics) { + std::ofstream dump("native-inline-qmd.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_registers.data() + 0xc8), 256); + std::ofstream registers("native-inline-compute-registers.bin", std::ios::binary); + registers.write(reinterpret_cast<const char*>(native_registers.data()), sizeof(native_registers)); + if (memory_manager.IsFullyMappedRange(program, 0x10000)) { + std::vector<u8> code(0x10000); + memory_manager.ReadBlockUnsafe(program, code.data(), code.size()); + std::ofstream shader("native-inline-compute-shader.bin", std::ios::binary); + shader.write(reinterpret_cast<const char*>(code.data()), code.size()); + } + } + if (!memory_manager.IsFullyMappedRange(address, 256)) { + native_macro->native_execution_failed = true; + LOG_ERROR(HW_GPU, "Unmapped native inline QMD destination"); + return true; + } + rasterizer->AccelerateInlineToMemory(address, 256, + std::span<const u8>{reinterpret_cast<const u8*>(native_registers.data() + 0xc8), 256}); + if ((control & (1U << 24)) || !(native_registers[0xcc] & (1U << 12))) { + const u32 previous = native_registers[0xad]; + native_registers[0xad] = static_cast<u32>(address >> 8); + ProcessNativeMethod(system, 0xb0, 3, true); + native_registers[0xad] = previous; + } + return true; + } + switch (method) { + case 0x45: native_macro->ClearCode(argument); return true; + case 0x46: native_macro->AddCode(0, argument); return true; + case 0x47: native_bind_pointer = argument; return true; + case 0x48: + if (native_bind_pointer < native_macro_positions.size()) native_macro_positions[native_bind_pointer++] = argument; + else LOG_ERROR(HW_GPU, "Native compute macro bind out of bounds"); + return true; + case 0xae: case 0xaf: case 0xb0: { + const GPUVAddr qmd_address = GPUVAddr{native_registers[0xad]} << 8; + VideoCommon::TraceNativeQuery(9, memory_manager.GetID(), qmd_address, + memory_manager.GpuToCpuAddress(qmd_address).value_or(0), method, argument, 0); + if (method == 0xae || (method == 0xaf && !(argument & 2)) || + (method == 0xb0 && ((argument & 15) == 0 || (argument & 15) == 1))) return true; + static const bool qmd_diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr; + if ((argument & 15) == 12 && qmd_diagnostics) { + static u32 captured{}; + if (captured++ == 0 && memory_manager.IsFullyMappedRange(qmd_address, 256)) { + std::array<u32, 64> descriptor{}; + memory_manager.ReadBlockUnsafe(qmd_address, descriptor.data(), sizeof(descriptor)); + std::ofstream dump("native-increment-qmd.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(descriptor.data()), sizeof(descriptor)); + LOG_WARNING(HW_GPU, "Native increment dependency descriptor: address={:#x}, words={:08x}", + qmd_address, fmt::join(descriptor, ",")); + } + } + const auto error = native_qmd.Signal(qmd_address, argument, + [&](u64 address, NativeQmdScheduler::Descriptor& descriptor) { + if (!memory_manager.IsFullyMappedRange(address, sizeof(descriptor))) return false; + memory_manager.ReadBlockUnsafe(address, descriptor.data(), sizeof(descriptor)); + if (static u32 logged{}; (descriptor[53] & (1U << 23)) && logged < 8) { + ++logged; + LOG_INFO(HW_GPU, "Native QMD {:#x} releases: {:08x} | {:08x} | {:08x}", address, + fmt::join(std::span{descriptor}.subspan(24, 4), ","), + fmt::join(std::span{descriptor}.subspan(28, 4), ","), + fmt::join(std::span{descriptor}.subspan(52, 4), ",")); + } + for (u32 slot = 0; slot < 2; ++slot) { + const u32 flags = descriptor[25 + slot * 4]; + if (!(flags & (1U << 23))) continue; + const GPUVAddr destination = (GPUVAddr{flags & 0xff} << 32) | descriptor[24 + slot * 4]; + VideoCommon::TraceNativeQuery(7 + slot, memory_manager.GetID(), destination, + memory_manager.GpuToCpuAddress(destination).value_or(0), + descriptor[4], descriptor[26 + slot * 4], flags); + } + return true; + }, + [&](u64 address, u32 payload, u32 flags) { + if (!memory_manager.IsFullyMappedRange(address, sizeof(payload))) return false; + if (flags & (1U << 26)) { + rasterizer->SignalFence([this, address, payload] { + const u32 value = memory_manager.Read<u32>(address) + payload; + memory_manager.Write<u32>(address, value); + if (const auto device = memory_manager.GpuToCpuAddress(address)) { + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(*device, value); + } + static const u64 watch = [] { + const char* text = std::getenv("EDEN_NX2_SEM_WATCH"); + return text ? u64{std::strtoull(text, nullptr, 16)} : u64{}; + }(); + if (watch != 0 && (address & ~u64{0xf}) == watch) { + LOG_INFO(HW_GPU, "Semaphore watch: QMD release add {:#x} -> {:#x}", address, value); + } + VideoCommon::TraceNativeQuery(1, memory_manager.GetID(), address, + memory_manager.GpuToCpuAddress(address).value_or(0), 0, value, 1U << 26); + }); + } else if (flags & (1U << 19)) { + // NVN2 tracks QMD completion with a max release (bit 19) into a word other + // releases also advance: a plain write would move it backwards and leave + // GEQ acquires waiting forever. + rasterizer->SignalFence([this, address, payload] { + const u32 value = std::max(memory_manager.Read<u32>(address), payload); + memory_manager.Write<u32>(address, value); + if (const auto device = memory_manager.GpuToCpuAddress(address)) { + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(*device, value); + } + }); + } else { + static const u64 watch = [] { + const char* text = std::getenv("EDEN_NX2_SEM_WATCH"); + return text ? u64{std::strtoull(text, nullptr, 16)} : u64{}; + }(); + if (watch != 0 && (address & ~u64{0xf}) == (watch & ~u64{0xf})) { + LOG_INFO(HW_GPU, "Semaphore watch: QMD release {:#x} = {:#x} flags {:#x}", address, payload, flags); + } + rasterizer->Query(address, VideoCommon::QueryType::Payload, + VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); + } + return true; + }, [&](const auto& descriptor) { return DispatchNativeCompute(descriptor); }); + if (error.empty()) { + LOG_DEBUG(HW_GPU, "Native QMD control action={:#x}, QMD={:#x}", argument, qmd_address); + return true; + } + native_macro->native_execution_failed = true; + std::array<u32, 128> qmd{}; + if (memory_manager.IsFullyMappedRange(qmd_address, sizeof(qmd))) { + memory_manager.ReadBlockUnsafe(qmd_address, qmd.data(), sizeof(qmd)); + } + // Log a failing program's first few dispatches only (QMD words 48-49 hold its code + // address); it fails the same way every frame. + static std::unordered_map<u64, u32> failures; + const u64 program = (u64{qmd[49]} << 32) | qmd[48]; + if (failures[program ^ std::hash<std::string>{}(std::string{error})]++ >= 2) { + return true; + } + LOG_ERROR(HW_GPU, "Native QMD failed: {}", error); + if (qmd_diagnostics && memory_manager.IsFullyMappedRange(qmd_address, sizeof(qmd))) { + std::ofstream dump("native-qmd.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(qmd.data()), sizeof(qmd)); + std::ofstream macros("native-dispatch-mme.bin", std::ios::binary); + macros.write(reinterpret_cast<const char*>(native_macro->native_code.data()), sizeof(native_macro->native_code)); + std::ofstream registers("native-dispatch-registers.bin", std::ios::binary); + registers.write(reinterpret_cast<const char*>(native_registers.data()), sizeof(native_registers)); + } + LOG_ERROR(HW_GPU, "Native compute dispatch unsupported: method={:#x}, value={:#x}, QMD={:#x}, words={:08x}", method, argument, + qmd_address, fmt::join(qmd, ",")); + return true; + } + default: + // The inline upload register block is shared with the older class. + if (method < Regs::NUM_REGS) regs.reg_array[method] = argument; + return method != KEPLER_COMPUTE_REG_INDEX(exec_upload) && method != KEPLER_COMPUTE_REG_INDEX(data_upload); + } +} + void KeplerCompute::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) { - ASSERT_MSG(method < Regs::NUM_REGS, "Invalid KeplerCompute register, increase the size of the Regs structure"); + if (native_mode && ProcessNativeMethod(system, method, method_argument, is_last_call)) return; + if (method >= Regs::NUM_REGS) { + LOG_ERROR(HW_GPU, "Compute register out of range: {:#x}", method); + return; + } regs.reg_array[method] = method_argument; @@ -85,6 +438,10 @@ void KeplerCompute::CallMethod(Core::System& system, u32 method, u32 method_argu } void KeplerCompute::CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) { + if (native_mode) { + for (u32 i = 0; i < amount; ++i) CallMethod(system, method, base_start[i], methods_pending - i <= 1); + return; + } switch (method) { case KEPLER_COMPUTE_REG_INDEX(data_upload): upload_address = current_dma_segment; @@ -100,6 +457,223 @@ void KeplerCompute::CallMultiMethod(Core::System& system, u32 method, const u32* } } +bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& qmd) { + launch_description = {}; + launch_description.grid_dim_x.Assign(qmd[12]); + launch_description.grid_dim_y.Assign(qmd[13] & 0xffff); + launch_description.grid_dim_z.Assign(qmd[14] & 0xffff); + launch_description.shared_alloc.Assign(qmd[17] & 0x3ffff); + launch_description.block_dim_x.Assign(qmd[18] >> 16); + launch_description.block_dim_y.Assign(qmd[19] & 0xffff); + launch_description.block_dim_z.Assign(qmd[19] >> 16); + launch_description.const_buffer_enable_mask.Assign(qmd[20] & 0xff); + launch_description.local_pos_alloc.Assign(qmd[23] & 0xffffff); + launch_description.local_crs_alloc.Assign(qmd[50] & 0xffffff); + regs.code_loc.address_low = qmd[48]; + regs.code_loc.address_high = qmd[49] & 0x1ffff; + for (u32 slot = 0; slot < NumConstBuffers; ++slot) { + const u32 high = qmd[33 + slot * 2]; + if (high & 0x1ff00) { + LOG_ERROR(HW_GPU, "Native compute constant address exceeds supported GPU width"); + return false; + } + auto& buffer = launch_description.const_buffer_config[slot]; + buffer.address_low = qmd[32 + slot * 2]; + buffer.address_high.Assign(high & 0xff); + buffer.size.Assign(((high >> 19) & 0x1fff) << 4); + } + if (!launch_description.block_dim_x || !launch_description.block_dim_y || !launch_description.block_dim_z) + return false; + try { + rasterizer->DispatchCompute(); + } catch (const Shader::Exception& error) { + static std::unordered_set<std::string> logged; + if (logged.insert(error.what()).second) { + LOG_ERROR(HW_GPU, "Native compute shader rejected: {}", error.what()); + } + if (std::string_view{error.what()}.find("matrix multiply") != std::string_view::npos) { + DescribeNativeResources(qmd); + // The console runs this on tensor cores; stand in with a plain upscale of its input. + return UpscaleInsteadOfNativeProgram(); + } + return false; + } + return true; +} + +// Logs the images a rejected program's handles (constant bank 0, bytes 0x80-0x11c) point to, so a +// substitute (e.g. an upscale blit for a tensor-core pass) can find its inputs and output. +void KeplerCompute::DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd) { + static u32 described{}; + if (described++ >= 2) return; + const auto& cbuf = launch_description.const_buffer_config[0]; + const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low; + LOG_INFO(HW_GPU, "Native resources: cbuf0 {:#x} size {:#x}, grid {}x{}x{}, block {}x{}x{}, tic pool {:#x}", + cbuf_address, cbuf.size.Value(), qmd[12], qmd[13] & 0xffff, qmd[14] & 0xffff, + launch_description.block_dim_x.Value(), launch_description.block_dim_y.Value(), + launch_description.block_dim_z.Value(), regs.tic.Address()); + for (u32 offset = 0x80; offset < 0x120; offset += 4) { + const u32 handle = memory_manager.Read<u32>(cbuf_address + offset); + const u32 tic_index = handle & 0xfffff; + if (handle == 0 || tic_index > regs.tic.limit) continue; + const auto tic = GetTICEntry(tic_index); + LOG_INFO(HW_GPU, " handle @{:#x} = {:#x}: tic {} address {:#x} {}x{} format {:#x} types {}/{}/{}/{}", + offset, handle, tic_index, tic.Address(), tic.Width(), tic.Height(), + static_cast<u32>(tic.format.Value()), static_cast<u32>(tic.r_type.Value()), + static_cast<u32>(tic.g_type.Value()), static_cast<u32>(tic.b_type.Value()), + static_cast<u32>(tic.a_type.Value())); + } +} + +std::atomic<u32> g_nx2_trace_draws_after_upscale{}; + +// Title 4's upscaler reads the tonemapped frame through the texture handle at cbuf0+0x90 and +// writes the output resolution image through the surface handle at cbuf0+0xa0 (from its +// TLD/SUST operands). Blit one to the other with bilinear filtering when both are plain 2D images +// of the same format and the output is larger. +bool KeplerCompute::UpscaleInsteadOfNativeProgram() { + const auto& cbuf = launch_description.const_buffer_config[0]; + const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low; + const auto image = [&](u32 offset) -> std::optional<Texture::TICEntry> { + const u32 tic_index = memory_manager.Read<u32>(cbuf_address + offset) & 0xfffff; + if (tic_index > regs.tic.limit) return std::nullopt; + return GetTICEntry(tic_index); + }; + const auto input = image(0x90), output = image(0xa0); + if (!input || !output || input->format != output->format || input->Address() == 0 || + output->Address() == 0 || output->Width() <= input->Width() || !input->IsBlockLinear() || + !output->IsBlockLinear()) + return false; + using RTFormat = Tegra::RenderTargetFormat; + RTFormat format{}; + switch (input->format) { + case Texture::TextureFormat::A2B10G10R10: format = RTFormat::A2B10G10R10_UNORM; break; + case Texture::TextureFormat::A8B8G8R8: format = RTFormat::A8B8G8R8_UNORM; break; + case Texture::TextureFormat::B10G11R11: format = RTFormat::B10G11R11_FLOAT; break; + case Texture::TextureFormat::R16G16B16A16: format = RTFormat::R16G16B16A16_FLOAT; break; + default: return false; + } + const auto surface = [&](const Texture::TICEntry& tic) { + Fermi2D::Surface result{}; + result.format = format; + result.linear = Fermi2D::MemoryLayout::BlockLinear; + result.block_width.Assign(tic.block_width); + result.block_height.Assign(tic.block_height); + result.block_depth.Assign(tic.block_depth); + result.depth = 1; + result.width = tic.Width(); + result.height = tic.Height(); + result.addr_upper = static_cast<u32>(tic.Address() >> 32); + result.addr_lower = static_cast<u32>(tic.Address()); + return result; + }; + const Fermi2D::Config config{ + .operation = Fermi2D::Operation::SrcCopy, + .filter = Fermi2D::Filter::Bilinear, + .must_accelerate = true, + .dst_x0 = 0, + .dst_y0 = 0, + .dst_x1 = static_cast<s32>(output->Width()), + .dst_y1 = static_cast<s32>(output->Height()), + .src_x0 = 0, + .src_y0 = 0, + .src_x1 = static_cast<s32>(input->Width()), + .src_y1 = static_cast<s32>(input->Height()), + }; + // Debug: does the input hold anything? Download it once and count non-zero words. + static const u32 sample_at = [] { + const char* value = std::getenv("EDEN_NX2_SAMPLE_AT"); + return value ? static_cast<u32>(std::strtoul(value, nullptr, 10)) : 20U; + }(); + if (static u32 sampled{}; sampled++ == sample_at) { + const auto sample = [&](const Texture::TICEntry& tic) { + const u64 size = u64{tic.Width()} * tic.Height() * 4; + if (const auto device = memory_manager.GpuToCpuAddress(tic.Address())) { + rasterizer->FlushRegion(*device, size, VideoCommon::CacheType::All); + } + u32 nonzero{}, samples{}; + u64 sum{}; + for (u64 offset = 0; offset < size; offset += 4096 + 4) { + const u32 word = memory_manager.Read<u32>(tic.Address() + offset); + ++samples; + nonzero += word != 0; + sum += (word & 0x3ff) + ((word >> 10) & 0x3ff) + ((word >> 20) & 0x3ff); + } + LOG_INFO(HW_GPU, "Upscale fallback image {:#x}: {}/{} sampled words non-zero, mean channel {}", + tic.Address(), nonzero, samples, samples ? sum / (samples * 3) : 0); + }; + sample(*input); + sample(*output); + // Every 2560x1440 target the in-game passes draw to (from EDEN_NX2_RT_TRACE). + for (const GPUVAddr address : {0x8116a03000ULL, 0x8117ae5000ULL, 0x8114b62000ULL, + 0x8115a63000ULL, 0x8117904000ULL, 0x8114a02000ULL}) { + Texture::TICEntry fake = *input; + const u64 size = 2560ULL * 1440 * 4; + if (const auto device = memory_manager.GpuToCpuAddress(address)) { + rasterizer->FlushRegion(*device, size, VideoCommon::CacheType::All); + } + u32 nonzero{}, samples{}, distinct{}; + u32 last{}; + for (u64 offset = 0; offset < size; offset += 4096 + 4) { + const u32 word = memory_manager.Read<u32>(address + offset); + ++samples; + nonzero += word != 0; + distinct += word != last; + last = word; + } + LOG_INFO(HW_GPU, "Upscale fallback target {:#x}: {}/{} non-zero, {} changes", address, nonzero, + samples, distinct); + (void)fake; + } + } + auto source = surface(*input); + // EDEN_NX2_UPSCALE_SOURCE=<gpu address>: blit from another same-sized image (debug aid). + if (static const u64 override_source = [] { + const char* value = std::getenv("EDEN_NX2_UPSCALE_SOURCE"); + return value ? std::strtoull(value, nullptr, 16) : 0ULL; + }(); + override_source != 0) { + source.addr_upper = static_cast<u32>(override_source >> 32); + source.addr_lower = static_cast<u32>(override_source); + if (const char* value = std::getenv("EDEN_NX2_UPSCALE_SOURCE_FORMAT")) { + source.format = static_cast<RTFormat>(std::strtoul(value, nullptr, 16)); + } + } + const bool done = rasterizer->AccelerateSurfaceCopy(source, surface(*output), config); + if (static u32 checked{}; checked++ == sample_at) { + const u64 size = u64{output->Width()} * output->Height() * 4; + if (const auto device = memory_manager.GpuToCpuAddress(output->Address())) { + rasterizer->FlushRegion(*device, size, VideoCommon::CacheType::All); + } + u32 nonzero{}, samples{}, changes{}, last{}; + for (u64 offset = 0; offset < size; offset += 4096 + 4) { + const u32 word = memory_manager.Read<u32>(output->Address() + offset); + ++samples; + nonzero += word != 0; + changes += word != last; + last = word; + } + LOG_INFO(HW_GPU, "Upscale fallback output after blit: {}/{} non-zero, {} changes", nonzero, samples, + changes); + } + // Debug: trace the draws that follow one upscale (EDEN_NX2_RT_TRACE_AFTER_UPSCALE). + static const u32 trace_at = [] { + const char* value = std::getenv("EDEN_NX2_TRACE_AFTER_UPSCALE"); + return value ? static_cast<u32>(std::strtoul(value, nullptr, 10)) : 30U; + }(); + if (static u32 count{}; ++count == trace_at) g_nx2_trace_draws_after_upscale = 40; + static u32 logged{}; + if (logged++ < 4) { + LOG_INFO(HW_GPU, "Upscale fallback {:#x} {}x{} block {}/{}/{} -> {:#x} {}x{} block {}/{}/{} levels {}: {}", + input->Address(), input->Width(), input->Height(), input->block_width.Value(), + input->block_height.Value(), input->block_depth.Value(), output->Address(), + output->Width(), output->Height(), output->block_width.Value(), + output->block_height.Value(), output->block_depth.Value(), output->max_mip_level.Value(), + done ? "blitted" : "not accelerated"); + } + return done; +} + void KeplerCompute::ProcessLaunch() { const GPUVAddr launch_desc_loc = regs.launch_desc_loc.Address(); memory_manager.ReadBlockUnsafe(launch_desc_loc, &launch_description, diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index 078d3d45be..5ebf8edf38 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -7,6 +7,7 @@ #pragma once #include <array> +#include <atomic> #include <cstddef> #include <optional> #include <vector> @@ -15,6 +16,8 @@ #include "common/common_types.h" #include "video_core/engines/engine_interface.h" #include "video_core/engines/engine_upload.h" +#include "video_core/macro.h" +#include "video_core/native_qmd.h" #include "video_core/textures/texture.h" namespace Core { @@ -31,6 +34,9 @@ class RasterizerInterface; namespace Tegra::Engines { +/// Draws left to trace after an upscale fallback (debug aid). +extern std::atomic<u32> g_nx2_trace_draws_after_upscale; + /** * This Engine is known as GK104_Compute. Documentation can be found in: * https://github.com/envytools/envytools/blob/master/rnndb/graph/gk104_compute.xml @@ -51,6 +57,24 @@ public: /// Binds a rasterizer to this engine. void BindRasterizer(VideoCore::RasterizerInterface* rasterizer); + /// The channel's MME data RAM, shared with its 3D engine. + void SetNativeDataRam(std::array<u32, 0xc00>* data_ram) { native_data_ram = data_ram; } + void SetNativeMacroMode(MacroEngine* shared_macro, std::span<u32> positions, NativeMacroContext* context = nullptr) { + if (native_context != context) native_compute_uploaded = false; + native_context = context; + native_mode = shared_macro != nullptr; + upload_state.SetNativePartialTransfers(native_mode); + native_macro = shared_macro; + native_macro_positions = positions; + if (native_mode) execution_mask.set(); + else { + execution_mask.reset(); + execution_mask[KEPLER_COMPUTE_REG_INDEX(exec_upload)] = true; + execution_mask[KEPLER_COMPUTE_REG_INDEX(data_upload)] = true; + execution_mask[KEPLER_COMPUTE_REG_INDEX(launch)] = true; + } + } + static constexpr std::size_t NumConstBuffers = 8; struct Regs { @@ -211,7 +235,29 @@ public: return indirect_compute; } + bool IsNativeMode() const { return native_mode; } + private: + bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor); + void ProcessNativeReportSemaphore(u32 argument); + void DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd); + bool UpscaleInsteadOfNativeProgram(); + bool ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call); + NativeQmdScheduler native_qmd; + std::array<u32, 0xc00>* native_data_ram{}; + bool native_mme_dumped{}; + std::optional<u32> native_executing_macro_logged; + bool native_mode{}; + NativeMacroContext* native_context{}; + bool native_compute_uploaded{}; + MacroEngine* native_macro{}; + std::array<u32, 0x1000> native_registers{}; + std::span<u32> native_macro_positions; + std::vector<u32> native_parameters; + std::vector<u32> native_memo_parameters; ///< The current call's parameters, for the memo + std::optional<u32> native_executing_macro; + u32 native_bind_pointer{}; + void ProcessLaunch(); void ConsumeSinkImpl(Core::System& system) override; diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index e7da9d9138..60efef9b8e 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <cstring> +#include <limits> #include <optional> #include "common/assert.h" #include "common/bit_util.h" @@ -253,7 +254,10 @@ u32 Maxwell3D::GetMaxCurrentVertices() { const auto& limit = regs.vertex_stream_limits[index]; const GPUVAddr gpu_addr_begin = array.Address(); const GPUVAddr gpu_addr_end = limit.Address() + 1; - const u32 address_size = static_cast<u32>(gpu_addr_end - gpu_addr_begin); + const u64 stream_size = GetNativeMacroEngine().native_turing + ? regs.NativeVertexStreamSize(index) + : gpu_addr_end - gpu_addr_begin; + const u32 address_size = static_cast<u32>(std::min<u64>(stream_size, std::numeric_limits<u32>::max())); num_vertices = (std::max)( num_vertices, address_size / (std::max)(attribute.SizeInBytes(), array.stride.Value())); break; @@ -305,6 +309,9 @@ void Maxwell3D::ProcessDirtyRegisters(u32 method, u32 argument) { } void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argument, bool is_last_call) { + if (GetNativeMacroEngine().native_turing && method == MAXWELL3D_REG_INDEX(clear_surface)) { + LOG_DEBUG(HW_GPU, "Native GPU surface clear: value={:#x}", argument); + } switch (method) { case MAXWELL3D_REG_INDEX(wait_for_idle): return rasterizer->WaitForIdle(); @@ -312,9 +319,9 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume shadow_state.shadow_ram_control = static_cast<Regs::ShadowRamControl>(nonshadow_argument); return; case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): - return macro_engine.ClearCode(regs.load_mme.instruction_ptr); + return GetNativeMacroEngine().ClearCode(regs.load_mme.instruction_ptr); case MAXWELL3D_REG_INDEX(load_mme.instruction): - return macro_engine.AddCode(regs.load_mme.instruction_ptr, argument); + return GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr, argument); case MAXWELL3D_REG_INDEX(load_mme.start_address): return ProcessMacroBind(argument); case MAXWELL3D_REG_INDEX(falcon[4]): @@ -378,19 +385,34 @@ void Maxwell3D::CallMacroMethod(Core::System& system, u32 method, const std::vec // Lookup the macro offset const u32 entry = - ((method - MacroRegistersStart) >> 1) % static_cast<u32>(macro_positions.size()); + ((method - MacroRegistersStart) >> 1) % static_cast<u32>(GetNativeMacroPositions().size()); // Execute the current macro. - macro_engine.Execute(system, *this, macro_positions[entry], parameters); + GetNativeMacroEngine().Execute(system, *this, GetNativeMacroPositions()[entry], parameters); draw_manager.DrawDeferred(*this); } +// EDEN_NX2_3D_METHODS: log the first write to each 3D register (finds methods Eden doesn't model). +static const bool g_trace_first_method_writes = std::getenv("EDEN_NX2_3D_METHODS") != nullptr; + +void Maxwell3D::TraceFirstMethodWrite(u32 method, u32 argument) { + static std::array<bool, Regs::NUM_REGS> seen{}; + if (method < seen.size() && !seen[method]) { + seen[method] = true; + LOG_INFO(HW_GPU, "3D method first write: {:#x} (byte {:#x}) = {:#x}", method, method * 4, argument); + } +} + void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) { // It is an error to write to a register other than the current macro's ARG register before // it has finished execution. + if (executing_macro != 0 && method != executing_macro + 1) { + DumpRecentWrites(method); + } if (executing_macro != 0) { ASSERT(method == executing_macro + 1); } + RecordWrite(method, method_argument, 1, is_last_call ? 1 : 2); // Methods after 0xE00 are special, they're actually triggers for some microcode that was // uploaded to the GPU during initialization. @@ -398,6 +420,9 @@ void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument ProcessMacro(system, method, &method_argument, 1, is_last_call); return; } + if (g_trace_first_method_writes) [[unlikely]] { + TraceFirstMethodWrite(method, method_argument); + } ASSERT(method < Regs::NUM_REGS && "Invalid Maxwell3D register, increase the size of the Regs structure"); const u32 argument = ProcessShadowRam(method, method_argument); @@ -405,7 +430,24 @@ void Maxwell3D::CallMethod(Core::System& system, u32 method, u32 method_argument ProcessMethodCall(method, argument, method_argument, is_last_call); } +void Maxwell3D::DumpRecentWrites(u32 method) { + static u32 dumps{}; + if (dumps++ >= 4) { + return; + } + LOG_ERROR(HW_GPU, "Write to {:#x} while macro {:#x} collects parameters ({} so far); last writes:", + method, executing_macro, macro_params.size()); + for (size_t i = 0; i < recent_writes.size(); ++i) { + const auto& w = recent_writes[(recent_write_index + i) % recent_writes.size()]; + if (w.amount != 0) { + LOG_ERROR(HW_GPU, " method {:#x} arg {:#x} amount {} pending {}", w.method, w.argument, + w.amount, w.pending); + } + } +} + void Maxwell3D::CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) { + RecordWrite(method, amount ? base_start[0] : 0, amount, methods_pending); // Methods after 0xE00 are special, they're actually triggers for some microcode that was // uploaded to the GPU during initialization. if (method >= MacroRegistersStart) { @@ -453,11 +495,11 @@ void Maxwell3D::CallMultiMethod(Core::System& system, u32 method, const u32* bas } void Maxwell3D::ProcessMacroUpload(u32 data) { - macro_engine.AddCode(regs.load_mme.instruction_ptr++, data); + GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr++, data); } void Maxwell3D::ProcessMacroBind(u32 data) { - macro_positions[regs.load_mme.start_address_ptr++] = data; + GetNativeMacroPositions()[regs.load_mme.start_address_ptr++] = data; } void Maxwell3D::ProcessFirmwareCall4() { @@ -586,6 +628,9 @@ void Maxwell3D::ProcessCounterReset() { } void Maxwell3D::ProcessSyncPoint() { + if (GetNativeMacroEngine().native_turing) { + LOG_DEBUG(HW_GPU, "Native 3D syncpoint increment: id={}, raw={:#x}", regs.sync_info.sync_point.Value(), regs.reg_array[MAXWELL3D_REG_INDEX(sync_info)]); + } const u32 sync_point = regs.sync_info.sync_point.Value(); [[maybe_unused]] const u32 cache_flush = regs.sync_info.clean_l2.Value(); rasterizer->SignalSyncPoint(sync_point); @@ -595,6 +640,15 @@ void Maxwell3D::ProcessCBBind(size_t stage_index) { // Bind the buffer currently in CB_ADDRESS to the specified index in the desired shader // stage. const auto& bind_data = regs.bind_groups[stage_index]; + static std::array<std::array<bool, 32>, 5> seen{}; + if (!seen[stage_index][bind_data.shader_slot]) { + seen[stage_index][bind_data.shader_slot] = true; + LOG_INFO(HW_GPU, "Constant buffer bind: stage {} slot {} address {:#x} size {:#x}", stage_index, + bind_data.shader_slot.Value(), regs.const_buffer.Address(), regs.const_buffer.size); + } + if (bind_data.shader_slot >= Regs::MaxConstBuffers) { + return; // slot field is 5 bits; the state array holds 18 + } auto& buffer = state.shader_stages[stage_index].const_buffers[bind_data.shader_slot]; buffer.enabled = bind_data.valid.Value() != 0; buffer.address = regs.const_buffer.Address(); diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 0cab2ae986..82e716b822 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -2605,6 +2605,12 @@ public: u32 stop_shader_mask; }; + // Native C597/C797 use byte counts at 0x600 rather than legacy end addresses. + u64 NativeVertexStreamSize(std::size_t index) const { + const std::size_t method = 0x600 / sizeof(u32) + index * 2; + return (u64{reg_array[method] & 0xffU} << 32) | reg_array[method + 1]; + } + // clang-format off union { struct { @@ -3129,6 +3135,26 @@ public: /// Reads a register value located at the input method address u32 GetRegisterValue(u32 method) const; + void TraceFirstMethodWrite(u32 method, u32 argument); + + /// GPU memory, for Turing macros that read memory into their FIFO or data RAM. + MemoryManager& GetMemoryManager() { + return memory_manager; + } + + void SetNativeTuringMacroMode(bool enabled) { + GetNativeMacroEngine().native_turing = enabled; + upload_state.SetNativePartialTransfers(enabled); + } + void SetNativeMacroContext(std::shared_ptr<NativeMacroContext> context) { + native_macro_context = std::move(context); + } + MacroEngine& GetNativeMacroEngine() { return native_macro_context ? native_macro_context->engine : macro_engine; } + /// This channel's MME data RAM (shared with its compute engine). + std::array<u32, 0xc00>& GetNativeDataRam() { return native_data_ram; } + std::span<u32> GetNativeMacroPositions() { return native_macro_context ? std::span<u32>{native_macro_context->positions} : std::span<u32>{macro_positions}; } + + /// Write the value to the register identified by method. void CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) override; @@ -3260,10 +3286,25 @@ private: /// Macro method that is currently being executed / being fed parameters. u32 executing_macro = 0; + + struct RecentWrite { + u32 method; + u32 argument; + u32 amount; + u32 pending; + }; + std::array<RecentWrite, 48> recent_writes{}; + size_t recent_write_index = 0; + void RecordWrite(u32 method, u32 argument, u32 amount, u32 pending) { + recent_writes[recent_write_index++ % recent_writes.size()] = {method, argument, amount, pending}; + } + void DumpRecentWrites(u32 method); /// Parameters that have been submitted to the macro call so far. std::vector<u32> macro_params; /// Interpreter for the macro codes uploaded to the GPU. + std::shared_ptr<NativeMacroContext> native_macro_context; + std::array<u32, 0xc00> native_data_ram{}; MacroEngine macro_engine; Upload::State upload_state; diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp index ed4675ed12..0b0541799b 100644 --- a/src/video_core/engines/maxwell_dma.cpp +++ b/src/video_core/engines/maxwell_dma.cpp @@ -4,6 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> #include "common/algorithm.h" #include "common/assert.h" #include "common/logging.h" @@ -42,7 +43,11 @@ void MaxwellDMA::ConsumeSinkImpl(Core::System& system) { } void MaxwellDMA::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) { - ASSERT_MSG(method < NUM_REGS, "Invalid MaxwellDMA register"); + if (method >= NUM_REGS) { + // Ampere copy (Switch 2) can use methods past this register file; don't write past it. + LOG_DEBUG(HW_GPU, "Ignoring MaxwellDMA method {:#x} past the register file", method); + return; + } regs.reg_array[method] = method_argument; @@ -59,6 +64,32 @@ void MaxwellDMA::CallMultiMethod(Core::System& system, u32 method, const u32* ba void MaxwellDMA::Launch() { LOG_TRACE(Render_OpenGL, "DMA copy {:#x} -> {:#x}", static_cast<GPUVAddr>(regs.offset_in), GPUVAddr(regs.offset_out)); + // EDEN_NX2_TRACE_DST=<hex>: also log copies writing within 32 MiB above that GPU address. + static const u64 trace_destination = [] { + const char* value = std::getenv("EDEN_NX2_TRACE_DST"); + return value != nullptr ? std::strtoull(value, nullptr, 16) : 0ULL; + }(); + if (trace_destination != 0 && GPUVAddr(regs.offset_out) >= trace_destination && + GPUVAddr(regs.offset_out) < trace_destination + (32ULL << 20)) { + static u32 traced_destination{}; + if (traced_destination++ < 40) { + LOG_INFO(HW_GPU, + "nx2 DMA to traced dst: {:#x} -> {:#x} lines={} length={} pitch in={:#x} " + "out={:#x} launch={:#x} src_layout={} dst_layout={}", + static_cast<GPUVAddr>(regs.offset_in), GPUVAddr(regs.offset_out), + regs.line_count, regs.line_length_in, regs.pitch_in, regs.pitch_out, + regs.reg_array[0xc0], regs.launch_dma.src_memory_layout.Value() == LaunchDMA::MemoryLayout::PITCH ? "pitch" : "block", + regs.launch_dma.dst_memory_layout.Value() == LaunchDMA::MemoryLayout::PITCH ? "pitch" : "block"); + } + } + if (static const bool trace = std::getenv("EDEN_NX2_RT_TRACE") != nullptr; trace) { + static u32 traced{}; + if (traced++ < 64) { + LOG_INFO(HW_GPU, "nx2 DMA {:#x} -> {:#x} lines={} length={} launch={:#x}", + static_cast<GPUVAddr>(regs.offset_in), GPUVAddr(regs.offset_out), + regs.line_count, regs.line_length_in, regs.reg_array[0xc0]); + } + } // TODO(Subv): Perform more research and implement all features of this engine. const LaunchDMA& launch = regs.launch_dma; @@ -219,7 +250,7 @@ void MaxwellDMA::CopyBlockLinearToPitch() { UnswizzleSubrect(tmp_write_buffer, tmp_read_buffer, bytes_per_pixel, width, height, depth, x_offset, src_params.origin.y, x_elements, regs.line_count, block_height, - block_depth, dst_operand.pitch); + block_depth, dst_operand.pitch, memory_manager.UsesNativeTiling()); } void MaxwellDMA::CopyPitchToBlockLinear() { @@ -284,7 +315,7 @@ void MaxwellDMA::CopyPitchToBlockLinear() { // If the input is linear and the output is tiled, swizzle the input and copy it over. SwizzleSubrect(tmp_write_buffer, tmp_read_buffer, bytes_per_pixel, width, height, depth, x_offset, dst_params.origin.y, x_elements, regs.line_count, block_height, - block_depth, regs.pitch_in); + block_depth, regs.pitch_in, memory_manager.UsesNativeTiling()); } void MaxwellDMA::CopyBlockLinearToBlockLinear() { @@ -337,11 +368,11 @@ void MaxwellDMA::CopyBlockLinearToBlockLinear() { UnswizzleSubrect(intermediate_buffer, tmp_read_buffer, bytes_per_pixel, src_width, src.height, src.depth, src_x_offset, src.origin.y, x_elements, regs.line_count, - src.block_size.height, src.block_size.depth, pitch); + src.block_size.height, src.block_size.depth, pitch, memory_manager.UsesNativeTiling()); SwizzleSubrect(tmp_write_buffer, intermediate_buffer, bytes_per_pixel, dst_width, dst.height, dst.depth, dst_x_offset, dst.origin.y, x_elements, regs.line_count, - dst.block_size.height, dst.block_size.depth, pitch); + dst.block_size.height, dst.block_size.depth, pitch, memory_manager.UsesNativeTiling()); } void MaxwellDMA::ReleaseSemaphore() { diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 0b05d31718..b607f20161 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -4,6 +4,9 @@ // SPDX-FileCopyrightText: 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include <atomic> +#include <cstdlib> + #include "common/assert.h" #include "common/logging.h" #include "common/settings.h" @@ -18,6 +21,7 @@ #include "video_core/engines/puller.h" #include "video_core/gpu.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" #include "video_core/rasterizer_interface.h" namespace Tegra::Engines { @@ -27,14 +31,102 @@ namespace { constexpr u32 Gf100BindClassMask = 0xffff; constexpr u32 Gf100BindValidMask = 0x1f0000 | Gf100BindClassMask; +// Reads a semaphore word, flushing a GPU-written copy out of the buffer cache first. +u32 ReadSemaphoreWord(DmaPusher& dma_pusher, GPUVAddr address) { + if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(address); + device && dma_pusher.rasterizer->MustFlushRegion(*device, sizeof(u32), + VideoCommon::CacheType::BufferCache)) { + dma_pusher.rasterizer->FlushRegion(*device, sizeof(u32), + VideoCommon::CacheType::BufferCache); + } + return dma_pusher.memory_manager.Read<u32>(address); +} + } // Anonymous namespace +bool Puller::PendingAcquire::Matches(u32 current) const { + switch (static_cast<GpuSemaphoreOperation>(operation)) { + case GpuSemaphoreOperation::AcquireEqual: + return current == value; + case GpuSemaphoreOperation::AcquireGequal: + return current >= value; + case GpuSemaphoreOperation::AcquireMask: + return (current & value) != 0; + default: + return true; + } +} + +bool Puller::CanResumeAcquire(DmaPusher& dma_pusher) { + if (!pending_acquire) { + return true; + } + const u32 current = ReadSemaphoreWord(dma_pusher, pending_acquire->address); + if (Nx2SemaphoreWaiters::Get().Latched(this)) { + if (static std::atomic<u32> rescued{}; + !pending_acquire->Matches(current) && rescued.fetch_add(1) < 64) { + LOG_INFO(HW_GPU, "Channel {} met semaphore {:#x} value {:#x} at its release (now {:#x})", + dma_pusher.channel_state.bind_id, pending_acquire->address, + pending_acquire->value, current); + } + return true; + } + if (pending_acquire->Matches(current)) { + return true; + } + if (!pending_acquire->reported && + std::chrono::steady_clock::now() - pending_acquire->since > std::chrono::seconds{2}) { + pending_acquire->reported = true; + const auto device = dma_pusher.memory_manager.GpuToCpuAddress(pending_acquire->address); + LOG_WARNING(HW_GPU, "Channel {} waits on semaphore {:#x} (op {}, {:#x}, now {:#x}) for 2 s; " + "device {:#x}, buffer cache holds GPU data: {}, words {:08x} {:08x} {:08x} {:08x}", + dma_pusher.channel_state.bind_id, pending_acquire->address, + pending_acquire->operation, pending_acquire->value, current, device.value_or(0), + device && dma_pusher.rasterizer->MustFlushRegion(*device, 16, VideoCommon::CacheType::All), + dma_pusher.memory_manager.Read<u32>(pending_acquire->address & ~GPUVAddr{0xf}), + dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 4), + dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 8), + dma_pusher.memory_manager.Read<u32>((pending_acquire->address & ~GPUVAddr{0xf}) + 12)); + } + return false; +} + +bool Puller::ResumeAcquire(DmaPusher& dma_pusher) { + if (!CanResumeAcquire(dma_pusher)) { + return false; + } + pending_acquire.reset(); + Nx2SemaphoreWaiters::Get().Remove(this); + return true; +} + void Puller::ProcessBindMethod(DmaPusher& dma_pusher, const MethodCall& method_call) { LOG_DEBUG(HW_GPU, "Binding subchannel {} to engine {:#x}", method_call.subchannel, method_call.argument); u32 engine = method_call.argument; if ((engine & ~Gf100BindClassMask) != 0 && (engine & ~Gf100BindValidMask) == 0) { engine &= Gf100BindClassMask; } + // Switch 2 binds Ampere classes; run them on the Maxwell engines (same method layout for + // most state). Class 0 unbinds the subchannel. + switch (engine & 0xFFFF) { + case 0xC797: // AMPERE_B + engine = static_cast<u32>(EngineID::MAXWELL_B); + break; + case 0xC7C0: // AMPERE_COMPUTE_B + engine = static_cast<u32>(EngineID::KEPLER_COMPUTE_B); + break; + case 0xC7B5: // AMPERE_DMA_COPY_B + engine = static_cast<u32>(EngineID::MAXWELL_DMA_COPY_A); + break; + case 0x0: + if (g_nx2_ampere) { + LOG_DEBUG(HW_GPU, "Switch 2 unbinds subchannel {}", method_call.subchannel); + return; + } + break; + default: + break; + } const auto engine_id = static_cast<EngineID>(engine); bound_engines[method_call.subchannel] = engine_id; @@ -82,10 +174,64 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { const auto semaphoreOperationMask = 0xF; const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask); + // EDEN_NX2_SEM_WATCH=<gpu address>: log every release/acquire on the 16 bytes there. + static const GPUVAddr watch = [] { + const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); + return value ? GPUVAddr{std::strtoull(value, nullptr, 16)} : GPUVAddr{}; + }(); + if (watch != 0 && (regs.semaphore_address.SemaphoreAddress() & ~GPUVAddr{0xf}) == watch) { + LOG_INFO(HW_GPU, "Semaphore watch: channel {} as {} device {:#x} op {} {:#x} value {:#x} (now {:#x})", + dma_pusher.channel_state.bind_id, dma_pusher.memory_manager.GetID(), + dma_pusher.memory_manager.GpuToCpuAddress(regs.semaphore_address.SemaphoreAddress()).value_or(0), + static_cast<u32>(op), + regs.semaphore_address.SemaphoreAddress(), regs.semaphore_sequence, + dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress())); + } if (op == GpuSemaphoreOperation::WriteLong) { const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()}; const u32 payload = regs.semaphore_sequence; - dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::HasTimeout, payload, 0); + // RELEASE_SIZE (bit 24) set means a 4-byte release without the timestamp; writing 16 + // bytes there clobbers the words next to it (Switch 2 packs semaphores tightly). + const auto properties = g_nx2_ampere && (regs.semaphore_trigger & (1U << 24)) != 0 + ? VideoCommon::QueryPropertiesFlags::IsAFence + : VideoCommon::QueryPropertiesFlags::HasTimeout; + dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0); + if (const GPUVAddr mirror = g_nx2_ampere ? Nx2SemaphoreMirror(sequence_address) : 0; + mirror != 0) { + LOG_DEBUG(HW_GPU, "Switch 2 semaphore {:#x}={:#x} mirrored to {:#x}", sequence_address, payload, mirror); + dma_pusher.rasterizer->Query(mirror, VideoCommon::QueryType::Payload, properties, payload, 0); + } + } else if (g_nx2_ampere) { + // Switch 2 waits on semaphores the CPU (or another channel) releases, e.g. once it has + // written the commands this channel runs next: suspend the channel until the value is + // met instead of reading ahead (approach from changes.patch). + PendingAcquire acquire{ + .address = regs.semaphore_address.SemaphoreAddress(), + .value = regs.semaphore_sequence, + .operation = static_cast<u32>(op), + .since = std::chrono::steady_clock::now(), + .reported = false, + }; + if (op != GpuSemaphoreOperation::AcquireEqual && op != GpuSemaphoreOperation::AcquireGequal && + op != GpuSemaphoreOperation::AcquireMask) { + LOG_ERROR(HW_GPU, "Invalid semaphore operation {}", static_cast<u32>(op)); + return; + } + if (acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { + return; + } + // Registered before the fences run, so a release among them that meets the value counts + // even if a later one overwrites it. + auto& waiters = Nx2SemaphoreWaiters::Get(); + if (const auto device = dma_pusher.memory_manager.GpuToCpuAddress(acquire.address)) { + waiters.Add(this, *device, acquire.value, acquire.operation); + } + dma_pusher.rasterizer->ReleaseFences(); + if (!waiters.Latched(this) && !acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { + pending_acquire = acquire; + } else { + waiters.Remove(this); + } } else { do { const u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress()); @@ -137,8 +283,43 @@ void Puller::ProcessSemaphoreAcquire(DmaPusher& dma_pusher) { } } +// Ampere SEM_EXECUTE (method 0x1B): operation in bits 2:0 on the semaphore at +// SEM_ADDR_HI:SEM_ADDR_LO (methods 0x18/0x17) with payload SEM_PAYLOAD_LO (0x19). +void Puller::ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument) { + const GPUVAddr address = (GPUVAddr{regs.reg_array[0x18] & 0x1FFFFFF} << 32) | + (regs.reg_array[0x17] & ~3U); + const u32 payload = regs.reg_array[0x19]; + const u32 operation = argument & 7; + if (operation == 1) { // RELEASE + dma_pusher.rasterizer->Query(address, VideoCommon::QueryType::Payload, + VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); + return; + } + // Acquires: check once after flushing pending fences; never block the GPU thread on them. + dma_pusher.rasterizer->ReleaseFences(); + const u32 word = dma_pusher.memory_manager.Read<u32>(address); + const bool ready = operation == 0 ? word == payload + : operation == 2 ? word >= payload + : operation == 3 ? static_cast<s32>(word - payload) >= 0 + : operation == 4 ? (word & payload) != 0 + : operation == 5 ? (word & payload) == 0 + : true; + if (!ready) { + static std::atomic<u32> reported{}; + if (reported.fetch_add(1) < 8) { + LOG_WARNING(HW_GPU, "SEM_EXECUTE op {} at {:#x}: {:#x} vs {:#x} not met, continuing", + operation, address, word, payload); + } + } +} + /// Calls a GPU puller method. void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_call) { + static const bool nx2_trace = std::getenv("EDEN_NX2_PULLER_TRACE") != nullptr; + if (nx2_trace) { + LOG_INFO(HW_GPU, "nx2 puller sub={} method={:#x} arg={:#x}", method_call.subchannel, + method_call.method, method_call.argument); + } regs.reg_array[method_call.method] = method_call.argument; const auto method = static_cast<BufferMethods>(method_call.method); @@ -172,7 +353,7 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca break; } case BufferMethods::MemOpA: { - LOG_ERROR(HW_GPU, "Memory Operation A"); + LOG_TRACE(HW_GPU, "Memory Operation A"); break; } case BufferMethods::MemOpB: { @@ -182,15 +363,22 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca } case BufferMethods::MemOpC: case BufferMethods::MemOpD: { - LOG_ERROR(HW_GPU, "Memory Operation C,D"); + LOG_TRACE(HW_GPU, "Memory Operation C,D"); break; } case BufferMethods::SemaphoreAcquire: { - ProcessSemaphoreAcquire(dma_pusher); + // On Ampere host classes 0x17..0x1A are SEM_ADDR_LO/HI and SEM_PAYLOAD_LO/HI. + if (!g_nx2_ampere) { + ProcessSemaphoreAcquire(dma_pusher); + } break; } case BufferMethods::SemaphoreRelease: { - ProcessSemaphoreRelease(dma_pusher); + if (g_nx2_ampere) { + ProcessSemaphoreExecute(dma_pusher, method_call.argument); + } else { + ProcessSemaphoreRelease(dma_pusher); + } break; } case BufferMethods::Yield: { @@ -200,13 +388,27 @@ void Puller::CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_ca } default: LOG_ERROR(HW_GPU, "Special puller engine method {:X} not implemented", method); + if (static bool dumped = false; g_nx2_ampere && !dumped) { + dumped = true; + dma_pusher.DumpCurrentEntry("unknown host method"); + } break; } } /// Calls a GPU engine method. +static bool Nx2MethodTrace() { + static const bool trace = std::getenv("EDEN_NX2_METHOD_TRACE") != nullptr; + return trace; +} + void Puller::CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_call) { const EngineID engine = bound_engines[method_call.subchannel]; + if (Nx2MethodTrace()) { + LOG_INFO(HW_GPU, "nx2 method sub={} engine={:04X} method={:#x} arg={:#x}", + method_call.subchannel, static_cast<u32>(engine), method_call.method, + method_call.argument); + } switch (engine) { case EngineID::FERMI_TWOD_A: dma_pusher.channel_state.payload->fermi_2d.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); @@ -235,6 +437,10 @@ void Puller::CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_ca /// Calls a GPU engine multivalue method. void Puller::CallEngineMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending) { const EngineID engine = bound_engines[subchannel]; + if (Nx2MethodTrace()) { + LOG_INFO(HW_GPU, "nx2 multimethod sub={} engine={:04X} method={:#x} amount={} first={:#x}", + subchannel, static_cast<u32>(engine), method, amount, base_start[0]); + } switch (engine) { case EngineID::FERMI_TWOD_A: dma_pusher.channel_state.payload->fermi_2d.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index e8994f5640..2b9e2c5df3 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -7,7 +7,9 @@ #pragma once #include <array> +#include <chrono> #include <cstddef> +#include <optional> #include <vector> #include "common/bit_field.h" #include "common/common_funcs.h" @@ -70,6 +72,24 @@ public: BitField<8, 24, u32> syncpoint_id; }; + /// Switch 2: a semaphore acquire this channel is suspended on. + struct PendingAcquire { + GPUVAddr address; + u32 value; + u32 operation; + std::chrono::steady_clock::time_point since; + bool reported; + + [[nodiscard]] bool Matches(u32 current) const; + }; + + [[nodiscard]] bool HasPendingAcquire() const { + return pending_acquire.has_value(); + } + [[nodiscard]] bool CanResumeAcquire(DmaPusher& dma_pusher); + /// Clears the pending acquire once it is met; false while the channel has to keep waiting. + bool ResumeAcquire(DmaPusher& dma_pusher); + void CallMethod(DmaPusher& dma_pusher, const MethodCall& method_call); void CallMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending); void BindRasterizer(DmaPusher& dma_pusher, VideoCore::RasterizerInterface* rasterizer); @@ -125,8 +145,11 @@ private: void ProcessSemaphoreAcquire(DmaPusher& dma_pusher); void ProcessSemaphoreRelease(DmaPusher& dma_pusher); void ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher); + void ProcessSemaphoreExecute(DmaPusher& dma_pusher, u32 argument); [[nodiscard]] bool ExecuteMethodOnEngine(DmaPusher& dma_pusher, u32 method); + std::optional<PendingAcquire> pending_acquire; + /// Mapping of command subchannels to their bound engine ids std::array<EngineID, 8> bound_engines{}; diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index 776c01e3fa..eb72529da0 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -68,6 +68,10 @@ public: } void SyncOperation(std::function<void()>&& func) { + // Not only the GPU thread queues these (query cache work runs elsewhere too), while + // SignalFence moves the list into pending_operations: unguarded, the list was corrupted + // and the fencing thread crashed walking it. Its own lock, held for no callbacks. + std::scoped_lock lock{uncommitted_guard}; uncommitted_operations.emplace_back(std::move(func)); } @@ -77,15 +81,20 @@ public: if constexpr (!can_async_check) { TryReleasePendingFences<false>(); } - CommitAsyncFlushes(); + const u8 committed_flushes = CommitAsyncFlushes(); TFence new_fence = CreateFence(!should_flush); if constexpr (can_async_check) { guard.lock(); } - if (delay_fence) { - uncommitted_operations.emplace_back(std::move(func)); + { + std::scoped_lock lock{uncommitted_guard}; + if (delay_fence) { + uncommitted_operations.emplace_back(std::move(func)); + } + pending_operations.emplace_back(std::move(uncommitted_operations)); + uncommitted_operations.clear(); // moved-from: make it empty explicitly } - pending_operations.emplace_back(std::move(uncommitted_operations)); + pending_flushes.push_back(committed_flushes); QueueFence(new_fence); if (!delay_fence) { func(); @@ -178,7 +187,8 @@ private: return; } } - PopAsyncFlushes(); + PopAsyncFlushes(pending_flushes.front()); + pending_flushes.pop_front(); auto operations = std::move(pending_operations.front()); pending_operations.pop_front(); for (auto& operation : operations) { @@ -199,6 +209,7 @@ private: TFence current_fence; std::deque<std::function<void()>> current_operations; + u8 current_flushes{}; while (!stop_token.stop_requested()) { { std::unique_lock lock(guard); @@ -208,13 +219,15 @@ private: } current_fence = std::move(fences.front()); current_operations = std::move(pending_operations.front()); + current_flushes = pending_flushes.front(); fences.pop(); pending_operations.pop_front(); + pending_flushes.pop_front(); } if (!current_fence->IsStubbed()) { WaitFence(current_fence); } - PopAsyncFlushes(); + PopAsyncFlushes(current_flushes); for (auto& operation : current_operations) { operation(); } @@ -237,30 +250,55 @@ private: query_cache.HasUncommittedFlushes(); } - void PopAsyncFlushes() { - { - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; + static constexpr u8 TextureFlushes = 1; + static constexpr u8 BufferFlushes = 2; + + /// committed: what CommitAsyncFlushes returned for this fence. + void PopAsyncFlushes(u8 committed) { + if (committed & TextureFlushes) { + std::scoped_lock lock{texture_cache.mutex}; texture_cache.PopAsyncFlushes(); + } + if (committed & BufferFlushes) { + std::scoped_lock lock{buffer_cache.mutex}; buffer_cache.PopAsyncFlushes(); } query_cache.PopAsyncFlushes(); } - void CommitAsyncFlushes() { + /// Returns which caches committed downloads for this fence (TextureFlushes, BufferFlushes). + /// Switch 2 titles signal a fence per compute dispatch (thousands a second), mostly with + /// nothing or only buffer data to download: the fencing thread then takes only the lock of + /// the cache with work, instead of holding both while it copies (the GPU thread's draws and + /// uploads waited for it). + u8 CommitAsyncFlushes() { + u8 committed = 0; { - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; - texture_cache.CommitAsyncFlushes(); - buffer_cache.CommitAsyncFlushes(); + std::scoped_lock lock{texture_cache.mutex}; + if (texture_cache.HasAsyncFlushWork()) { + texture_cache.CommitAsyncFlushes(); + committed |= TextureFlushes; + } + } + { + std::scoped_lock lock{buffer_cache.mutex}; + if (buffer_cache.HasUncommittedFlushes()) { + buffer_cache.CommitAsyncFlushes(); + committed |= BufferFlushes; + } } query_cache.CommitAsyncFlushes(); + return committed; } std::queue<TFence> fences; std::deque<std::function<void()>> uncommitted_operations; std::deque<std::deque<std::function<void()>>> pending_operations; + std::deque<u8> pending_flushes; ///< Per fence: the caches that committed downloads std::mutex guard; std::mutex ring_guard; + std::mutex uncommitted_guard; ///< Guards uncommitted_operations std::condition_variable cv; std::jthread fence_thread; diff --git a/src/video_core/framebuffer_config.h b/src/video_core/framebuffer_config.h index 1fc70583f9..bb041cf12f 100644 --- a/src/video_core/framebuffer_config.h +++ b/src/video_core/framebuffer_config.h @@ -37,6 +37,7 @@ struct FramebufferConfig { Service::android::BufferTransformFlags transform_flags{}; Common::Rectangle<int> crop_rect{}; BlendMode blending{}; + bool native_tiling{}; u32 layer_stack_mask{Service::Nvnflinger::DefaultLayerStackMask}; }; diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 1621b24125..6441b9511a 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -206,6 +206,10 @@ struct GPU::Impl { gpu_thread.SubmitList(channel, std::move(entries), is_async); } + void RunInOrder(std::function<void()>&& func) { + gpu_thread.Run(std::move(func), is_async); + } + /// Notify rasterizer that any caches of the specified region should be flushed to Switch memory void FlushRegion(DAddr addr, u64 size) { gpu_thread.FlushRegion(addr, size, is_async); @@ -407,6 +411,10 @@ u64 GPU::RequestFlush(DAddr addr, std::size_t size) { }); } +u64 GPU::RunOnGpuThread(std::function<void()>&& action) { + return impl->RequestSyncOperation(std::move(action)); +} + u64 GPU::CurrentSyncRequestFence() const { return impl->CurrentSyncRequestFence(); } @@ -530,6 +538,10 @@ void GPU::FlushRegion(DAddr addr, u64 size) { impl->FlushRegion(addr, size); } +void GPU::RunInOrder(std::function<void()>&& func) { + impl->RunInOrder(std::move(func)); +} + void GPU::InvalidateRegion(DAddr addr, u64 size, bool preserve_gpu_writes) { impl->InvalidateRegion(addr, size, preserve_gpu_writes); } diff --git a/src/video_core/gpu.h b/src/video_core/gpu.h index 974d214e10..3a6546b41b 100644 --- a/src/video_core/gpu.h +++ b/src/video_core/gpu.h @@ -6,6 +6,7 @@ #pragma once +#include <functional> #include <memory> #include "common/bit_field.h" @@ -163,6 +164,9 @@ public: /// Request a host GPU memory flush from the CPU. [[nodiscard]] u64 RequestFlush(DAddr addr, std::size_t size); + /// Runs an action on the GPU thread between command lists (returns its sync fence). + u64 RunOnGpuThread(std::function<void()>&& action); + /// Obtains current flush request fence id. [[nodiscard]] u64 CurrentSyncRequestFence() const; @@ -247,6 +251,9 @@ public: /// Notify rasterizer that any caches of the specified region should be flushed to Switch memory void FlushRegion(DAddr addr, u64 size); + /// Runs func on the GPU thread after the command lists submitted so far. + void RunInOrder(std::function<void()>&& func); + /// Notify rasterizer that any caches of the specified region should be invalidated void InvalidateRegion(DAddr addr, u64 size, bool preserve_gpu_writes = false); diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index a01375e7be..72da1904e1 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -4,6 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <chrono> +#include <thread> + #include "common/assert.h" #include "common/scope_exit.h" #include "common/settings.h" @@ -14,6 +17,7 @@ #include "video_core/dma_pusher.h" #include "video_core/gpu.h" #include "video_core/gpu_thread.h" +#include "video_core/nx2_profile.h" #include "video_core/host1x/host1x.h" #include "video_core/renderer_base.h" @@ -35,18 +39,52 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten auto current_context = context.Acquire(); CommandDataContainer next; + // When a pending channel last moved or a command arrived (see the polling below). + auto last_progress = std::chrono::steady_clock::now(); + namespace Profile = VideoCommon::Nx2Profile; while (!stop_token.stop_requested()) { - state.queue.PopWait(next, stop_token); + Profile::Report(); + if (scheduler.HasPendingChannels()) { + // Suspended channels wait on semaphores the CPU releases: poll them meanwhile. + { + Profile::Scope scope{Profile::Zone::ResumePending}; + scheduler.ResumePending(system.GPU()); + } + if (!state.queue.TryPop(next)) { + // The CPU usually releases the semaphore soon after: poll for half a + // millisecond before sleeping, or every wait also waits out a host sleep + // (title 1 waits ~25 times a second). + Profile::Scope scope{Profile::Zone::PendingSleep}; + if (std::chrono::steady_clock::now() - last_progress < + std::chrono::microseconds{500}) { + std::this_thread::yield(); + } else { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + continue; + } + } else { + Profile::Scope scope{Profile::Zone::Idle}; + state.queue.PopWait(next, stop_token); + } if (stop_token.stop_requested()) { break; } + last_progress = std::chrono::steady_clock::now(); if (auto* submit_list = std::get_if<SubmitListCommand>(&next.data)) { + Profile::Scope scope{Profile::Zone::Push}; scheduler.Push(system.GPU(), submit_list->channel, std::move(submit_list->entries)); } else if (std::holds_alternative<GPUTickCommand>(next.data)) { + Profile::Scope scope{Profile::Zone::Tick}; system.GPU().TickWork(); + } else if (auto* run = std::get_if<RunCommand>(&next.data)) { + Profile::Scope scope{Profile::Zone::Run}; + run->func(); } else if (const auto* flush = std::get_if<FlushRegionCommand>(&next.data)) { + Profile::Scope scope{Profile::Zone::FlushInvalidate}; renderer.ReadRasterizer()->FlushRegion(flush->addr, flush->size); } else if (const auto* invalidate = std::get_if<InvalidateRegionCommand>(&next.data)) { + Profile::Scope scope{Profile::Zone::FlushInvalidate}; renderer.ReadRasterizer()->OnCacheInvalidation(invalidate->addr, invalidate->size); } else { ASSERT(false); @@ -77,6 +115,10 @@ void ThreadManager::TickGPU(bool is_async) { PushCommand(GPUTickCommand(), false, is_async); } +void ThreadManager::Run(std::function<void()>&& func, bool is_async) { + PushCommand(RunCommand{std::move(func)}, false, is_async); +} + void ThreadManager::InvalidateRegion(DAddr addr, u64 size) { rasterizer->OnCacheInvalidation(addr, size); } diff --git a/src/video_core/gpu_thread.h b/src/video_core/gpu_thread.h index 631a680db4..796c0d89f6 100644 --- a/src/video_core/gpu_thread.h +++ b/src/video_core/gpu_thread.h @@ -6,6 +6,7 @@ #pragma once +#include <functional> #include <atomic> #include <condition_variable> #include <mutex> @@ -76,9 +77,14 @@ struct FlushAndInvalidateRegionCommand final { /// Command to make the gpu look into pending requests struct GPUTickCommand final {}; +/// Command to run a function on the GPU thread after the commands queued before it +struct RunCommand final { + std::function<void()> func; +}; + using CommandData = std::variant<std::monostate, SubmitListCommand, FlushRegionCommand, InvalidateRegionCommand, - FlushAndInvalidateRegionCommand, GPUTickCommand>; + FlushAndInvalidateRegionCommand, GPUTickCommand, RunCommand>; struct CommandDataContainer { CommandDataContainer() = default; @@ -125,6 +131,9 @@ public: void TickGPU(bool is_async); + /// Runs func on the GPU thread once the commands queued before it have been processed. + void Run(std::function<void()>&& func, bool is_async); + void NotifyShutdown(); private: diff --git a/src/video_core/host1x/gpu_device_memory_manager.h b/src/video_core/host1x/gpu_device_memory_manager.h index a9f249991f..1b4f57fe24 100644 --- a/src/video_core/host1x/gpu_device_memory_manager.h +++ b/src/video_core/host1x/gpu_device_memory_manager.h @@ -14,7 +14,9 @@ namespace Tegra { struct MaxwellDeviceMethods; struct MaxwellDeviceTraits { - static constexpr size_t device_virtual_bits = 34; + // 64 GiB: Switch 2 titles map most of their memory twice plus GPU-only regions (title 1 + // ran out of 16 GiB). + static constexpr size_t device_virtual_bits = 36; using DeviceInterface = typename VideoCore::RasterizerInterface; using DeviceMethods = MaxwellDeviceMethods; }; diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index c4e7ac3f6f..8082e830d6 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -15,6 +15,7 @@ set(GLSL_INCLUDES set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/astc_decoder.comp + ${CMAKE_CURRENT_SOURCE_DIR}/bc_encoder.comp ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_float.frag ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d.comp ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_msaa.frag diff --git a/src/video_core/host_shaders/astc_decoder.comp b/src/video_core/host_shaders/astc_decoder.comp index e059c683ff..01bec11f17 100644 --- a/src/video_core/host_shaders/astc_decoder.comp +++ b/src/video_core/host_shaders/astc_decoder.comp @@ -33,6 +33,7 @@ UNIFORM(3) uint block_size; UNIFORM(4) uint x_shift; UNIFORM(5) uint block_height; UNIFORM(6) uint block_height_mask; +UNIFORM(7) uint native_tiling; END_PUSH_CONSTANTS struct EncodingData { @@ -1404,7 +1405,12 @@ uint SwizzleOffset(uvec2 pos) { void main() { uvec3 pos = gl_GlobalInvocationID; pos.x <<= BYTES_PER_BLOCK_LOG2; - const uint swizzle = SwizzleOffset(pos.xy); + uint swizzle = SwizzleOffset(pos.xy); + if (native_tiling != 0u) { + // Switch 2 GOBs swap offset bits 5 and 6 (see block_linear_unswizzle_2d.comp). + const uint difference = ((swizzle >> 5u) ^ (swizzle >> 6u)) & 1u; + swizzle ^= (difference << 5u) | (difference << 6u); + } const uint block_y = pos.y >> GOB_SIZE_Y_SHIFT; uint offset = 0; diff --git a/src/video_core/host_shaders/bc_encoder.comp b/src/video_core/host_shaders/bc_encoder.comp new file mode 100644 index 0000000000..3a87e92dcb --- /dev/null +++ b/src/video_core/host_shaders/bc_encoder.comp @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// Compresses one level of an RGBA8 image (the ASTC decoder's output) to BC1 or BC3, one 4x4 block +// per invocation, so re-compressed ASTC never round-trips through the CPU. Colour endpoints are +// the block's bounding box, oriented along the colours' covariance and inset by 1/16; each texel +// takes the nearest palette entry. Values are encoded as stored (sRGB data stays sRGB, like the +// CPU path). + +#version 460 core + +layout (local_size_x = 8, local_size_y = 8, local_size_z = 1) in; + +layout (set = 0, binding = 0, rgba8) uniform readonly image2DArray source; + +layout (std430, set = 0, binding = 1) writeonly buffer Output { + uvec2 blocks[]; +}; + +layout (push_constant) uniform PushConstants { + uint blocks_x; + uint blocks_y; + uint width; + uint height; + uint bc3; + uint out_offset; // in 8-byte units +}; + +uint Pack565(vec3 color) { + const uvec3 q = uvec3(round(clamp(color, 0.0, 1.0) * vec3(31.0, 63.0, 31.0))); + return (q.r << 11) | (q.g << 5) | q.b; +} + +vec3 Unpack565(uint value) { + const uvec3 q = uvec3((value >> 11) & 31u, (value >> 5) & 63u, value & 31u); + const uvec3 e = uvec3((q.r << 3) | (q.r >> 2), (q.g << 2) | (q.g >> 4), (q.b << 3) | (q.b >> 2)); + return vec3(e) / 255.0; +} + +float Distance2(vec3 a, vec3 b) { + const vec3 d = a - b; + return dot(d, d); +} + +// BC1 colour block. With punch_through (BC1 only), texels with alpha < 0.5 use the transparent +// index of the three-colour mode; otherwise the four-colour mode is used (BC3 always decodes so). +uvec2 EncodeColor(in vec4 texels[16], bool punch_through) { + vec3 lo = vec3(1.0); + vec3 hi = vec3(0.0); + for (int i = 0; i < 16; ++i) { + if (punch_through && texels[i].a < 0.5) { + continue; + } + lo = min(lo, texels[i].rgb); + hi = max(hi, texels[i].rgb); + } + if (any(greaterThan(lo, hi))) { + // Every texel transparent: colour0 == colour1 selects three-colour mode, index 3. + return uvec2(0u, 0xFFFFFFFFu); + } + const vec3 center = (lo + hi) * 0.5; + const vec3 extent = hi - lo; + const int major = extent.g >= extent.r && extent.g >= extent.b ? 1 : (extent.r >= extent.b ? 0 : 2); + vec3 covariance = vec3(0.0); + for (int i = 0; i < 16; ++i) { + if (punch_through && texels[i].a < 0.5) { + continue; + } + const vec3 d = texels[i].rgb - center; + covariance += d * d[major]; + } + vec3 e0 = hi; + vec3 e1 = lo; + for (int c = 0; c < 3; ++c) { + if (covariance[c] < 0.0) { + e0[c] = lo[c]; + e1[c] = hi[c]; + } + } + const vec3 inset = (e0 - e1) / 16.0; + e0 -= inset; + e1 += inset; + + uint q0 = Pack565(e0); + uint q1 = Pack565(e1); + // Four-colour mode needs colour0 > colour1, three-colour mode colour0 <= colour1. + if (punch_through ? q0 > q1 : q0 < q1) { + const uint t = q0; + q0 = q1; + q1 = t; + } + if (!punch_through && q0 == q1) { + return uvec2(q0 | (q1 << 16), 0u); + } + vec3 palette[4]; + palette[0] = Unpack565(q0); + palette[1] = Unpack565(q1); + if (punch_through) { + palette[2] = (palette[0] + palette[1]) * 0.5; + palette[3] = vec3(0.0); + } else { + palette[2] = (2.0 * palette[0] + palette[1]) / 3.0; + palette[3] = (palette[0] + 2.0 * palette[1]) / 3.0; + } + const int entries = punch_through ? 3 : 4; + uint indices = 0u; + for (int i = 0; i < 16; ++i) { + uint best = 0u; + if (punch_through && texels[i].a < 0.5) { + best = 3u; + } else { + float best_distance = Distance2(texels[i].rgb, palette[0]); + for (int p = 1; p < entries; ++p) { + const float distance = Distance2(texels[i].rgb, palette[p]); + if (distance < best_distance) { + best_distance = distance; + best = uint(p); + } + } + } + indices |= best << (2 * i); + } + return uvec2(q0 | (q1 << 16), indices); +} + +// BC3 alpha block: eight-value mode (alpha0 > alpha1), indices 0 = alpha0, 1 = alpha1, 2..7 the +// ramp from alpha0 to alpha1. +uvec2 EncodeAlpha(in vec4 texels[16]) { + float lo = 1.0; + float hi = 0.0; + for (int i = 0; i < 16; ++i) { + lo = min(lo, texels[i].a); + hi = max(hi, texels[i].a); + } + const uint a0 = uint(round(hi * 255.0)); + const uint a1 = uint(round(lo * 255.0)); + uint bits_lo = a0 | (a1 << 8); + uint bits_hi = 0u; + if (a0 == a1) { + return uvec2(bits_lo, bits_hi); + } + for (uint i = 0u; i < 16u; ++i) { + const float t = (texels[i].a * 255.0 - float(a1)) / float(a0 - a1); + const uint q = uint(clamp(round(t * 7.0), 0.0, 7.0)); + const uint index = q == 7u ? 0u : (q == 0u ? 1u : 8u - q); + const uint bit = 16u + 3u * i; + if (bit < 32u) { + bits_lo |= index << bit; + if (bit > 29u) { + bits_hi |= index >> (32u - bit); + } + } else { + bits_hi |= index << (bit - 32u); + } + } + return uvec2(bits_lo, bits_hi); +} + +void main() { + const uvec3 id = gl_GlobalInvocationID; + if (id.x >= blocks_x || id.y >= blocks_y) { + return; + } + vec4 texels[16]; + const ivec2 base = ivec2(id.xy) * 4; + const ivec2 last = ivec2(width, height) - 1; + bool any_transparent = false; + for (int y = 0; y < 4; ++y) { + for (int x = 0; x < 4; ++x) { + // Edge blocks repeat the last row/column so padding doesn't widen the endpoints. + const ivec2 coord = min(base + ivec2(x, y), last); + texels[y * 4 + x] = imageLoad(source, ivec3(coord, int(id.z))); + any_transparent = any_transparent || texels[y * 4 + x].a < 0.5; + } + } + const uint block = (id.z * blocks_y + id.y) * blocks_x + id.x; + if (bc3 != 0u) { + blocks[out_offset + block * 2u] = EncodeAlpha(texels); + blocks[out_offset + block * 2u + 1u] = EncodeColor(texels, false); + } else { + blocks[out_offset + block] = EncodeColor(texels, any_transparent); + } +} diff --git a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp index 2a3446ca66..bbe73acd1a 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp @@ -39,6 +39,7 @@ UNIFORM(4) uint block_size; UNIFORM(5) uint x_shift; UNIFORM(6) uint block_height; UNIFORM(7) uint block_height_mask; +UNIFORM(8) uint native_tiling; END_PUSH_CONSTANTS #if HAS_EXTENDED_TYPES @@ -77,7 +78,12 @@ uint SwizzleTable(uint pos) { uint SwizzleOffset(uvec2 pos) { pos = pos & SWIZZLE_MASK; - return SwizzleTable(pos.y * 64 + pos.x); + uint offset = SwizzleTable(pos.y * 64 + pos.x); + if (native_tiling != 0) { + uint difference = ((offset >> 5) ^ (offset >> 6)) & 1; + offset ^= (difference << 5) | (difference << 6); + } + return offset; } uvec4 ReadTexel(uint offset) { diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp index 34e4f93307..8dfaf0cd1a 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp @@ -41,6 +41,7 @@ UNIFORM(6) uint block_height; UNIFORM(7) uint block_height_mask; UNIFORM(8) uint block_depth; UNIFORM(9) uint block_depth_mask; +UNIFORM(10) uint native_tiling; END_PUSH_CONSTANTS #if HAS_EXTENDED_TYPES @@ -79,7 +80,12 @@ uint SwizzleTable(uint pos) { uint SwizzleOffset(uvec2 pos) { pos = pos & SWIZZLE_MASK; - return SwizzleTable(pos.y * 64 + pos.x); + uint offset = SwizzleTable(pos.y * 64 + pos.x); + if (native_tiling != 0) { + uint difference = ((offset >> 5) ^ (offset >> 6)) & 1; + offset ^= (difference << 5) | (difference << 6); + } + return offset; } uvec4 ReadTexel(uint offset) { diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp index e084837f25..28b1dcf599 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp @@ -42,7 +42,7 @@ layout(push_constant) uniform PushConstants { uint block_depth; // Offset 48 uint block_depth_mask; // Offset 52 - int _pad; // Offset 56 + uint native_tiling; // Offset 56 ivec3 destination; // Offset 60 } pc; @@ -59,6 +59,7 @@ BEGIN_PUSH_CONSTANTS UNIFORM(8) uint block_depth; UNIFORM(9) uint block_depth_mask; UNIFORM(10) uvec3 blocks_dim; + UNIFORM(11) uint native_tiling; END_PUSH_CONSTANTS #define pc // Map pc prefix to nothing for OpenGL compatibility #endif @@ -103,7 +104,12 @@ uint SwizzleTable(uint pos) { // --- Helpers --- uint SwizzleOffset(uvec2 pos) { pos &= SWIZZLE_MASK; - return SwizzleTable(pos.y * 64u + pos.x); + uint offset = SwizzleTable(pos.y * 64 + pos.x); + if (pc.native_tiling != 0) { + uint difference = ((offset >> 5) ^ (offset >> 6)) & 1; + offset ^= (difference << 5) | (difference << 6); + } + return offset; } uvec4 ReadTexel(uint offset) { diff --git a/src/video_core/macro.cpp b/src/video_core/macro.cpp index 86f528af15..3422cb16fa 100644 --- a/src/video_core/macro.cpp +++ b/src/video_core/macro.cpp @@ -4,6 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <array> +#include <cstdlib> +#include <unordered_set> #include <cstring> #include <fstream> #include <optional> @@ -25,6 +28,7 @@ #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/dirty_flags.h" +#include "video_core/memory_manager.h" #include "video_core/rasterizer_interface.h" #include "video_core/macro.h" @@ -32,6 +36,7 @@ #include "common/bit_field.h" #include "common/logging.h" #include "video_core/engines/maxwell_3d.h" +#include "video_core/nx2_profile.h" namespace Tegra { @@ -1339,6 +1344,173 @@ static void Dump(u64 hash, std::span<const u32> code, bool decompiled = false) { void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u32 method, std::span<const u32> parameters) { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Macro}; + if (native_turing) { + maxwell3d.RefreshParameters(); + const auto context = reinterpret_cast<std::uintptr_t>(this); + static std::unordered_set<std::uintptr_t> captured_contexts; + static const bool capture = std::getenv("EDEN_NATIVE_FRAME_CAPTURE") != nullptr; + if (capture && captured_contexts.insert(context).second) { + std::ofstream dump("native-graphics-mme-" + std::to_string(context) + ".bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code)); + } + // EDEN_NX2_MME_LOG: log every macro call (runs ~750,000 times a second in title 1). + if (static const bool log_calls = std::getenv("EDEN_NX2_MME_LOG") != nullptr; log_calls) { + LOG_DEBUG(HW_GPU, "Native MME context={:#x}", context); + LOG_DEBUG(HW_GPU, "Native MME start={:#x}, parameters={}, data={:08x}", method, parameters.size(), fmt::join(parameters, ",")); + } + // Reused between calls (a macro never calls another: methods >= 0xe00 are refused below); + // filled only for calls that are interpreted. + auto& fifo = native_fifo; + native_interpreter.active_dram = &maxwell3d.GetNativeDataRam(); + // EDEN_NX2_MME_RECORD=<file>: write the first 200000 calls (u32 start, u32 count, the + // parameters) and then the macro code to <file>.code, to check interpreter changes + // offline against the previous interpreter. EDEN_NX2_MME_RECORD_SKIP=<n>: skip the first + // n calls (title 1 makes ~1,000,000 a second in 3D scenes). + if (static const char* const record_path = std::getenv("EDEN_NX2_MME_RECORD"); record_path != nullptr) { + static std::ofstream record{record_path, std::ios::binary}; + static u32 recorded{}; + static u64 to_skip = [] { + const char* value = std::getenv("EDEN_NX2_MME_RECORD_SKIP"); + return value != nullptr ? std::strtoull(value, nullptr, 10) : 0ULL; + }(); + if (to_skip > 0) { + --to_skip; + } else if (recorded < 200000) { + const u32 count = static_cast<u32>(parameters.size()); + record.write(reinterpret_cast<const char*>(&method), sizeof(method)); + record.write(reinterpret_cast<const char*>(&count), sizeof(count)); + record.write(reinterpret_cast<const char*>(parameters.data()), parameters.size_bytes()); + if (++recorded == 200000) { + record.flush(); + std::ofstream code{std::string{record_path} + ".code", std::ios::binary}; + code.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code)); + LOG_INFO(HW_GPU, "Recorded 200000 macro calls to {}", record_path); + } + } + } + // A method write of the macro (also what a remembered call replays). + bool read_memory = false; + const auto emit = [&](u32 address, u32 value) { + if (address >= 0xe00) return false; + if (address == 0x556) { + // SET_RENDER_ENABLE_OVERRIDE: 0 use the render-enable condition, 1 always, + // 2 never. Log a histogram every 20000 macro writes (pop-in diagnostics). + static std::array<u64, 4> modes{}; + static u64 enable_writes{}; + ++modes[value & 3]; + if (++enable_writes % 20000 == 0) { + LOG_INFO(HW_GPU, "Native MME render enable modes: use {} always {} never {} other {}", + modes[0], modes[1], modes[2], modes[3]); + } + } + maxwell3d.CallMethod(system, address, value, true); + // Turing MME memory reads (methods 0x550 SET_MME_MEM_ADDRESS_A/B, 0x558 data RAM + // address, 0x55c MME_DMA_READ into data RAM, 0x560 MME_DMA_READ_FIFOED into + // the load FIFO, which the macro then reads after a load barrier). + constexpr u32 MemAddressA = 0x154, MemAddressB = 0x155, DataRamAddress = 0x156; + constexpr u32 DmaRead = 0x157, DmaReadFifoed = 0x158; + if (address == DmaRead || address == DmaReadFifoed) { + read_memory = true; + const GPUVAddr source = (GPUVAddr{maxwell3d.GetRegisterValue(MemAddressA) & 0x1ffffff} << 32) | + maxwell3d.GetRegisterValue(MemAddressB); + auto& memory = maxwell3d.GetMemoryManager(); + const u32 ram = maxwell3d.GetRegisterValue(DataRamAddress); + // The words may have just been written by the GPU (e.g. indirect draw arguments + // from a culling dispatch) and still be only in host buffers: flush them first, + // or the macro draws with an older frame's arguments (objects pop in and out). + const size_t read_bytes = size_t{std::min<u32>(value, 0x1000)} * 4; + if (memory.IsMemoryDirty(source, read_bytes, VideoCommon::CacheType::BufferCache)) { + memory.FlushRegion(source, read_bytes, VideoCommon::CacheType::BufferCache); + static u64 flushes{}; + if (++flushes % 2000 == 1) { + LOG_INFO(HW_GPU, "Native MME DMA read flushed GPU-written memory ({} times)", flushes); + } + } + static u32 dma_logs{}; + if (dma_logs < 400) { + ++dma_logs; + std::array<u32, 8> words{}; + for (u32 i = 0; i < std::min<u32>(value, 8); ++i) words[i] = memory.Read<u32>(source + u64{i} * 4); + LOG_INFO(HW_GPU, "Native MME DMA read{} start={:#x} source={:#x} count={} words={:08x}", + address == DmaReadFifoed ? " fifoed" : "", method, source, value, fmt::join(words, ",")); + } + for (u32 i = 0; i < std::min<u32>(value, 0x1000); ++i) { + const u32 word = memory.Read<u32>(source + u64{i} * 4); + if (address == DmaReadFifoed) { + fifo.push_back(word); + } else if (ram + i < native_interpreter.active_dram->size()) { + if (static const bool mme_dump = std::getenv("EDEN_NX2_MME_DUMP") != nullptr; + mme_dump && ram + i == 0x27 && (*native_interpreter.active_dram)[0x27] != word) { + LOG_INFO(HW_GPU, "MME data RAM[0x27] {:#x} -> {:#x} by 3D engine {} (DMA from {:#x})", + (*native_interpreter.active_dram)[0x27], word, fmt::ptr(&maxwell3d), source); + } + (*native_interpreter.active_dram)[ram + i] = word; + } + } + } + return true; + }; + // EDEN_NX2_MME_MEMO=0: interpret every call (see macro_memo.h). + static const bool use_memo = [] { + const char* value = std::getenv("EDEN_NX2_MME_MEMO"); + return value == nullptr || value[0] != '0'; + }(); + if (use_memo) { + // EDEN_NX2_MME_MEMO_STATS: log the memo's counters every 2^24 calls. + if (static const bool memo_stats = std::getenv("EDEN_NX2_MME_MEMO_STATS") != nullptr; + memo_stats && (++native_calls & 0xffffff) == 0) { + const auto& stats = native_memo.GetStatistics(); + LOG_INFO(HW_GPU, + "Macro memo: {} hits, {} misses, {} guard failures, {} stores ({} in " + "place), {} clears; {} entries, {} KiB", + stats.hits, stats.misses, stats.guard_failures, stats.stores, + stats.overwrites, stats.clears, native_memo.Size(), + native_memo.ArenaWords() * 4 / 1024); + } + native_memo.Synchronize(native_code_generation); + if (decltype(native_memo)::Effects effects; + native_memo.Find(method, parameters, *native_interpreter.active_dram, effects)) { + auto& data_ram = *native_interpreter.active_dram; + for (size_t i = 0; i < effects.data_ram_writes.size(); i += 2) { + data_ram[effects.data_ram_writes[i]] = effects.data_ram_writes[i + 1]; + } + for (size_t i = 0; i < effects.writes.size(); i += 2) { + emit(effects.writes[i], effects.writes[i + 1]); + } + return; + } + native_writes.clear(); + } + fifo.assign(parameters.begin(), parameters.end()); + const u64 code_generation = native_code_generation; + const auto result = native_interpreter.Execute(native_code, method, fifo, + [&](u32 address) { return address < Maxwell3D::Regs::NUM_REGS ? maxwell3d.GetRegisterValue(address) : 0U; }, + [&](u32 address, u32 value) { + if (use_memo) { + native_writes.emplace_back(address, value); + } + return emit(address, value); + }); + // Remember the call if its effects depend only on its parameters and the data RAM words + // it read (and the macro code did not change under it). + if (use_memo && result.success && !read_memory && !native_interpreter.read_registers && + !native_interpreter.reread_data_ram && code_generation == native_code_generation) { + native_memo.Store(method, parameters, native_interpreter.data_ram_reads, + native_interpreter.data_ram_writes, native_writes); + } + if (!result.success) { + native_execution_failed = true; + if (static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr; + diagnostics) { + std::ofstream dump("native-mme-" + std::to_string(method) + ".bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code)); + } + LOG_ERROR(HW_GPU, "Native MME failed: start={:#x}, pc={:#x}, consumed={}/{}, error={}", + method, result.pc, result.consumed, parameters.size(), result.error); + } + return; + } const auto execute_variant = [&system, &maxwell3d, ¶meters, method](AnyCachedMacro& cached) { if (std::holds_alternative<MacroInterpreterImpl>(cached) || diff --git a/src/video_core/macro.h b/src/video_core/macro.h index 90519a5429..a6ca4c4bd2 100644 --- a/src/video_core/macro.h +++ b/src/video_core/macro.h @@ -7,6 +7,8 @@ #pragma once #include <memory> +#include "video_core/macro_memo.h" +#include "video_core/macro_turing.h" #include <span> #include <variant> #include <vector> @@ -223,10 +225,16 @@ struct MacroEngine { MacroEngine(bool is_interpreted_) noexcept : is_interpreted{is_interpreted_} {} // Store the uploaded macro code to compile them when they're called. inline void AddCode(u32 method, u32 data) noexcept { + if (native_turing) { + if (native_write_offset < native_code.size()) native_code[native_write_offset++] = data; + ++native_code_generation; + return; + } uploaded_macro_code[method].push_back(data); } // Clear the code associated with a method. inline void ClearCode(u32 method) noexcept { + if (native_turing) { native_write_offset = method < 0x1000 ? method * 3 : static_cast<u32>(native_code.size()); return; } macro_cache.erase(method); uploaded_macro_code.erase(method); } @@ -240,6 +248,25 @@ struct MacroEngine { ::Common::unordered_map<u32, CacheInfo> macro_cache; ::Common::unordered_map<u32, std::vector<u32>> uploaded_macro_code; bool is_interpreted; + bool native_turing{}; + bool native_execution_failed{}; + u32 native_write_offset{}; + std::array<u32, 0x3000> native_code{}; + TuringMacroInterpreter native_interpreter; + std::vector<u32> native_fifo; + // Calls replayed instead of interpreted (macro_memo.h), and what the current call wrote. + u64 native_code_generation{}; + BasicMacroMemo<::Common::unordered_map<u64, u32, MacroMemoHash>> native_memo; + MacroMemoPairs native_writes; + u64 native_calls{}; + +}; + +struct NativeMacroContext { + MacroEngine engine{true}; + std::array<u32, 0x80> positions{}; + MacroEngine compute_engine{true}; + std::array<u32, 0x80> compute_positions{}; }; } // namespace Tegra diff --git a/src/video_core/macro_memo.h b/src/video_core/macro_memo.h new file mode 100644 index 0000000000..0e994fc78a --- /dev/null +++ b/src/video_core/macro_memo.h @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <algorithm> +#include <array> +#include <cstddef> +#include <cstdint> +#include <span> +#include <type_traits> +#include <utility> +#include <vector> + +namespace Tegra { + +using MacroMemoPairs = std::vector<std::pair<std::uint32_t, std::uint32_t>>; + +/// The memo's keys are already well mixed hashes: use them as they are (std::hash<u64> is a +/// byte-wise FNV-1a on MSVC, ~20% of a lookup). +struct MacroMemoHash { + using is_avalanching = std::true_type; + std::size_t operator()(std::uint64_t key) const noexcept { + return static_cast<std::size_t>(key); + } +}; + +// The effects of native macro calls that depend on nothing but their start address, their +// parameters and data RAM words they read (no register or memory reads), so that a repeated +// call replays its data RAM and method writes instead of being interpreted. In title 1's 3D +// scenes ~96% of the ~1,000,000 calls a second repeat an earlier one exactly, and they run 7 +// instructions on average: the per-call interpreter cost, not the instructions, was the time. +// `Map` maps a u64 (the call's hash) to a u32 (its record's offset in the arena). +template <typename Map> +class BasicMacroMemo { +public: + /// A remembered call's effects: (address, value) pairs, flattened. + struct Effects { + std::span<const std::uint32_t> data_ram_writes; + std::span<const std::uint32_t> writes; ///< (method, value) in order + }; + + struct Statistics { + std::uint64_t hits{}, misses{}, guard_failures{}, stores{}, overwrites{}, clears{}; + }; + + BasicMacroMemo() { + last_by_start.fill({0, NoRecord}); + } + + /// An earlier call with this start and these parameters whose data RAM reads still hold. + bool Find(std::uint32_t start, std::span<const std::uint32_t> parameters, + std::span<const std::uint32_t> data_ram, Effects& effects) { + // Nearly half the calls repeat the previous call of the same macro: try its record before + // hashing (no probe, and the record is still in the cache). + auto& last = last_by_start[start % last_by_start.size()]; + if (last.first == start && last.second != NoRecord && + Match(last.second, start, parameters, data_ram, effects)) { + ++statistics.hits; + return true; + } + const auto it = entries.find(Hash(start, parameters)); + if (it == entries.end()) { + ++statistics.misses; + return false; + } + if (!Match(it->second, start, parameters, data_ram, effects)) { + ++statistics.guard_failures; // or another call with the same hash + return false; + } + last = {start, it->second}; + ++statistics.hits; + return true; + } + +private: + static constexpr std::uint32_t NoRecord = ~0U; + + /// Whether the record at `offset` is this call with its data RAM reads holding; fills + /// `effects` if so. + bool Match(std::uint32_t offset, std::uint32_t start, std::span<const std::uint32_t> parameters, + std::span<const std::uint32_t> data_ram, Effects& effects) { + // Record: start, parameter count, read count, data RAM write count, write count, then + // the parameters and the (address, value) pairs. + const std::uint32_t* record = arena.data() + offset; + const std::uint32_t num_parameters = record[1], num_reads = record[2]; + const std::uint32_t num_data_ram_writes = record[3], num_writes = record[4]; + if (record[0] != start || num_parameters != parameters.size() || + !std::equal(parameters.begin(), parameters.end(), record + HeaderWords)) { + return false; // another call + } + const std::uint32_t* reads = record + HeaderWords + num_parameters; + for (std::uint32_t i = 0; i < num_reads; ++i) { + if (data_ram[reads[i * 2]] != reads[i * 2 + 1]) { + return false; + } + } + const std::uint32_t* data_ram_writes = reads + num_reads * 2; + effects.data_ram_writes = {data_ram_writes, num_data_ram_writes * 2}; + effects.writes = {data_ram_writes + num_data_ram_writes * 2, num_writes * 2}; + return true; + } + +public: + void Store(std::uint32_t start, std::span<const std::uint32_t> parameters, + const MacroMemoPairs& data_ram_reads, const MacroMemoPairs& data_ram_writes, + const MacroMemoPairs& writes) { + ++statistics.stores; + const std::uint64_t hash = Hash(start, parameters); + // A call stored again (its data RAM reads no longer held) usually has the same shape: + // rewrite its record in place, or the arena fills with stale copies. + if (const auto it = entries.find(hash); it != entries.end()) { + std::uint32_t* record = arena.data() + it->second; + if (record[0] == start && record[1] == parameters.size() && + record[2] == data_ram_reads.size() && record[3] == data_ram_writes.size() && + record[4] == writes.size() && + std::equal(parameters.begin(), parameters.end(), record + HeaderWords)) { + std::uint32_t* out = record + HeaderWords + parameters.size(); + for (const MacroMemoPairs* pairs : {&data_ram_reads, &data_ram_writes, &writes}) { + for (const auto& [address, value] : *pairs) { + *out++ = address; + *out++ = value; + } + } + ++statistics.overwrites; + return; + } + } + if (arena.size() >= MaxArenaWords) { + Clear(); + ++statistics.clears; + } + const auto offset = static_cast<std::uint32_t>(arena.size()); + arena.push_back(start); + arena.push_back(static_cast<std::uint32_t>(parameters.size())); + arena.push_back(static_cast<std::uint32_t>(data_ram_reads.size())); + arena.push_back(static_cast<std::uint32_t>(data_ram_writes.size())); + arena.push_back(static_cast<std::uint32_t>(writes.size())); + arena.insert(arena.end(), parameters.begin(), parameters.end()); + for (const MacroMemoPairs* pairs : {&data_ram_reads, &data_ram_writes, &writes}) { + for (const auto& [address, value] : *pairs) { + arena.push_back(address); + arena.push_back(value); + } + } + entries.insert_or_assign(hash, offset); + last_by_start[start % last_by_start.size()] = {start, offset}; + } + + const Statistics& GetStatistics() const { + return statistics; + } + + std::size_t Size() const { + return entries.size(); + } + + std::size_t ArenaWords() const { + return arena.size(); + } + + /// Forgets every call when the macro code is no longer the code they ran. + void Synchronize(std::uint64_t code_generation) { + if (code_generation != generation) { + generation = code_generation; + Clear(); + } + } + +private: + static constexpr std::size_t HeaderWords = 5; + static constexpr std::size_t MaxArenaWords = std::size_t{1} << 22; // 16 MiB + + static std::uint64_t Hash(std::uint32_t start, std::span<const std::uint32_t> parameters) { + std::uint64_t hash = 0x9e3779b97f4a7c15ULL ^ (std::uint64_t{start} << 32) ^ parameters.size(); + for (const std::uint32_t word : parameters) { + hash = (hash ^ word) * 0xff51afd7ed558ccdULL; + hash ^= hash >> 32; + } + return hash; + } + + void Clear() { + arena.clear(); + entries.clear(); + last_by_start.fill({0, NoRecord}); + } + + Map entries; + std::vector<std::uint32_t> arena; + /// The last record found or stored per macro start (direct mapped). + std::array<std::pair<std::uint32_t, std::uint32_t>, 256> last_by_start{}; + std::uint64_t generation{}; + Statistics statistics; +}; + +} // namespace Tegra diff --git a/src/video_core/macro_turing.h b/src/video_core/macro_turing.h new file mode 100644 index 0000000000..600db48119 --- /dev/null +++ b/src/video_core/macro_turing.h @@ -0,0 +1,274 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +// Instruction semantics adapted from Mesa mme_tu104_sim.c and mme_tu104.c: +// Copyright © 2022 Collabora Ltd. SPDX-License-Identifier: MIT +#pragma once + +#include <algorithm> +#include <array> +#include <bit> +#include <cstdint> +#include <span> +#include <string_view> +#include <vector> + +namespace Tegra { + +// TU104/GA10x MME has two ALUs and two outputs per 96-bit instruction. +class TuringMacroInterpreter { +public: + using Word = std::uint32_t; + struct Result { + bool success; + Word pc; + std::size_t consumed; + std::string_view error; + }; + + // One instruction's fields, decoded once and reused while its words stay the same (macro + // code runs thousands of times a frame; decoding it every time was most of the interpreter). + struct Decoded { + std::array<Word, 3> raw{~0U, ~0U, ~0U}; + std::array<Word, 2> imm, op, dst, src_a, src_b, out_method, out_emit; + Word pred_reg, pred_mode; + // Lane/output enable masks when the predicate register is non-zero / zero. + Word mask_true, mask_false; + bool end; + std::array<bool, 2> load; + }; + + static Decoded Decode(const std::array<Word, 3>& bits) { + const auto field = [&](Word low, Word count) { + std::uint64_t value = bits[low / 32]; + if (low / 32 < 2) value |= std::uint64_t{bits[low / 32 + 1]} << 32; + return Word((value >> (low % 32)) & ((std::uint64_t{1} << count) - 1)); + }; + Decoded d; + d.raw = bits; + d.imm = {field(30, 16), field(66, 16)}; + d.op = {field(10, 5), field(46, 5)}; + d.dst = {field(15, 5), field(51, 5)}; + d.src_a = {field(20, 5), field(56, 5)}; + d.src_b = {field(25, 5), field(61, 5)}; + d.out_method = {field(82, 3), field(89, 3)}; + d.out_emit = {field(85, 4), field(92, 4)}; + d.pred_reg = field(5, 5); + d.pred_mode = field(1, 4); + d.end = field(0, 1) != 0; + constexpr std::array<std::string_view, 16> predicates{ + "UUUU", "TTTT", "FFFF", "TTUU", "FFUU", "TFUU", "TUUU", "FUUU", + "UUTT", "UUTF", "UUTU", "UUFT", "UUFF", "UUFU", "UUUT", "UUUF"}; + d.mask_true = d.mask_false = 15; + if (d.pred_mode) { + d.mask_true = d.mask_false = 0; + for (Word i = 0; i < 4; ++i) { + if (predicates[d.pred_mode][i] != 'T') d.mask_true |= 1U << i; + if (predicates[d.pred_mode][i] != 'F') d.mask_false |= 1U << i; + } + } + for (Word i = 0; i < 2; ++i) { + const Word reg = 28 + i, out = 3 + i; + d.load[i] = d.pred_reg == reg || d.src_a[0] == reg || d.src_a[1] == reg || + d.src_b[0] == reg || d.src_b[1] == reg || d.out_method[0] == out || + d.out_method[1] == out || d.out_emit[0] == out || d.out_emit[1] == out; + } + return d; + } + + template <typename Read, typename Emit> + // `parameters` is the load FIFO; `emit` may append to it (MME_DMA_READ_FIFOED). + Result Execute(std::span<const Word> code, Word start, + std::vector<Word>& parameters, Read read, Emit emit) { + std::array<Word, 24> regs{}; + Word initialized{}, carry{}, method{}, increment{}; + Word pc = start, loop_count{}, loop_start{}, loop_end{}; + std::size_t consumed{}; + bool end_next{}, stopped{}; + std::string_view error; + const auto fail = [&](std::string_view message) { if (error.empty()) error = message; }; + if (decoded.size() < code.size() / 3) decoded.resize(code.size() / 3); + read_registers = false; + reread_data_ram = false; + data_ram_reads.clear(); + data_ram_writes.clear(); + for (Word steps = 0; steps < 1000000; ++steps) { + if (static_cast<std::size_t>(pc) * 3 + 2 >= code.size()) { + fail("instruction outside uploaded RAM"); + break; + } + const std::array<Word, 3> bits{code[pc * 3 + 2], code[pc * 3 + 1], code[pc * 3]}; + Decoded& d = decoded[pc]; + if (d.raw != bits) d = Decode(bits); + std::array<Word, 2> load{}, alu{}; + for (Word i = 0; i < 2; ++i) { + if (d.load[i]) { + if (consumed >= parameters.size()) { fail("parameter FIFO underflow"); break; } + load[i] = parameters[consumed++]; + } + } + // Both ALUs read the register file before either destination is committed: their + // operands are read up front. Carry chaining remains explicit between the lanes. + const auto get = [&](Word reg, Word lane) -> Word { + if (reg < 24) { + if (!(initialized & (1U << reg))) fail("uninitialized register read"); + return regs[reg]; + } + switch (reg) { + case 24: return 0; + case 25: return Word(std::int32_t(std::int16_t(d.imm[lane]))); + case 26: return Word(std::int32_t(std::int16_t(d.imm[1 - lane]))); + case 27: return (d.imm[0] << 16) | d.imm[1]; + case 28: return load[0]; + case 29: return load[1]; + default: fail("invalid source register"); return 0; + } + }; + const Word mask = d.pred_mode ? (get(d.pred_reg, 0) ? d.mask_true : d.mask_false) : 15; + std::array<Word, 2> xs{}, ys{}; + for (Word lane = 0; lane < 2; ++lane) { + if (mask & (1U << lane)) { + xs[lane] = get(d.src_a[lane], lane); + ys[lane] = get(d.src_b[lane], lane); + } + } + Word next_pc = pc + 1; + bool branch_cancels_end{}; + for (Word lane = 0; lane < 2 && error.empty(); ++lane) { + if (!(mask & (1U << lane))) continue; + const Word x = xs[lane], y = ys[lane]; + const Word imm = d.imm[lane]; + Word value{}; + const auto condition = [&](Word comparison) { + switch (comparison) { + case 17: case 25: return std::int32_t(x) < std::int32_t(y); + case 18: case 26: return x < y; + case 19: case 27: return std::int32_t(x) <= std::int32_t(y); + case 20: case 28: return x <= y; + default: return x == y; + } + }; + switch (d.op[lane]) { + case 0: value = x + y; carry = value < x; break; + case 1: value = x + y + carry; break; + case 2: value = x - y; carry = value > x; break; + case 3: value = x - y - carry; break; + case 4: case 6: { + const std::uint64_t a = d.op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(x))) : x; + const std::uint64_t b = d.op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(y))) : y; + const auto product = a * b; + value = Word(product); carry = Word(product >> 32); break; + } + case 5: value = carry; break; + case 7: + // EXTENDED: x = 0x1000 is the load barrier (Mesa mme_tu104_load_barrier), + // nothing to do in a sequential interpreter. + if (x != 0x1000) fail("unsupported extended operation"); + break; + case 8: value = Word(std::countl_zero(x)); break; + case 9: value = x << (y & 31); break; + case 10: value = x >> (y & 31); break; + case 11: value = Word(std::int32_t(x) >> (y & 31)); break; + case 12: value = x & y; break; + case 13: value = ~(x & y); break; + case 14: value = x | y; break; + case 15: value = x ^ y; break; + case 16: { + const Word dest = (imm >> 10) & 63, count = (imm >> 5) & 31; + const Word source = imm & 31; + if (dest >= 32) { fail("invalid merge destination"); break; } + const Word bits_mask = (1U << count) - 1; + value = (x & ~(bits_mask << dest)) | (((y >> source) & bits_mask) << dest); + break; + } + case 17: case 18: case 19: case 20: case 21: + value = condition(d.op[lane]) ? ~0U : 0; break; + case 22: value = read((x + y) & 0xffff); read_registers = true; break; + case 23: + if (loop_count || imm < 2) { fail("invalid loop"); break; } + // The native SDK emits zero-count loops around FIFO reads. + // Unlike Mesa's test simulator, these must skip the body. + if (x == 0) { next_pc = pc + imm; break; } + loop_count = x - 1; loop_start = pc; loop_end = pc + imm - 1; + break; + case 24: next_pc = pc + (imm & 0x7fff); break; + case 25: case 26: case 27: case 28: case 29: + if (condition(d.op[lane]) == bool(imm & 0x8000)) { + const auto offset = std::int32_t(imm << 19) >> 19; + if (offset == -4096) stopped = true; + else { + next_pc = Word(std::int32_t(pc) + offset); + // Native back edges use bit 14 to suppress the + // fallthrough termination when the branch is taken. + branch_cancels_end = (imm & 0x4000) != 0; + } + } + break; + case 30: + if (x >= active_dram->size()) { fail("data RAM read out of bounds"); break; } + value = (*active_dram)[x]; + if (std::ranges::find(data_ram_writes, x, &DataRamAccess::first) != data_ram_writes.end()) { + reread_data_ram = true; + } + data_ram_reads.emplace_back(x, value); + break; + case 31: + if (x >= active_dram->size()) { fail("data RAM write out of bounds"); break; } + (*active_dram)[x] = y; + data_ram_writes.emplace_back(x, y); + break; + } + alu[lane] = value; + if (d.dst[lane] < 24) { regs[d.dst[lane]] = value; initialized |= 1U << d.dst[lane]; } + else if (d.dst[lane] != 24) fail("invalid destination register"); + } + const auto output = [&](Word selector) -> Word { + switch (selector) { + case 1: return alu[0]; case 2: return alu[1]; + case 3: return load[0]; case 4: return load[1]; + case 5: return d.imm[0]; case 6: return d.imm[1]; + case 8: return d.imm[0] >> 12; case 9: return d.imm[1] >> 12; + case 10: return (d.imm[0] << 16) | d.imm[1]; + default: fail("invalid output selector"); return 0; + } + }; + for (Word lane = 0; lane < 2 && error.empty(); ++lane) { + if (!(mask & (4U << lane))) continue; + if (d.out_method[lane]) { + const Word address = output(d.out_method[lane]); + method = address & 0xfff; increment = (address >> 12) & 15; + } + if (d.out_emit[lane]) { + const Word value = output(d.out_emit[lane]); + if (error.empty() && !emit(method, value)) fail("unsupported emitted method"); + method += increment; + } + } + if (!error.empty()) break; + if (end_next || stopped) return {true, pc, consumed, {}}; + end_next = d.end && !branch_cancels_end; + if (loop_count && pc == loop_end) { --loop_count; next_pc = loop_start + 1; } + pc = next_pc; + } + if (error.empty()) error = "instruction limit exceeded"; + return {false, pc, consumed, error}; + } + + using DataRam = std::array<Word, 0xc00>; + DataRam dram{}; + // The data RAM DREAD/DWRITE use: a channel's own copy when the caller sets one. Macro code is + // shared by the channels of a process, but each channel keeps its own data RAM (title 1's + // channels store their own QMD ring base in word 0x27). + DataRam* active_dram{&dram}; + // Decoded instructions by PC, re-decoded whenever their words differ. + std::vector<Decoded> decoded; + // What the last Execute read besides its parameters, for the macro memo: whether it read a + // register (MME_READ), the data RAM words it read and wrote (address, value) in order, and + // whether it read a word it had written itself. + using DataRamAccess = std::pair<Word, Word>; + bool read_registers{}; + bool reread_data_ram{}; + std::vector<DataRamAccess> data_ram_reads; + std::vector<DataRamAccess> data_ram_writes; +}; + +} // namespace Tegra diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 8ffdcb6276..0eae8fac24 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -22,6 +22,10 @@ #include "video_core/texture_cache/util.h" namespace Tegra { +bool MemoryManager::IsShuttingDown() const { + return system.IsShuttingDown(); +} + using Tegra::Memory::GuestMemoryFlags; std::atomic<size_t> MemoryManager::unique_identifier_generator{}; @@ -56,6 +60,10 @@ MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, GPU : MemoryManager(system_, system_.Host1x().MemoryManager(), address_space_bits_, split_address_, big_page_bits_, page_bits_) {} +void MemoryManager::SetNativeSyncpointShim(GPUVAddr address) { + native_syncpoint_shim.store(address, std::memory_order_release); +} + MemoryManager::~MemoryManager() = default; MemoryManager::EntryType MemoryManager::GetEntry(size_t position, bool is_big_page) const { @@ -231,6 +239,15 @@ std::optional<DAddr> MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t s template <typename T> T MemoryManager::Read(GPUVAddr addr) const { + if constexpr (sizeof(T) == sizeof(u32)) { + const auto shim = native_syncpoint_shim.load(std::memory_order_acquire); + if (shim && addr >= shim && addr - shim < NumShimSyncpoints * sizeof(u32) && + Common::IsAligned(addr - shim, sizeof(u32))) { + return static_cast<T>(system.Host1x().GetSyncpointManager().GetGuestSyncpointValue( + static_cast<u32>((addr - shim) / sizeof(u32)))); + } + } + if (auto page_pointer{GetPointer(addr)}; page_pointer) { // NOTE: Avoid adding any extra logic to this fast-path block T value; @@ -238,7 +255,10 @@ T MemoryManager::Read(GPUVAddr addr) const { return value; } - ASSERT(false); + static std::atomic<u32> reported{}; + if (reported.fetch_add(1) < 16) { + LOG_ERROR(HW_GPU, "Read{} of unmapped GPU address {:#x}", sizeof(T) * 8, addr); + } return {}; } @@ -390,6 +410,16 @@ void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::siz } void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, const std::size_t size) const { + // Small reads within one 4 KiB page (texture and sampler descriptors, several per draw): one + // translation and a copy, as Read<T> does, instead of the page walk (~3% of the GPU thread + // in title 1). Anything else, unmapped or sparse pages included, takes the full path. + constexpr std::size_t SmallPage = 0x1000; + if (size <= SmallPage && (gpu_src_addr & (SmallPage - 1)) + size <= SmallPage) { + if (const u8* const pointer = GetPointer(gpu_src_addr)) { + std::memcpy(dest_buffer, pointer, size); + return; + } + } ReadBlockImpl(gpu_src_addr, dest_buffer, size, VideoCommon::CacheType::None, true); } diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index 08978567d2..105cb14e23 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -43,6 +43,19 @@ public: u64 big_page_bits_ = 16, u64 page_bits_ = 12); ~MemoryManager(); + // Switch 2: GPU address of the read-only syncpoint shim (u32 per syncpoint); 0 = none. + void SetNativeSyncpointShim(GPUVAddr address); + static constexpr u32 NumShimSyncpoints = 192; // Tegra::Host1x::SyncpointManager + + /// Switch 2 lays out block-linear surfaces with the console's GOB swizzle. + void EnableNativeTiling() { + native_tiling = true; + } + bool UsesNativeTiling() const { + return native_tiling; + } + bool IsShuttingDown() const; + static constexpr bool HAS_FLUSH_INVALIDATION = true; inline size_t GetID() const noexcept { @@ -178,6 +191,7 @@ private: GPUVAddr gpu_addr, std::size_t size, boost::container::small_vector<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result) const; + std::atomic<GPUVAddr> native_syncpoint_shim{}; Core::System& system; MaxwellDeviceMemoryManager& memory; @@ -220,6 +234,7 @@ private: boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash{}; boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash2{}; + bool native_tiling{}; mutable std::mutex guard; static constexpr size_t continuous_bits = 64; diff --git a/src/video_core/native_qmd.h b/src/video_core/native_qmd.h new file mode 100644 index 0000000000..15c59cb7c6 --- /dev/null +++ b/src/video_core/native_qmd.h @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once +#include <array> +#include <cstdint> +#include <functional> +#include <string_view> +#include <type_traits> +#include <unordered_map> + +namespace Tegra { + +// Native QMD 3.0 dependency scheduling; nonempty grids require a dispatch backend. +class NativeQmdScheduler { +public: + using Word = std::uint32_t; + using Address = std::uint64_t; + using Descriptor = std::array<Word, 64>; + struct RejectShaderDispatch { bool operator()(const Descriptor&) const { return false; } }; + // Optional diagnostics: called with "load"/"count"/"defer"/"run" events. + std::function<void(std::string_view, Address, std::int32_t)> trace; + template <typename Read, typename Release> + std::string_view Signal(Address address, Word argument, Read read, Release release) { + return Signal(address, argument, read, release, RejectShaderDispatch{}); + } + template <typename Read, typename Release, typename Dispatch> + std::string_view Signal(Address address, Word argument, Read read, Release release, Dispatch dispatch) { + struct Pending { Address address; Word action; Word count; }; + std::string_view error; + std::array<Pending, 256> pending{}; + std::size_t count = 1; + pending[0] = {address, argument & 15, ((argument >> 14) & 0x3ff) + 1}; + for (std::size_t i = 0; i < count; ++i) { + const auto request = pending[i]; + if (states.size() >= 4096 && !states.contains(request.address)) return "QMD cache limit"; + auto& state = states[request.address]; + if (request.action == 3 || request.action == 2) { + if (request.action == 3 || !state.loaded) { + if (!read(request.address, state.descriptor)) return "unmapped QMD"; + const auto& qmd = state.descriptor; + if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue"; + if constexpr (std::is_same_v<Dispatch, RejectShaderDispatch>) { + if (qmd[12] && (qmd[13] & 0xffff) && (qmd[14] & 0xffff)) + return "native shader dispatch required"; + } + state.dependencies = std::int32_t((qmd[4] >> 14) & 0xffff) + state.deferred; + if (trace) trace("base", request.address, std::int32_t((qmd[4] >> 14) & 0xffff)); + if (trace) trace("load", request.address, state.dependencies); + state.deferred = 0; + state.loaded = true; + state.completed = false; + } + state.scheduled = true; + } else if (request.action == 7 || request.action == 12) { + const auto delta = request.action == 7 ? -std::int32_t(request.count) : std::int32_t(request.count); + if (!state.loaded || state.completed) { + state.deferred += delta; + if (trace) trace("defer", request.address, state.deferred); + continue; + } + state.dependencies += delta; + if (trace) trace("count", request.address, state.dependencies); + } else if (request.action == 0 || request.action == 1) { + continue; + } else { + return "unsupported PCAS action"; + } + if (!state.loaded || !state.scheduled || state.completed || state.dependencies > 0) continue; + if (state.dependencies < 0) return "QMD dependency underflow"; + // The SDK patches release payloads while a scheduled QMD is + // waiting for its remaining dependencies. Fetch them at execution. + if (!read(request.address, state.descriptor)) return "unmapped QMD"; + const auto qmd = state.descriptor; + if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue"; + // Validate the release slots before making any visible. + for (const Word base : ReleaseSlots) { + const Word flags = qmd[base + 1]; + if (!(flags & (1U << 23))) continue; + const bool add = (flags & (1U << 26)) != 0; + if ((flags & (0x3bU << 24)) || (flags >> 30) != 1 || + (add && (flags & (7U << 20)))) return "unsupported QMD release format"; + if (add) { + if constexpr (!std::is_invocable_r_v<bool, Release, Address, Word, Word>) + return "unsupported QMD reduction callback"; + } + } + // A program Eden can't run is skipped, but the QMD still completes: its releases + // are what other queues and the CPU wait on (the console never fails a launch). + const bool dispatched = !(qmd[12] && (qmd[13] & 0xffff) && (qmd[14] & 0xffff)) || dispatch(qmd); + if (!dispatched) error = "native shader dispatch required"; + if (trace) trace("run", request.address, dispatched ? 1 : 0); + state.completed = true; + for (const Word base : ReleaseSlots) { + const Word flags = qmd[base + 1]; + if (!(flags & (1U << 23))) continue; + const Address destination = (Address{flags & 0xff} << 32) | qmd[base]; + bool released{}; + if constexpr (std::is_invocable_r_v<bool, Release, Address, Word, Word>) { + released = release(destination, qmd[base + 2], flags); + } else { + released = release(destination, qmd[base + 2]); + } + if (!released) return "unmapped QMD release"; + } + if (qmd[16] & 1) { + const Word action = (qmd[16] >> 1) & 7; + if (action != 4 && action != 3 && action != 1) return "unsupported dependent QMD action"; + if (count == pending.size()) return "QMD chain limit"; + pending[count++] = {Address{qmd[15]} << 8, action == 4 ? 7U : action == 3 ? 3U : 2U, 1}; + } + if (qmd[16] & (1U << 5)) return "second dependent QMD unsupported"; + } + return error; + } +private: + // Release slots (address low, flags | address high, payload). NVN2 patches a third one at + // word 52 for queue syncs (e.g. a compute queue releasing a semaphore another queue waits on). + static constexpr std::array<Word, 3> ReleaseSlots{24, 28, 52}; + + struct State { + Descriptor descriptor{}; + std::int32_t dependencies{}; + std::int32_t deferred{}; + bool loaded{}; + bool scheduled{}; + bool completed{}; + }; + // One table for every channel: a QMD's dependency counter is state of the QMD itself, and + // title 1 schedules a QMD on one channel while another decrements it (a decrement that + // arrived first was kept by the other channel's scheduler and the QMD never launched). + static inline std::unordered_map<Address, State> states; +}; +} // namespace Tegra diff --git a/src/video_core/native_query_trace.h b/src/video_core/native_query_trace.h new file mode 100644 index 0000000000..036c3de936 --- /dev/null +++ b/src/video_core/native_query_trace.h @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include <cstdlib> +#include <fstream> +#include <iomanip> +#include <mutex> +#include "common/common_types.h" + +namespace VideoCommon { +namespace NativeQueryDiagnostics { +struct State { + State() : file{"native-query-trace.csv", std::ios::trunc} { + file << "event,asid,gpu,cpu,type,value,flags\n"; + } + std::mutex mutex; + std::ofstream file; + size_t records{}; +}; +inline State& GetState() { + static State state; + return state; +} +} // namespace NativeQueryDiagnostics + +// Events: 0=request, 1=written, 2=invalidated, 3=unsynchronized, +// 4=channel suspended, 5=channel resumed, 6=buffer readback, +// 7/8=QMD release slots observed, 9=PCAS request, +// 10=DMA destination (value=source, flags=line bytes), 11=inline write. +// Keep this opt-in and bounded. +inline void TraceNativeQuery(u32 event, u64 asid, u64 gpu, u64 cpu, u32 type, u64 value, u32 flags) { + static const bool diagnostics = std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") != nullptr; + if (!diagnostics) return; + auto& state = NativeQueryDiagnostics::GetState(); + const std::scoped_lock lock{state.mutex}; + if (state.records++ < 1000000 || event == 4 || event == 5) { + state.file << event << ',' << asid << ',' << std::hex << gpu << ',' << cpu << ',' + << type << ',' << value << ',' << flags << std::dec << '\n'; + } + if (event == 4 || event == 5) state.file.flush(); +} +} // namespace VideoCommon diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h new file mode 100644 index 0000000000..5b6d89abdd --- /dev/null +++ b/src/video_core/nx2.h @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <algorithm> +#include <array> +#include <atomic> +#include <mutex> +#include <vector> + +#include "common/common_types.h" + +namespace Tegra { + +// Set once a Switch 2 (Ampere, NVN2) driver is talking to the GPU. Its command streams use +// Ampere classes and the Turing+ macro ISA, which Eden's Maxwell engines don't implement. +inline std::atomic<bool> g_nx2_ampere{false}; + +// NVN2 queues release their progress values into the second page of a two-page GPU-accessible +// region and GPU acquires read the same offsets in the first page (the CPU polls the second). +// Something on the console copies one into the other; until that is known, the puller mirrors +// releases that land in a second page. Every queue has such a region, so keep a small table. +inline constexpr size_t Nx2MaxSemaphoreRegions = 32; +inline std::array<std::atomic<u64>, Nx2MaxSemaphoreRegions> g_nx2_semaphore_regions{}; // starts + +inline void Nx2AddSemaphoreRegion(u64 start) { + for (auto& region : g_nx2_semaphore_regions) { + if (region.load() == start) { + return; + } + } + for (auto& region : g_nx2_semaphore_regions) { + u64 expected = 0; + if (region.compare_exchange_strong(expected, start)) { + return; + } + } +} + +/// The region's memory was destroyed: stop mirroring into it. +inline void Nx2RemoveSemaphoreRegion(u64 start) { + for (auto& region : g_nx2_semaphore_regions) { + u64 expected = start; + region.compare_exchange_strong(expected, 0); + } +} + +/// GPU VA in the first page that mirrors a release at `address`, or 0. +inline u64 Nx2SemaphoreMirror(u64 address) { + for (const auto& region : g_nx2_semaphore_regions) { + const u64 start = region.load(std::memory_order_relaxed); + if (start == 0) { + continue; + } + if (address >= start + 0x1000 && address < start + 0x2000) { + return address - 0x1000; + } + } + return 0; +} + +/// Channels suspended on a semaphore acquire. A console channel polls and sees every release; +/// Eden writes releases in bursts when fences signal and resumes waiters only between command +/// lists, so an acquire-equal could miss its value. Writes latch the waiters they satisfy. +class Nx2SemaphoreWaiters { +public: + // Puller GpuSemaphoreOperation values. + static constexpr u32 AcquireEqual = 0x1; + static constexpr u32 AcquireGequal = 0x4; + static constexpr u32 AcquireMask = 0x8; + + static Nx2SemaphoreWaiters& Get() { + static Nx2SemaphoreWaiters waiters; + return waiters; + } + + void Add(const void* owner, u64 device_address, u32 value, u32 operation) { + std::scoped_lock lk{mutex}; + std::erase_if(waiters, [owner](const Waiter& w) { return w.owner == owner; }); + waiters.push_back({owner, device_address, value, operation, false}); + count.store(waiters.size(), std::memory_order_release); + } + + void Remove(const void* owner) { + if (count.load(std::memory_order_acquire) == 0) { + return; + } + std::scoped_lock lk{mutex}; + std::erase_if(waiters, [owner](const Waiter& w) { return w.owner == owner; }); + count.store(waiters.size(), std::memory_order_release); + } + + [[nodiscard]] bool Latched(const void* owner) { + if (count.load(std::memory_order_acquire) == 0) { + return false; + } + std::scoped_lock lk{mutex}; + return std::ranges::any_of(waiters, + [owner](const Waiter& w) { return w.owner == owner && w.latched; }); + } + + /// A GPU write of `value` to the u32 at `device_address`. + void OnWrite(u64 device_address, u32 value) { + if (count.load(std::memory_order_acquire) == 0) { + return; + } + std::scoped_lock lk{mutex}; + for (auto& w : waiters) { + if (w.address == device_address && !w.latched && Matches(w, value)) { + w.latched = true; + } + } + } + +private: + struct Waiter { + const void* owner; + u64 address; + u32 value; + u32 operation; + bool latched; + }; + + static bool Matches(const Waiter& w, u32 current) { + switch (w.operation) { + case AcquireEqual: + return current == w.value; + case AcquireGequal: + return current >= w.value; + case AcquireMask: + return (current & w.value) != 0; + default: + return false; + } + } + + std::mutex mutex; + std::vector<Waiter> waiters; + std::atomic<size_t> count{}; +}; + +} // namespace Tegra diff --git a/src/video_core/nx2_profile.h b/src/video_core/nx2_profile.h new file mode 100644 index 0000000000..a162ce16a3 --- /dev/null +++ b/src/video_core/nx2_profile.h @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <array> +#include <chrono> +#include <cstdlib> +#include <string> + +#include "common/common_types.h" +#include "common/logging.h" +#include "common/cpu_features.h" + +// EDEN_NX2_GPU_PROFILE: time where the GPU thread spends its time. Each zone counts its own +// (exclusive) time, nested zones excluded; every 10 s the thread logs each zone's share. +namespace VideoCommon::Nx2Profile { + +enum class Zone : u32 { + Other, ///< GPU thread time outside every zone below (register writes, command parsing) + Idle, + PendingSleep, + ResumePending, + Push, + Tick, + Run, + FlushInvalidate, + Macro, + Draw, + Clear, + Compute, + FlushWork, + PipelineLookup, + Configure, + SyncDescriptors, + ReadHandles, + FillImageViews, + TextureBuffers, + UpdateBuffers, + DescriptorAcquire, + BindStageBuffers, + PushDescriptors, + RenderTargets, + ConfigureDraw, + DynamicState, + DrawRecord, + TextureUpload, + Count, +}; + +inline constexpr std::array<const char*, static_cast<size_t>(Zone::Count)> zone_names{ + "other", "idle", "pending sleep", "resume pending", "push", + "tick", "run", "flush/inval", "macro", "draw", + "clear", "compute", "flush work", "pipeline lookup", "configure", + "sync descriptors", "read handles", "fill image views", "texture buffers", "update buffers", + "descriptor acquire", "bind stage buffers", "push descriptors", "render targets", + "configure draw", "dynamic state", "draw record", "texture upload", +}; + +inline bool Enabled() { + static const bool enabled = std::getenv("EDEN_NX2_GPU_PROFILE") != nullptr; + return enabled; +} + +// Nanoseconds; from the host's own counter where the wall clock reads it directly (the zones +// run ~150,000 times a second, and QueryPerformanceCounter cost ~5% of the GPU thread). +inline u64 Now() { + return static_cast<u64>(Common::g_wall_clock.GetTimeNS().count()); +} + +struct ThreadData { + std::array<u64, static_cast<size_t>(Zone::Count)> exclusive_ns{}; + std::array<u64, static_cast<size_t>(Zone::Count)> count{}; + u64 child_ns{}; ///< Time of the zones nested in the current one + u64 window_start{}; +}; + +inline thread_local ThreadData t_data; + +class Scope { +public: + explicit Scope(Zone zone_) : zone{zone_}, enabled{Enabled()} { + if (!enabled) { + return; + } + parent_child_ns = t_data.child_ns; + t_data.child_ns = 0; + start = Now(); + } + + ~Scope() { + if (!enabled) { + return; + } + const u64 elapsed = Now() - start; + auto& data = t_data; + const auto index = static_cast<size_t>(zone); + data.exclusive_ns[index] += elapsed - data.child_ns; + ++data.count[index]; + data.child_ns = parent_child_ns + elapsed; + } + + Scope(const Scope&) = delete; + Scope& operator=(const Scope&) = delete; + +private: + Zone zone; + bool enabled; + u64 start{}; + u64 parent_child_ns{}; +}; + +/// Called by the GPU thread between commands: logs and resets the totals every 10 s. +inline void Report() { + if (!Enabled()) { + return; + } + auto& data = t_data; + const u64 now = Now(); + if (data.window_start == 0) { + data.window_start = now; + data.child_ns = 0; + return; + } + const u64 window = now - data.window_start; + if (window < 10'000'000'000ULL) { + return; + } + // Time in no zone at all is the wall time minus the zones' totals. + u64 zoned{}; + for (size_t i = 1; i < data.exclusive_ns.size(); ++i) { + zoned += data.exclusive_ns[i]; + } + data.exclusive_ns[0] = window > zoned ? window - zoned : 0; + std::string line; + for (size_t i = 0; i < data.exclusive_ns.size(); ++i) { + if (data.exclusive_ns[i] == 0 && data.count[i] == 0) { + continue; + } + // Per second of wall time: milliseconds spent and calls made. + const double ms = static_cast<double>(data.exclusive_ns[i]) * 1000.0 / window; + const double calls = static_cast<double>(data.count[i]) * 1e9 / window; + line += fmt::format(" | {} {:.1f} ms ({:.0f}/s)", zone_names[i], ms, calls); + } + LOG_INFO(HW_GPU, "GPU thread per second:{}", line); + data = {}; + data.window_start = now; +} + +} // namespace VideoCommon::Nx2Profile diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index b8c1bc4f45..245ab1996c 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -7,6 +7,7 @@ #pragma once #include <array> +#include <cstdlib> #include <deque> #include <memory> #include <mutex> @@ -22,6 +23,7 @@ #include "video_core/gpu.h" #include "video_core/host1x/gpu_device_memory_manager.h" #include "video_core/memory_manager.h" +#include "video_core/nx2.h" #include "video_core/query_cache/bank_base.h" #include "video_core/query_cache/query_base.h" #include "video_core/query_cache/query_cache_base.h" @@ -233,6 +235,16 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type QueryPropertiesFlags flags, u32 payload, u32 subreport) { const bool has_timestamp = True(flags & QueryPropertiesFlags::HasTimeout); const bool is_fence = True(flags & QueryPropertiesFlags::IsAFence); + static const GPUVAddr watch = [] { + const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); + return value ? GPUVAddr{std::strtoull(value, nullptr, 16)} : GPUVAddr{}; + }(); + if (static const bool trace = std::getenv("EDEN_NX2_QUERY_TRACE") != nullptr; + trace || (watch != 0 && (addr & ~GPUVAddr{0xf}) == watch)) { + LOG_INFO(HW_GPU, "nx2 report addr={:#x} type={} flags={:#x} payload={:#x} as={} device={:#x}", addr, + static_cast<u32>(counter_type), static_cast<u32>(flags), payload, gpu_memory->GetID(), + gpu_memory->GpuToCpuAddress(addr).value_or(0)); + } size_t streamer_id = static_cast<size_t>(counter_type); auto* streamer = impl->streamers[streamer_id]; if (streamer == nullptr) [[unlikely]] { @@ -262,7 +274,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type u8* pointer_timestamp = impl->device_memory.template GetPointer<u8>(cpu_addr + 8); bool is_synced = (Settings::IsGPUFenceBehaviorDefault() ? !Settings::IsGPULevelHigh() : !Settings::IsGPUFenceBehaviorBalanced() && !Settings::IsGPUFenceBehaviorAccurate()) && is_fence; std::function<void()> operation([this, is_synced, streamer, query_base = query, query_location, - pointer, pointer_timestamp] { + pointer, pointer_timestamp, cpu_addr] { if (True(query_base->flags & QueryFlagBits::IsInvalidated)) { if (!is_synced) [[likely]] { impl->pending_unregister.push_back(query_location); @@ -287,6 +299,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type u32 value = static_cast<u32>(query_base->value); std::memcpy(pointer, &value, sizeof(value)); } + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, static_cast<u32>(query_base->value)); if (!is_synced) [[likely]] { impl->pending_unregister.push_back(query_location); } @@ -303,6 +316,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type } else { std::memcpy(pointer, &payload, sizeof(payload)); } + Tegra::Nx2SemaphoreWaiters::Get().OnWrite(cpu_addr, payload); streamer->Free(new_query_id); return; } diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 1d77d28c46..4e0972d1d2 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -716,7 +716,7 @@ void RasterizerOpenGL::AccelerateInlineToMemory(GPUVAddr address, size_t copy_si } gpu_memory->WriteBlockUnsafe(address, memory.data(), copy_size); { - std::unique_lock<std::recursive_mutex> lock{buffer_cache.mutex}; + std::unique_lock lock{buffer_cache.mutex}; if (!buffer_cache.InlineMemory(*cpu_addr, copy_size, memory)) { buffer_cache.WriteMemory(*cpu_addr, copy_size); } diff --git a/src/video_core/renderer_opengl/util_shaders.cpp b/src/video_core/renderer_opengl/util_shaders.cpp index 7ced8c77b4..5ed96a2037 100644 --- a/src/video_core/renderer_opengl/util_shaders.cpp +++ b/src/video_core/renderer_opengl/util_shaders.cpp @@ -96,6 +96,7 @@ void UtilShaders::ASTCDecode(Image& image, const StagingBufferMap& map, glUniform1ui(4, params.x_shift); glUniform1ui(5, params.block_height); glUniform1ui(6, params.block_height_mask); + glUniform1ui(7, params.native_tiling); // ASTC texture data glBindBufferRange(GL_SHADER_STORAGE_BUFFER, BINDING_INPUT_BUFFER, map.buffer, input_offset, diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp index 9f4aaf390f..8a11c356ff 100644 --- a/src/video_core/renderer_vulkan/present/layer.cpp +++ b/src/video_core/renderer_vulkan/present/layer.cpp @@ -349,7 +349,7 @@ void Layer::UpdateRawImage(const Tegra::FramebufferConfig& framebuffer, size_t i if (host_ptr) { Tegra::Texture::UnswizzleTexture( mapped_span.subspan(image_offset, linear_size), std::span(host_ptr, tiled_size), - bytes_per_pixel, framebuffer.width, framebuffer.height, 1, block_height_log2, 0); + bytes_per_pixel, framebuffer.width, framebuffer.height, 1, block_height_log2, 0, 1, framebuffer.native_tiling); buffer.Flush(); // Ensure host writes are visible before the GPU copy. } diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index ab58560170..b3ebf7b3f6 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -5,6 +5,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <array> +#include <chrono> +#include <cstdlib> #include <memory> #include <numeric> #include <optional> @@ -14,9 +16,11 @@ #include "common/assert.h" #include "common/common_types.h" +#include "common/logging.h" #include "common/div_ceil.h" #include "common/vector_math.h" #include "video_core/host_shaders/astc_decoder_comp_spv.h" +#include "video_core/host_shaders/bc_encoder_comp_spv.h" #include "video_core/host_shaders/queries_prefix_scan_sum_comp_spv.h" #include "video_core/host_shaders/queries_prefix_scan_sum_nosubgroups_comp_spv.h" #include "video_core/host_shaders/resolve_conditional_render_comp_spv.h" @@ -31,6 +35,7 @@ #include "video_core/renderer_vulkan/vk_update_descriptor.h" #include "video_core/texture_cache/accelerated_swizzle.h" #include "video_core/texture_cache/types.h" +#include "video_core/texture_cache/util.h" #include "video_core/textures/decoders.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -183,6 +188,52 @@ struct AstcPushConstants { u32 x_shift; u32 block_height; u32 block_height_mask; + u32 native_tiling; +}; + +constexpr std::array<VkDescriptorSetLayoutBinding, 2> BC_ENCODER_DESCRIPTOR_SET_BINDINGS{{ + { + .binding = 0, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, + { + .binding = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, +}}; + +constexpr std::array<VkDescriptorUpdateTemplateEntry, 2> BC_ENCODER_DESCRIPTOR_UPDATE_TEMPLATE{{ + { + .dstBinding = 0, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, + .offset = 0, + .stride = sizeof(DescriptorUpdateEntry), + }, + { + .dstBinding = 1, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + }, +}}; + +struct BcEncoderPushConstants { + u32 blocks_x; + u32 blocks_y; + u32 width; + u32 height; + u32 bc3; + u32 out_offset; // in 8-byte units }; struct QueriesPrefixScanPushConstants { @@ -521,16 +572,31 @@ ASTCDecoderPass::~ASTCDecoderPass() = default; void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, std::span<const VideoCommon::SwizzleParameters> swizzles) { + const Target target{ + .image = image.Handle(), + .aspect_mask = image.AspectMask(), + .initialized = image.ExchangeInitialization(), + .level_view = [&image](s32 host_level) { return image.StorageImageView(host_level); }, + .wait = true, + }; + AssembleInto(image.info, image.host_level_offset, image.guest_size_bytes, target, map, + swizzles); +} + +void ASTCDecoderPass::AssembleInto(const VideoCommon::ImageInfo& info, u32 skip, + size_t guest_size_bytes, const Target& target, + const StagingBufferRef& map, + std::span<const VideoCommon::SwizzleParameters> swizzles) { using namespace VideoCommon::Accelerated; const std::array<u32, 2> block_dims{ - VideoCore::Surface::DefaultBlockWidth(image.info.format), - VideoCore::Surface::DefaultBlockHeight(image.info.format), + VideoCore::Surface::DefaultBlockWidth(info.format), + VideoCore::Surface::DefaultBlockHeight(info.format), }; scheduler.RequestOutsideRenderPassOperationContext(); const VkPipeline vk_pipeline = *pipeline; - const VkImageAspectFlags aspect_mask = image.AspectMask(); - const VkImage vk_image = image.Handle(); - const bool is_initialized = image.ExchangeInitialization(); + const VkImageAspectFlags aspect_mask = target.aspect_mask; + const VkImage vk_image = target.image; + const bool is_initialized = target.initialized; scheduler.Record([vk_pipeline, vk_image, aspect_mask, is_initialized](vk::CommandBuffer cmdbuf) { const VkImageMemoryBarrier image_barrier{ @@ -557,20 +623,30 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, image_barrier); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline); }); + // With trimmed host levels the input holds each layer's mip chain from host_level_offset on + // (see TextureCache::UploadImageContents), so its layer stride is that tail's size. + const u32 input_layer_stride = + skip != 0 ? info.layer_stride - + static_cast<u32>(VideoCommon::FullUploadSwizzles(info)[skip].buffer_offset) + : info.layer_stride; + const size_t input_size = skip != 0 ? size_t{input_layer_stride} * info.resources.layers + : guest_size_bytes; for (const VideoCommon::SwizzleParameters& swizzle : swizzles) { const size_t input_offset = swizzle.buffer_offset + map.offset; const u32 num_dispatches_x = Common::DivCeil(swizzle.num_tiles.width, 8U); const u32 num_dispatches_y = Common::DivCeil(swizzle.num_tiles.height, 8U); - const u32 num_dispatches_z = image.info.resources.layers; + const u32 num_dispatches_z = info.resources.layers; compute_pass_descriptor_queue.Acquire(scheduler, 2); compute_pass_descriptor_queue.AddBuffer(map.buffer, input_offset, - image.guest_size_bytes - swizzle.buffer_offset); - compute_pass_descriptor_queue.AddImage(image.StorageImageView(swizzle.level)); + input_size - swizzle.buffer_offset); + compute_pass_descriptor_queue.AddImage( + target.level_view(swizzle.level - static_cast<s32>(skip))); const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; // To unswizzle the ASTC data - const auto params = MakeBlockLinearSwizzle2DParams(swizzle, image.info); + auto params = MakeBlockLinearSwizzle2DParams(swizzle, info); + params.layer_stride = input_layer_stride; ASSERT(params.origin == (std::array<u32, 3>{0, 0, 0})); ASSERT(params.destination == (std::array<s32, 3>{0, 0, 0})); ASSERT(params.bytes_per_block_log2 == 4); @@ -583,6 +659,7 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, .x_shift = params.x_shift, .block_height = params.block_height, .block_height_mask = params.block_height_mask, + .native_tiling = params.native_tiling, }; const VkDescriptorSet set = descriptor_allocator.Commit(); device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); @@ -613,7 +690,74 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier); }); - scheduler.Finish(); + // Each decode ends with the GPU thread waiting for the host GPU to go idle. Measured here + // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=1 submits without waiting instead. + static const bool flush_only = [] { + const char* value = std::getenv("EDEN_NX2_ASTC_FLUSH"); + return value != nullptr && value[0] == '1'; + }(); + if (!target.wait) { + return; + } + const auto start = std::chrono::steady_clock::now(); + if (flush_only) { + scheduler.Flush(); + } else { + scheduler.Finish(); + } + const auto end = std::chrono::steady_clock::now(); + static u32 decodes{}; + static std::chrono::nanoseconds waited{}; + static auto window_start = end; + ++decodes; + waited += end - start; + if (end - window_start >= std::chrono::seconds{5}) { + LOG_INFO(Render_Vulkan, "ASTC GPU decodes: {} in {} ms, {} ms waiting ({})", decodes, + std::chrono::duration_cast<std::chrono::milliseconds>(end - window_start).count(), + std::chrono::duration_cast<std::chrono::milliseconds>(waited).count(), + flush_only ? "flush" : "finish"); + decodes = 0; + waited = {}; + window_start = end; + } +} + +BcEncoderPass::BcEncoderPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, BC_ENCODER_DESCRIPTOR_SET_BINDINGS, + BC_ENCODER_DESCRIPTOR_UPDATE_TEMPLATE, ASTC_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE<sizeof(BcEncoderPushConstants)>, BC_ENCODER_COMP_SPV), + scheduler{scheduler_}, compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +BcEncoderPass::~BcEncoderPass() = default; + +void BcEncoderPass::Encode(VkImageView source, u32 width, u32 height, u32 layers, bool bc3, + VkBuffer buffer, VkDeviceSize buffer_offset, VkDeviceSize buffer_size, + VkDeviceSize out_offset) { + scheduler.RequestOutsideRenderPassOperationContext(); + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddImage(source); + compute_pass_descriptor_queue.AddBuffer(buffer, buffer_offset, buffer_size); + const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; + const BcEncoderPushConstants uniforms{ + .blocks_x = Common::DivCeil(width, 4U), + .blocks_y = Common::DivCeil(height, 4U), + .width = width, + .height = height, + .bc3 = bc3 ? 1U : 0U, + .out_offset = static_cast<u32>(out_offset / 8), + }; + const VkPipeline vk_pipeline = *pipeline; + scheduler.Record([this, vk_pipeline, uniforms, layers, descriptor_data](vk::CommandBuffer cmdbuf) { + const VkDescriptorSet set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, uniforms); + cmdbuf.Dispatch(Common::DivCeil(uniforms.blocks_x, 8U), Common::DivCeil(uniforms.blocks_y, 8U), + layers); + }); } constexpr u32 BL3D_BINDING_INPUT_BUFFER = 0; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h index 3d30aa6bdc..5daf4eb3de 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.h +++ b/src/video_core/renderer_vulkan/vk_compute_pass.h @@ -6,6 +6,7 @@ #pragma once +#include <functional> #include <optional> #include <span> #include <utility> @@ -130,6 +131,21 @@ public: void Assemble(Image& image, const StagingBufferRef& map, std::span<const VideoCommon::SwizzleParameters> swizzles); + /// Where a decode writes: an RGBA8 image with a storage view per (host) level. + struct Target { + VkImage image; + VkImageAspectFlags aspect_mask; + bool initialized; + std::function<VkImageView(s32 host_level)> level_view; + bool wait; ///< Wait for the GPU afterwards (the decoded image itself is sampled). + }; + + /// Decodes `info`'s ASTC data in `map` into `target`. `skip` leading levels are absent on the + /// host (TrimmedLevels) and `guest_size_bytes` is the full guest size. + void AssembleInto(const VideoCommon::ImageInfo& info, u32 skip, size_t guest_size_bytes, + const Target& target, const StagingBufferRef& map, + std::span<const VideoCommon::SwizzleParameters> swizzles); + private: Scheduler& scheduler; StagingBufferPool& staging_buffer_pool; @@ -137,6 +153,24 @@ private: MemoryAllocator& memory_allocator; }; +/// Compresses an RGBA8 image level to BC1 or BC3 blocks in a buffer (bc_encoder.comp). +class BcEncoderPass final : public ComputePass { +public: + explicit BcEncoderPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BcEncoderPass(); + + /// Encodes every layer of one level (`source` is its RGBA8 storage view) into `buffer`, + /// `out_offset` bytes past `buffer_offset`; blocks are row-major, layer after layer. + void Encode(VkImageView source, u32 width, u32 height, u32 layers, bool bc3, VkBuffer buffer, + VkDeviceSize buffer_offset, VkDeviceSize buffer_size, VkDeviceSize out_offset); + +private: + Scheduler& scheduler; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + class BlockLinearUnswizzle3DPass final : public ComputePass { public: explicit BlockLinearUnswizzle3DPass(const Device& device_, Scheduler& scheduler_, diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 8b5b0bc9c5..f737ad9f70 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -29,6 +29,7 @@ #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/gpu_logging/gpu_logging.h" #include "common/settings.h" +#include "video_core/nx2_profile.h" #if defined(_MSC_VER) && defined(NDEBUG) #define LAMBDA_FORCEINLINE [[msvc::forceinline]] @@ -355,7 +356,11 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { views.reserve(num_image_elements); samplers.reserve(num_textures); - texture_cache.SynchronizeDescriptors(false); + namespace Profile = VideoCommon::Nx2Profile; + { + Profile::Scope scope{Profile::Zone::SyncDescriptors}; + texture_cache.SynchronizeDescriptors(false); + } buffer_cache.SetUniformBuffersState(enabled_uniform_buffer_masks, &uniform_buffer_sizes); @@ -432,24 +437,31 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { return true; }}; - if constexpr (Spec::enabled_stages[0]) { - config_stage(0); - } - if constexpr (Spec::enabled_stages[1]) { - config_stage(1); - } - if constexpr (Spec::enabled_stages[2]) { - config_stage(2); - } - if constexpr (Spec::enabled_stages[3]) { - config_stage(3); - } - if constexpr (Spec::enabled_stages[4]) { - config_stage(4); + { + Profile::Scope scope{Profile::Zone::ReadHandles}; + if constexpr (Spec::enabled_stages[0]) { + config_stage(0); + } + if constexpr (Spec::enabled_stages[1]) { + config_stage(1); + } + if constexpr (Spec::enabled_stages[2]) { + config_stage(2); + } + if constexpr (Spec::enabled_stages[3]) { + config_stage(3); + } + if constexpr (Spec::enabled_stages[4]) { + config_stage(4); + } } ASSERT(views.size() == num_image_elements); ASSERT(samplers.size() == num_textures); - texture_cache.FillImageViews(std::span(views.data(), views.size()), false, Spec::has_images); + { + Profile::Scope scope{Profile::Zone::FillImageViews}; + texture_cache.FillImageViews(std::span(views.data(), views.size()), false, + Spec::has_images); + } VideoCommon::ImageViewInOut* texture_buffer_it{views.data()}; const auto bind_stage_info{[&](size_t stage) LAMBDA_FORCEINLINE { @@ -493,44 +505,62 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { texture_buffer_it += Shader::NumDescriptors(info.image_descriptors); } }}; - if constexpr (Spec::enabled_stages[0]) { - bind_stage_info(0); - } - if constexpr (Spec::enabled_stages[1]) { - bind_stage_info(1); - } - if constexpr (Spec::enabled_stages[2]) { - bind_stage_info(2); - } - if constexpr (Spec::enabled_stages[3]) { - bind_stage_info(3); - } - if constexpr (Spec::enabled_stages[4]) { - bind_stage_info(4); + { + Profile::Scope scope{Profile::Zone::TextureBuffers}; + if constexpr (Spec::enabled_stages[0]) { + bind_stage_info(0); + } + if constexpr (Spec::enabled_stages[1]) { + bind_stage_info(1); + } + if constexpr (Spec::enabled_stages[2]) { + bind_stage_info(2); + } + if constexpr (Spec::enabled_stages[3]) { + bind_stage_info(3); + } + if constexpr (Spec::enabled_stages[4]) { + bind_stage_info(4); + } } if (regs.transform_feedback_enabled != 0) { scheduler.RequestOutsideRenderPassOperationContext(); } - buffer_cache.UpdateGraphicsBuffers(is_indexed); - buffer_cache.BindHostGeometryBuffers(is_indexed); + { + Profile::Scope scope{Profile::Zone::UpdateBuffers}; + buffer_cache.UpdateGraphicsBuffers(is_indexed); + buffer_cache.BindHostGeometryBuffers(is_indexed); + } - guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer); + { + Profile::Scope scope{Profile::Zone::DescriptorAcquire}; + guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer); + } RescalingPushConstant rescaling; RenderAreaPushConstant render_area; const VideoCommon::SamplerId* samplers_it{samplers.data()}; const VideoCommon::ImageViewInOut* views_it{views.data()}; const auto prepare_stage{[&](size_t stage) LAMBDA_FORCEINLINE { - buffer_cache.BindHostStageBuffers(stage); - PushImageDescriptors(texture_cache, guest_descriptor_queue, stage_infos[stage], rescaling, - samplers_it, views_it); + { + Profile::Scope scope{Profile::Zone::BindStageBuffers}; + buffer_cache.BindHostStageBuffers(stage); + } + { + Profile::Scope scope{Profile::Zone::PushDescriptors}; + PushImageDescriptors(texture_cache, guest_descriptor_queue, stage_infos[stage], + rescaling, samplers_it, views_it); + } const auto& info{stage_infos[stage]}; if (info.uses_render_area) { render_area.uses_render_area = true; - render_area.words = {static_cast<float>(regs.surface_clip.width), - static_cast<float>(regs.surface_clip.height)}; + // Switch 2 titles may never program the surface clip; use colour target 0 then. + const u32 clip_width = regs.surface_clip.width, clip_height = regs.surface_clip.height; + const bool clip_set = clip_width != 0 && clip_height != 0; + render_area.words = {static_cast<float>(clip_set ? clip_width : u32{regs.rt[0].width}), + static_cast<float>(clip_set ? clip_height : u32{regs.rt[0].height})}; } }}; if constexpr (Spec::enabled_stages[0]) { @@ -552,12 +582,16 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { buffer_cache.runtime.PostCopyBarrier(); buffer_cache.any_buffer_uploaded = false; } - texture_cache.UpdateRenderTargets(false); - texture_cache.CheckFeedbackLoop(std::span<const VideoCommon::ImageViewInOut>{views.data(), - views.size()}); + { + Profile::Scope scope{Profile::Zone::RenderTargets}; + texture_cache.UpdateRenderTargets(false); + texture_cache.CheckFeedbackLoop( + std::span<const VideoCommon::ImageViewInOut>{views.data(), views.size()}); + } if (IsBuilt() && !pipeline) { return false; } + Profile::Scope scope{Profile::Zone::ConfigureDraw}; return ConfigureDraw(rescaling, render_area); } diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index d4ffcd8317..60bb040984 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <chrono> #include <cstddef> #include <fstream> #include <iostream> @@ -15,6 +16,7 @@ #include <bit> #include <numeric> #include "common/cityhash.h" +#include "common/scope_exit.h" #include "common/fs/fs.h" #include "common/fs/path_util.h" #include "common/thread_worker.h" @@ -930,7 +932,38 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline( return nullptr; } +namespace { +// Pipelines built on the GPU thread at run time (shader translation; the driver compile is on +// workers): every one over 2 ms is logged, and the total every 10 s, to tell how much of the +// stutter on new content is translation. +void ReportRuntimePipeline(const char* kind, std::chrono::steady_clock::time_point begin) { + static std::chrono::steady_clock::duration total{}; + static u32 count{}, logged{}; + static auto window = std::chrono::steady_clock::now(); + const auto now = std::chrono::steady_clock::now(); + const auto elapsed = now - begin; + total += elapsed; + ++count; + const auto ms = [](auto d) { return std::chrono::duration<double, std::milli>(d).count(); }; + if (elapsed > std::chrono::milliseconds{2} && logged < 400) { + ++logged; + LOG_INFO(Render_Vulkan, "{} pipeline built on the GPU thread in {:.1f} ms", kind, ms(elapsed)); + } + if (now - window > std::chrono::seconds{10}) { + LOG_INFO(Render_Vulkan, "Runtime pipelines in the last 10 s: {} built, {:.1f} ms on the GPU thread", + count, ms(total)); + total = {}; + count = 0; + window = now; + } +} +} // Anonymous namespace + std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() { + const auto begin = std::chrono::steady_clock::now(); + SCOPE_EXIT { + ReportRuntimePipeline("Graphics", begin); + }; GraphicsEnvironments environments; GetGraphicsEnvironments(environments, graphics_key.unique_hashes); @@ -956,6 +989,10 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() { std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline( const ComputePipelineCacheKey& key, const ShaderInfo* shader) { + const auto begin = std::chrono::steady_clock::now(); + SCOPE_EXIT { + ReportRuntimePipeline("Compute", begin); + }; const GPUVAddr program_base{kepler_compute->regs.code_loc.Address()}; const auto& qmd{kepler_compute->launch_description}; ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start}; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 264ffb28b5..429b2d63ef 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -7,6 +7,11 @@ #include <algorithm> #include <array> #include <memory> +#include <cstdio> +#include <cstdlib> +#include <tuple> +#include <set> +#include <fstream> #include <mutex> #include <fmt/format.h> @@ -14,12 +19,14 @@ #include "video_core/renderer_vulkan/renderer_vulkan.h" #include "common/assert.h" +#include "shader_recompiler/exception.h" #include "common/logging.h" #include "common/scope_exit.h" #include "common/settings.h" #include "video_core/buffer_cache/buffer_cache.h" #include "video_core/gpu_logging/gpu_logging.h" #include "video_core/control/channel_state.h" +#include "video_core/native_query_trace.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/maxwell_3d.h" @@ -42,6 +49,7 @@ #include "video_core/texture_cache/texture_cache_base.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" +#include "video_core/nx2_profile.h" namespace Vulkan { @@ -238,28 +246,296 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { SCOPE_EXIT { gpu.TickWork(); }; - FlushWork(); - gpu_memory->FlushCaching(); - - GraphicsPipeline* const pipeline{pipeline_cache.CurrentGraphicsPipeline()}; + bool submitted = false; + SCOPE_EXIT { + auto& native = maxwell3d->GetNativeMacroEngine(); + if (native.native_turing && !submitted) { + native.native_execution_failed = true; + LOG_DEBUG(HW_GPU, "Native graphics draw failed; refusing submission completion"); + } + }; + namespace Profile = VideoCommon::Nx2Profile; + { + Profile::Scope scope{Profile::Zone::FlushWork}; + FlushWork(); + gpu_memory->FlushCaching(); + } + + const auto report_failure = [&](std::string_view reason) { + static std::set<std::tuple<u64, u64, std::string_view>> seen; + const auto& regs = maxwell3d->regs; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + const u64 ps = (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1]; + if (seen.size() < 64 && seen.emplace(vs, ps, reason).second) { + LOG_WARNING(HW_GPU, "Draw skipped ({}): vs={:#x} ps={:#x} rt0={:#x} {}x{}", reason, vs, ps, + regs.rt[0].Address(), regs.rt[0].width, regs.rt[0].height); + } + }; + GraphicsPipeline* const pipeline{[&] { + Profile::Scope scope{Profile::Zone::PipelineLookup}; + return pipeline_cache.CurrentGraphicsPipeline(); + }()}; if (!pipeline) { + report_failure("no pipeline"); return; } std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; // update engine as channel may be different. pipeline->SetEngine(maxwell3d, gpu_memory); - if (!pipeline->Configure(is_indexed)) + if (Profile::Scope scope{Profile::Zone::Configure}; !pipeline->Configure(is_indexed)) { + report_failure("configure failed"); return; + } - UpdateDynamicStates(); + { + Profile::Scope scope{Profile::Zone::DynamicState}; + UpdateDynamicStates(); + } query_cache.NotifySegment(true); HandleTransformFeedback(); query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable); - draw_func(); + { + Profile::Scope scope{Profile::Zone::DrawRecord}; + draw_func(); + } + submitted = true; + static u32 native_capture_count{}; + if (maxwell3d->GetNativeMacroEngine().native_turing && + native_capture_count < 16 && [] { + static const bool capture = std::getenv("EDEN_NATIVE_DRAW_CAPTURE") != nullptr; + return capture; + }()) { + const u32 capture = native_capture_count++; + const auto prefix = fmt::format("native-draw-{}", capture); + const auto dump = [&](const std::string& name, GPUVAddr address, size_t size) { + if (!size || !gpu_memory->IsFullyMappedRange(address, size)) return; + std::vector<u8> bytes(size); + gpu_memory->ReadBlockUnsafe(address, bytes.data(), bytes.size()); + std::ofstream file(prefix + name + ".bin", std::ios::binary); + file.write(reinterpret_cast<const char*>(bytes.data()), bytes.size()); + LOG_INFO(HW_GPU, "Native draw capture {}{}: address={:#x}, bytes={}, nonzero={}", + capture, name, address, size, + std::count_if(bytes.begin(), bytes.end(), [](u8 byte) { return byte != 0; })); + }; + const auto& regs = maxwell3d->regs; + const auto& rt = regs.rt[0]; + const size_t size = static_cast<size_t>(rt.array_pitch) * 4; + if (size && size <= 16 * 1024 * 1024) { + if (const auto address = gpu_memory->GpuToCpuAddress(rt.Address(), size)) { + LOG_INFO(HW_GPU, "Native draw {} render target physical address={:#x}", capture, *address); + texture_cache.DownloadMemory(*address, size); + dump("-rt0", rt.Address(), size); + } + } + std::ofstream registers(prefix + "-registers.bin", std::ios::binary); + registers.write(reinterpret_cast<const char*>(®s), sizeof(regs)); + for (size_t stream = 0; stream < 4; ++stream) { + if (regs.vertex_streams[stream].IsEnabled()) + dump(fmt::format("-vertex-{}", stream), regs.vertex_streams[stream].Address(), 128); + } + for (size_t stage : {0U, 4U}) { + for (size_t bank : {2U, 3U}) { + const auto& cb = maxwell3d->state.shader_stages[stage].const_buffers[bank]; + if (cb.enabled) dump(fmt::format("-cbuf-{}-{}", stage, bank), cb.address, std::min<size_t>(cb.size, 128)); + } + } + } } + +namespace { +// EDEN_NX2_WINDOW_TRACE: describe clears and draws into screen-sized colour targets (debug aid). +void TraceWindowTarget(const Tegra::Engines::Maxwell3D& maxwell3d, Tegra::MemoryManager& memory, + const char* kind, u32 count) { + static const bool trace = std::getenv("EDEN_NX2_WINDOW_TRACE") != nullptr; + static u32 traced{}; + const auto& regs = maxwell3d.regs; + static const u64 skip = [] { + const char* value = std::getenv("EDEN_NX2_WINDOW_TRACE_SKIP"); + return value != nullptr ? std::strtoull(value, nullptr, 10) : 0ULL; + }(); + static u64 seen_events{}; + static const u64 only_ps = [] { + const char* value = std::getenv("EDEN_NX2_TRACE_PS"); + return value != nullptr ? std::strtoull(value, nullptr, 16) : 0ULL; + }(); + const u64 pixel_shader = + (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1]; + if (!trace || regs.rt[0].width != 3840 || traced >= 400 || seen_events++ < skip || + (only_ps != 0 && pixel_shader != only_ps)) { + return; + } + ++traced; + const auto& vp = regs.viewport_transform[0]; + const auto& sc = regs.scissor_test[0]; + LOG_INFO(HW_GPU, + "nx2 window {} rt0={:#x} count={} clear={:#x} color=({},{},{},{}) mask_common={} " + "mask0={:#x} rt_count={} vp scale=({},{}) translate=({},{}) scissor en={} x={}..{} " + "y={}..{} blend0={} vs={:#x} ps={:#x} rasterize={}", + kind, regs.rt[0].Address(), count, regs.clear_surface.raw, regs.clear_color[0], + regs.clear_color[1], regs.clear_color[2], regs.clear_color[3], regs.color_mask_common, + regs.color_mask[0].raw, regs.rt_control.count.Value(), vp.scale_x, vp.scale_y, + vp.translate_x, vp.translate_y, sc.enable, sc.min_x.Value(), sc.max_x.Value(), + sc.min_y.Value(), sc.max_y.Value(), regs.blend.enable[0], + (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1], + (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1], + regs.rasterize_enable); + if (kind[0] != 'd') { + return; + } + // Pixel-stage constant buffer 3: colour constants of the loading-screen passes. + if (const auto& cb3 = maxwell3d.state.shader_stages[4].const_buffers[3]; cb3.enabled) { + std::array<f32, 8> values{}; + memory.ReadBlockUnsafe(cb3.address, values.data(), sizeof(values)); + LOG_INFO(HW_GPU, "nx2 window ps c[3]@{:#x} size={}: {}", cb3.address, cb3.size, + fmt::join(values, " ")); + } + static u32 texture_traces{}; + if (texture_traces++ > 200) { + return; + } + // Texture used by the pixel shader's first bound TEX (handle in c[2][0x20]) and its TIC. + const auto& cb = maxwell3d.state.shader_stages[4].const_buffers[2]; + const u32 handle = cb.enabled ? memory.Read<u32>(cb.address + 0x20) : 0; + const u32 tic_index = handle & 0xfffff; + const GPUVAddr tic_address = regs.tex_header.Address() + u64{tic_index} * 32; + std::array<u32, 8> tic{}; + memory.ReadBlockUnsafe(tic_address, tic.data(), sizeof(tic)); + LOG_INFO(HW_GPU, + "nx2 window texture: cb2={:#x} en={} handle={:#x} tic pool={:#x} limit={} " + "sampler pool={:#x} tic[{}]={:08x}", + cb.address, cb.enabled, handle, regs.tex_header.Address(), regs.tex_header.limit, + regs.tex_sampler.Address(), tic_index, fmt::join(tic, " ")); + // First bytes of the texture's memory (block-linear, so just a sample). + const GPUVAddr texture = (GPUVAddr{tic[2] & 0xffff} << 32) | tic[1]; + std::array<u32, 16> texels{}; + memory.ReadBlockUnsafe(texture, texels.data(), sizeof(texels)); + LOG_INFO(HW_GPU, "nx2 window texture {:#x} {}x{}: {:08x}", texture, (tic[4] & 0xffff) + 1, + (tic[5] & 0xffff) + 1, fmt::join(texels, " ")); + // EDEN_NX2_TEXTURE_DUMP=<file>: save the first 4 MiB of the last traced texture's memory. + if (const char* path = std::getenv("EDEN_NX2_TEXTURE_DUMP"); path != nullptr) { + std::vector<u8> bytes(2048u * 10240u); + memory.ReadBlockUnsafe(texture, bytes.data(), bytes.size()); + if (std::FILE* file = std::fopen(path, "wb")) { + std::fwrite(bytes.data(), 1, bytes.size(), file); + std::fclose(file); + } + } +} +} // Anonymous namespace + void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Draw}; + TraceWindowTarget(*maxwell3d, *gpu_memory, is_indexed ? "draw-indexed" : "draw", + is_indexed ? maxwell3d->draw_manager.draw_state.index_buffer.count + : maxwell3d->draw_manager.draw_state.vertex_buffer.count); + // EDEN_NX2_CB24_TRACE: for draws whose vertex shader reads bank 24, show the inline + // constant buffer upload target (CB_ADDRESS) and its first words. + if (static const bool trace = std::getenv("EDEN_NX2_CB24_TRACE") != nullptr; trace) { + const auto& regs = maxwell3d->regs; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + static u32 logged{}; + if ((vs == 0x11471c9a00 || vs == 0x11471ca600 || vs == 0x11471cd300) && logged++ < 3) { + std::array<u32, 24> words{}; + gpu_memory->ReadBlockUnsafe(regs.const_buffer.Address(), words.data(), sizeof(words)); + LOG_INFO(HW_GPU, "bank24 draw vs={:#x}: CB_ADDRESS {:#x} size {:#x} offset {:#x}: {:08x}", vs, + regs.const_buffer.Address(), regs.const_buffer.size, regs.const_buffer.offset, + fmt::join(words, " ")); + // Search every bound vertex-stage buffer for render-target-size values. + const auto& stage = maxwell3d->state.shader_stages[0]; + for (u32 slot = 0; slot < Tegra::Engines::Maxwell3D::Regs::MaxConstBuffers; ++slot) { + const auto& buffer = stage.const_buffers[slot]; + if (!buffer.enabled || buffer.size == 0) continue; + std::vector<u32> data(std::min<u32>(buffer.size, 0x10000) / 4); + gpu_memory->ReadBlockUnsafe(buffer.address, data.data(), data.size() * 4); + for (size_t i = 0; i < data.size(); ++i) { + const u32 w = data[i]; + if (w == 0x45200000 || w == 0x44b40000 || w == 0x39cccccd || w == 0x3a360b61 || + w == 0x44a00000 || w == 0x44340000 || w == 2559 || w == 1439) { + LOG_INFO(HW_GPU, " slot {} +{:#x} = {:#x}", slot, i * 4, w); + } + } + } + } + } + // EDEN_NX2_TRACE_VS=<address>: describe the next few draws using that vertex shader like the + // after-upscale trace does (render target, blend, fragment textures). + static const u64 trace_vs = [] { + const char* value = std::getenv("EDEN_NX2_TRACE_VS"); + return value ? std::strtoull(value, nullptr, 16) : 0ULL; + }(); + if (trace_vs != 0) { + const auto& regs = maxwell3d->regs; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + static u32 traced{}; + if (vs == trace_vs && traced++ % 200 == 0 && traced < 1000) { + Tegra::Engines::g_nx2_trace_draws_after_upscale = 1; + } + } + if (Tegra::Engines::g_nx2_trace_draws_after_upscale > 0) { + --Tegra::Engines::g_nx2_trace_draws_after_upscale; + const auto& regs = maxwell3d->regs; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + const u64 ps = (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1]; + LOG_INFO(HW_GPU, "after upscale: draw rt0 {:#x} {}x{} fmt {:#x} block {}/{}/{} pitch_linear {} depth {} array_pitch {:#x} ctrl {:#x} vs {:#x} ps {:#x}", + regs.rt[0].Address(), regs.rt[0].width, regs.rt[0].height, + static_cast<u32>(regs.rt[0].format), regs.rt[0].tile_mode.block_width.Value(), + regs.rt[0].tile_mode.block_height.Value(), regs.rt[0].tile_mode.block_depth.Value(), + regs.rt[0].tile_mode.is_pitch_linear.Value(), regs.rt[0].depth.Value(), + regs.rt[0].array_pitch, regs.rt_control.count.Value(), vs, ps); + // Fragment-stage texture handles (bound buffer 2) and the images they name. + const auto& handles = maxwell3d->state.shader_stages[4].const_buffers[2]; + if (handles.enabled) { + for (u32 i = 0; i < 24; ++i) { + const u32 handle = gpu_memory->Read<u32>(handles.address + i * 4); + if (handle == 0 || handle == 0xffffffff) continue; + Tegra::Texture::TICEntry tic{}; + gpu_memory->ReadBlockUnsafe(regs.tex_header.Address() + u64{handle & 0xfffff} * sizeof(tic), + &tic, sizeof(tic)); + LOG_INFO(HW_GPU, " fs handle[{}] = {:#x}: {:#x} {}x{} format {:#x}", i, handle, + tic.Address(), tic.Width(), tic.Height(), static_cast<u32>(tic.format.Value())); + } + } + const auto& bt = regs.blend_per_target[0]; + LOG_INFO(HW_GPU, " blend enable {} per-target {} global op {} src {:#x} dst {:#x} | rt0 op {} src {:#x} dst {:#x} alpha {:#x}/{:#x} mask {:#x}", + regs.blend.enable[0], regs.blend_per_target_enabled, + static_cast<u32>(regs.blend.color_op), static_cast<u32>(regs.blend.color_source), + static_cast<u32>(regs.blend.color_dest), static_cast<u32>(bt.color_op), + static_cast<u32>(bt.color_source), static_cast<u32>(bt.color_dest), + static_cast<u32>(bt.alpha_source), static_cast<u32>(bt.alpha_dest), + regs.color_mask[0].raw); + } + // EDEN_NX2_SKIP_VS=<address>[,<address>...]: drop draws using those vertex shaders (debug aid). + static const std::vector<u64> skip_list = [] { + std::vector<u64> list; + if (const char* value = std::getenv("EDEN_NX2_SKIP_VS")) { + for (char* end = nullptr; *value; value = *end ? end + 1 : end) { + list.push_back(std::strtoull(value, &end, 16)); + if (end == value) break; + } + } + return list; + }(); + if (!skip_list.empty()) { + const auto& regs = maxwell3d->regs; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + if (std::find(skip_list.begin(), skip_list.end(), vs) != skip_list.end()) return; + } + // EDEN_NX2_RT_TRACE: log the first draw to every distinct colour target (debug aid). + if (static const bool trace = std::getenv("EDEN_NX2_RT_TRACE") != nullptr; trace) { + static std::set<std::tuple<GPUVAddr, u32, u32, u64, u64>> seen; + const auto& regs = maxwell3d->regs; + const auto& rt = regs.rt[0]; + const u64 vs = (u64{regs.pipelines[1].reserved[0] & 0xff} << 32) | regs.pipelines[1].reserved[1]; + const u64 ps = (u64{regs.pipelines[5].reserved[0] & 0xff} << 32) | regs.pipelines[5].reserved[1]; + const auto key = std::make_tuple(rt.Address(), rt.width, static_cast<u32>(rt.format), vs, ps); + if (rt.width >= 640 && seen.size() < 600 && seen.insert(key).second) { + LOG_INFO(HW_GPU, "nx2 draw target {:#x} {}x{} format={:#x} vs={:#x} ps={:#x} blend={} (pairs so far {})", + rt.Address(), rt.width, rt.height, static_cast<u32>(rt.format), vs, ps, + regs.blend.enable[0], seen.size()); + } + } PrepareDraw(is_indexed, [this, is_indexed, instance_count] { const auto& draw_state = maxwell3d->draw_manager.draw_state; const u32 num_instances{instance_count}; @@ -293,6 +569,7 @@ void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) { } void RasterizerVulkan::DrawIndirect() { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Draw}; const auto& params = maxwell3d->draw_manager.indirect_state; buffer_cache.SetDrawIndirect(¶ms); PrepareDraw(params.is_indexed, [this, ¶ms] { @@ -398,6 +675,15 @@ void RasterizerVulkan::DrawTexture() { } void RasterizerVulkan::Clear(u32 layer_count) { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Clear}; + if (Tegra::Engines::g_nx2_trace_draws_after_upscale > 0) { + const auto& regs = maxwell3d->regs; + LOG_INFO(HW_GPU, "after upscale: clear rt0 {:#x} {}x{} fmt {:#x} surface {:#x} colour {} {} {} {}", + regs.rt[0].Address(), regs.rt[0].width, regs.rt[0].height, + static_cast<u32>(regs.rt[0].format), regs.clear_surface.raw, regs.clear_color[0], + regs.clear_color[1], regs.clear_color[2], regs.clear_color[3]); + } + TraceWindowTarget(*maxwell3d, *gpu_memory, "clear", layer_count); FlushWork(); gpu_memory->FlushCaching(); @@ -583,16 +869,19 @@ void RasterizerVulkan::Clear(u32 layer_count) { } void RasterizerVulkan::DispatchCompute() { + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::Compute}; FlushWork(); gpu_memory->FlushCaching(); ComputePipeline* const pipeline{pipeline_cache.CurrentComputePipeline()}; if (!pipeline) { + if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute pipeline unavailable"); return; } std::scoped_lock lock{texture_cache.mutex, buffer_cache.mutex}; if (!pipeline->Configure(*kepler_compute, *gpu_memory, scheduler, buffer_cache, texture_cache)) { + if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute resources unavailable"); return; } @@ -942,19 +1231,35 @@ Tegra::Engines::AccelerateDMAInterface& RasterizerVulkan::AccessAccelerateDMA() void RasterizerVulkan::AccelerateInlineToMemory(GPUVAddr address, size_t copy_size, std::span<const u8> memory) { auto cpu_addr = gpu_memory->GpuToCpuAddress(address); + if (gpu_memory->UsesNativeTiling()) { + u32 first_word{}; + if (memory.size() >= sizeof(first_word)) std::memcpy(&first_word, memory.data(), sizeof(first_word)); + VideoCommon::TraceNativeQuery(11, gpu_memory->GetID(), address, cpu_addr.value_or(0), + 0, first_word, static_cast<u32>(copy_size)); + static const GPUVAddr watch = [] { + const char* value = std::getenv("EDEN_NX2_SEM_WATCH"); + return value ? GPUVAddr{std::strtoull(value, nullptr, 16)} : GPUVAddr{}; + }(); + if (watch != 0 && address < watch + 16 && address + copy_size > watch) { + LOG_INFO(Render_Vulkan, "Semaphore watch: inline write {:#x} size {} first word {:#x}", + address, copy_size, first_word); + } + } if (!cpu_addr) [[unlikely]] { gpu_memory->WriteBlock(address, memory.data(), copy_size); return; } gpu_memory->WriteBlockUnsafe(address, memory.data(), copy_size); + // One cache at a time: holding both while waiting for either (std::lock) waited longer, the + // fencing thread and CPU write traps holding one of them most of the time. { - std::unique_lock<std::recursive_mutex> lock{buffer_cache.mutex}; + std::scoped_lock lock{buffer_cache.mutex}; if (!buffer_cache.InlineMemory(*cpu_addr, copy_size, memory)) { buffer_cache.WriteMemory(*cpu_addr, copy_size); } } { - std::scoped_lock lock_texture{texture_cache.mutex}; + std::scoped_lock lock{texture_cache.mutex}; texture_cache.WriteMemory(*cpu_addr, copy_size); } pipeline_cache.InvalidateRegion(*cpu_addr, copy_size); @@ -969,6 +1274,13 @@ std::optional<FramebufferTextureInfo> RasterizerVulkan::AccelerateDisplay( std::scoped_lock lock{texture_cache.mutex}; const auto [image_view, scaled] = texture_cache.TryFindFramebufferImageView(config, framebuffer_addr); + static u32 display_trace_count{}; + static const bool frame_capture = std::getenv("EDEN_NATIVE_FRAME_CAPTURE") != nullptr; + if (frame_capture && display_trace_count++ < 64) { + LOG_INFO(HW_GPU, "Native display source: address={:#x}, offset={:#x}, dimensions={}x{}, stride={}, accelerated={}, mask={:#x}", + framebuffer_addr, config.offset, config.width, config.height, pixel_stride, + image_view != nullptr, config.layer_stack_mask); + } if (!image_view) { return {}; } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 583cb6edaf..f5fcdd1380 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -7,6 +7,7 @@ #include <algorithm> #include <limits> #include <array> +#include <cstdlib> #include <optional> #include <span> #include <memory> @@ -148,10 +149,68 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) { return usage; } +// An ASTC 2D array whose RGBA8 decode exceeds 1 GiB (title 1's 8192x8192x68 terrain textures). +// These stay on the GPU decoder as RGBA8 with trimmed levels (TrimmedHostInfo) even when other +// ASTC images are re-compressed to BC1/BC3 on the CPU: re-compressing them would not fit either. +[[nodiscard]] bool IsHugeAstcArray(const Device& device, const ImageInfo& info) { + if (!IsPixelFormatASTC(info.format) || device.IsOptimalAstcSupported() || + info.type != ImageType::e2D || info.size.depth != 1) { + return false; + } + u64 bytes = 0; + for (s32 level = 0; level < info.resources.levels; ++level) { + bytes += u64{(std::max)(info.size.width >> level, 1U)} * + (std::max)(info.size.height >> level, 1U) * 4; + } + return bytes * info.resources.layers > (u64{1} << 30); +} + +// The host format of a huge ASTC array: the GPU decoder's RGBA8 (sRGB kept for sampling). +[[nodiscard]] std::optional<VkFormat> HugeAstcHostFormat(const Device& device, const ImageInfo& info) { + if (!IsHugeAstcArray(device, info)) { + return std::nullopt; + } + return VideoCore::Surface::IsPixelFormatSRGB(info.format) ? VK_FORMAT_A8B8G8R8_SRGB_PACK32 + : VK_FORMAT_A8B8G8R8_UNORM_PACK32; +} + +constexpr std::array<VkFormat, 2> HugeAstcViewFormats{VK_FORMAT_A8B8G8R8_UNORM_PACK32, + VK_FORMAT_A8B8G8R8_SRGB_PACK32}; + +// Opaque ASTC (ImageInfo::astc_opaque) re-compressed to BC1 instead of BC3. +[[nodiscard]] bool IsOpaqueBc1Astc(const Device& device, const ImageInfo& info) { + return info.astc_opaque && IsPixelFormatASTC(info.format) && !IsHugeAstcArray(device, info); +} + +constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM_BLOCK, + VK_FORMAT_BC1_RGBA_SRGB_BLOCK}; + +[[nodiscard]] std::optional<VkFormat> AstcHostFormat(const Device& device, const ImageInfo& info) { + if (IsOpaqueBc1Astc(device, info)) { + return VideoCore::Surface::IsPixelFormatSRGB(info.format) ? VK_FORMAT_BC1_RGBA_SRGB_BLOCK + : VK_FORMAT_BC1_RGBA_UNORM_BLOCK; + } + return HugeAstcHostFormat(device, info); +} + +// ASTC that the GPU decodes and re-compresses to BC1/BC3 itself (TranscodeAstcToBcn): no CPU +// decode, so streamed textures are ready the frame they are first used instead of seconds later. +[[nodiscard]] bool WillUseGpuBcnTranscode(const Device& device, const ImageInfo& info) { + if (!IsPixelFormatASTC(info.format) || device.IsOptimalAstcSupported() || + IsHugeAstcArray(device, info) || info.type != ImageType::e2D || info.size.depth != 1) { + return false; + } + return Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu && + Settings::values.astc_recompression.GetValue() != Settings::AstcRecompression::Uncompressed; +} + [[nodiscard]] bool WillUseAcceleratedAstcDecode(const Device& device, const ImageInfo& info) { if (!IsPixelFormatASTC(info.format) || device.IsOptimalAstcSupported()) { return false; } + if (IsHugeAstcArray(device, info)) { + return true; + } if (Settings::values.accelerate_astc.GetValue() != Settings::AstcDecodeMode::Gpu) { return false; } @@ -160,6 +219,40 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) { info.size.depth == 1; } +// The host shape of an image whose decoded copy would not fit in VRAM: a GPU-decoded ASTC array +// over 1 GiB as RGBA8 (title 1's 8192x8192x68 ASTC 8x8 virtual textures are 24 GB each) keeps only +// its mip levels from `level_offset` on, so host mip 0 is guest mip `level_offset`. Normalized +// coordinates and derivative LOD still address it correctly; explicit LODs sample one level lower. +[[nodiscard]] ImageInfo TrimmedHostInfo(const Device& device, const ImageInfo& info, u32& level_offset) { + level_offset = 0; + if (!WillUseAcceleratedAstcDecode(device, info) || info.type != ImageType::e2D) { + return info; + } + const auto decoded_bytes = [&](u32 skip) { + u64 bytes = 0; + for (s32 level = static_cast<s32>(skip); level < info.resources.levels; ++level) { + bytes += u64{(std::max)(info.size.width >> level, 1U)} * + (std::max)(info.size.height >> level, 1U) * 4; + } + return bytes * info.resources.layers; + }; + if (decoded_bytes(0) <= (u64{1} << 30)) { + return info; + } + while (level_offset + 1 < static_cast<u32>(info.resources.levels) && + decoded_bytes(level_offset) > (u64{512} << 20)) { + ++level_offset; + } + ImageInfo host = info; + host.size.width = (std::max)(info.size.width >> level_offset, 1U); + host.size.height = (std::max)(info.size.height >> level_offset, 1U); + host.resources.levels -= static_cast<s32>(level_offset); + LOG_WARNING(Render_Vulkan, "{}x{}x{} ASTC image would be {} MiB decoded; keeping mip levels {}+ ({}x{})", + info.size.width, info.size.height, info.resources.layers, decoded_bytes(0) >> 20, + level_offset, host.size.width, host.size.height); + return host; +} + [[nodiscard]] VkImageCreateInfo MakeImageCreateInfo(const Device& device, const ImageInfo& info, std::optional<VkFormat> format_override = {}) { auto format_info = @@ -527,6 +620,20 @@ TransformBufferCopies(std::span<const VideoCommon::BufferCopy> copies, size_t bu }; } +// A guest view's levels on a host image that keeps only levels >= host_level_offset. +[[nodiscard]] SubresourceRange HostRange(const Image& image, SubresourceRange range) { + const s32 skip = static_cast<s32>(image.host_level_offset); + if (skip == 0) { + return range; + } + const s32 host_levels = image.info.resources.levels - skip; + const s32 base = std::clamp(range.base.level - skip, 0, host_levels - 1); + const s32 end = std::clamp(range.base.level + range.extent.levels - skip, base + 1, host_levels); + range.base.level = base; + range.extent.levels = end - base; + return range; +} + [[nodiscard]] VkImageSubresourceRange MakeSubresourceRange(const ImageView* image_view) { SubresourceRange range = image_view->range; if (True(image_view->flags & VideoCommon::ImageViewFlagBits::Slice)) { @@ -942,9 +1049,16 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched : device{device_}, scheduler{scheduler_}, memory_allocator{memory_allocator_}, staging_buffer_pool{staging_buffer_pool_}, blit_image_helper{blit_image_helper_}, render_pass_cache{render_pass_cache_}, resolution{Settings::values.resolution_info} { - if (Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu) { + // Huge ASTC arrays always decode on the GPU (IsHugeAstcArray), whatever the decode setting. + if (Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu || + !device.IsOptimalAstcSupported()) { astc_decoder_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, compute_pass_descriptor_queue, memory_allocator); + if (Settings::values.astc_recompression.GetValue() != + Settings::AstcRecompression::Uncompressed) { + bc_encoder_pass.emplace(device, scheduler, descriptor_pool, + compute_pass_descriptor_queue); + } } if (!device.IsKhrImageFormatListSupported()) { return; @@ -1796,6 +1910,10 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 TextureCacheRuntime::GetDeviceMemoryBudget() const { + return device.GetDeviceMemoryBudget(); +} + bool TextureCacheRuntime::CanDownloadMsaa(const VideoCommon::ImageInfo& info) const { const VkImageAspectFlags aspect_mask = ImageAspectMask(info.format); if ((aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0) { @@ -1814,6 +1932,20 @@ bool TextureCacheRuntime::CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } +void TextureCacheRuntime::LogMemoryStatistics() const { + Vulkan::LogMemoryStatistics(device); +} + +bool TextureCacheRuntime::UseOpaqueAstcFormat() const { + // EDEN_NX2_ASTC_BC1=0 keeps every re-compressed ASTC image in BC3. + static const bool enabled = [] { + const char* value = std::getenv("EDEN_NX2_ASTC_BC1"); + return value == nullptr || value[0] != '0'; + }(); + return enabled && !device.IsOptimalAstcSupported() && + Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Bc3; +} + void TextureCacheRuntime::FlushDeferredClear() { scheduler.FlushDeferredClear(); } @@ -1859,6 +1991,9 @@ void TextureCacheRuntime::ReleaseMsaaScratchImage(VkImage image) { } void TextureCacheRuntime::TickFrame() { + while (!pending_transcodes.empty() && scheduler.IsFree(pending_transcodes.front().tick)) { + pending_transcodes.pop_front(); + } static constexpr u32 MAX_UNUSED_SCRATCH_FRAMES = 60; std::erase_if(msaa_scratch_images, [this](MsaaScratchImage& scratch) { if (!scheduler.IsFree(scratch.tick)) { @@ -1876,13 +2011,30 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu VAddr cpu_addr_) : VideoCommon::ImageBase(info_, gpu_addr_, cpu_addr_), scheduler{&runtime_.scheduler}, runtime{&runtime_}, - original_image(MakeImage(runtime_.device, runtime_.memory_allocator, info, - runtime->ViewFormats(info.format))), + original_image(MakeImage(runtime_.device, runtime_.memory_allocator, + TrimmedHostInfo(runtime_.device, info, host_level_offset), + IsHugeAstcArray(runtime_.device, info) + ? std::span<const VkFormat>{HugeAstcViewFormats} + : IsOpaqueBc1Astc(runtime_.device, info) + ? std::span<const VkFormat>{OpaqueAstcViewFormats} + : runtime->ViewFormats(info.format), + AstcHostFormat(runtime_.device, info))), aspect_mask(ImageAspectMask(info.format)) { - if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) { + if (original_image.IsUnbacked()) { + flags |= VideoCommon::ImageFlagBits::Unbacked; + } + if (host_level_offset != 0) { + flags |= VideoCommon::ImageFlagBits::TrimmedLevels; + } + if (IsHugeAstcArray(runtime->device, info)) { + flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; + flags |= VideoCommon::ImageFlagBits::Converted; + flags |= VideoCommon::ImageFlagBits::CostlyLoad; + } else if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) { switch (Settings::values.accelerate_astc.GetValue()) { case Settings::AstcDecodeMode::Gpu: - if (WillUseAcceleratedAstcDecode(runtime->device, info)) { + if (WillUseAcceleratedAstcDecode(runtime->device, info) || + WillUseGpuBcnTranscode(runtime->device, info)) { flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; } break; @@ -1903,11 +2055,12 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu original_image.SetObjectNameEXT(VideoCommon::Name(*this).c_str()); } current_image = &Image::original_image; - storage_image_views.resize(info.resources.levels); + const s32 host_levels = info.resources.levels - static_cast<s32>(host_level_offset); + storage_image_views.resize(host_levels); if (WillUseAcceleratedAstcDecode(runtime->device, info)) { const auto& device = runtime->device.GetLogical(); const VkFormat storage_format = VK_FORMAT_A8B8G8R8_UNORM_PACK32; - for (s32 level = 0; level < info.resources.levels; ++level) { + for (s32 level = 0; level < host_levels; ++level) { storage_image_views[level] = MakeStorageView(device, level, *original_image, storage_format); } @@ -2085,6 +2238,11 @@ void Image::DownloadMemory(VkBuffer buffer, size_t offset, void Image::DownloadMemory(std::span<VkBuffer> buffers_span, std::span<size_t> offsets_span, std::span<const VideoCommon::BufferImageCopy> copies) { + if (!original_image) { + LOG_CRITICAL(Render_Vulkan, "Missing image at download: gpu={:#x}, cpu={:#x}, type={}, format={}, flags={:#x}, size={}, dimensions={}x{}", + gpu_addr, cpu_addr, static_cast<u32>(info.type), static_cast<u32>(info.format), + static_cast<u32>(flags), unswizzled_size_bytes, info.size.width, info.size.height); + } const bool is_rescaled = True(flags & ImageFlagBits::Rescaled); if (is_rescaled) { ScaleDown(); @@ -2519,6 +2677,15 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI } } auto format_info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format); + if (IsHugeAstcArray(*device, image.info)) { + format_info.format = VideoCore::Surface::IsPixelFormatSRGB(format) + ? VK_FORMAT_A8B8G8R8_SRGB_PACK32 + : VK_FORMAT_A8B8G8R8_UNORM_PACK32; + } else if (IsOpaqueBc1Astc(*device, image.info) && IsPixelFormatASTC(format)) { + format_info.format = VideoCore::Surface::IsPixelFormatSRGB(format) + ? VK_FORMAT_BC1_RGBA_SRGB_BLOCK + : VK_FORMAT_BC1_RGBA_UNORM_BLOCK; + } if (device->ApiVersion() >= VK_API_VERSION_1_3) { const VkFormatProperties3 properties3 = device->GetPhysical().GetFormatProperties3(format_info.format); @@ -2567,7 +2734,7 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI .viewType = VkImageViewType{}, .format = format_info.format, .components = swizzle_mapping, - .subresourceRange = MakeSubresourceRange(aspect_mask, info.range), + .subresourceRange = MakeSubresourceRange(aspect_mask, HostRange(image, info.range)), }; const auto create = [&](TextureType tex_type, std::optional<u32> num_layers) { VkImageViewCreateInfo ci{create_info}; @@ -2821,6 +2988,11 @@ Sampler::Sampler(TextureCacheRuntime& runtime, const Tegra::Texture::TSCEntry& t if (tsc.mipmap_filter != TextureMipmapFilter::None) { min_lod = tsc.MinLod(); max_lod = tsc.MaxLod(); + // EDEN_NX2_FORCE_LOD0: sample only the top mip level (debug aid for mip layout problems). + if (static const bool force_lod0 = std::getenv("EDEN_NX2_FORCE_LOD0") != nullptr; force_lod0) { + min_lod = 0.0f; + max_lod = 0.25f; + } } base_ci = VkSamplerCreateInfo{ .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, @@ -3133,6 +3305,9 @@ void TextureCacheRuntime::AccelerateImageUpload( u32 z_start, u32 z_count) { if (IsPixelFormatASTC(image.info.format)) { + if (bc_encoder_pass && WillUseGpuBcnTranscode(device, image.info)) { + return TranscodeAstcToBcn(image, map, swizzles); + } return astc_decoder_pass->Assemble(image, map, swizzles); } @@ -3151,6 +3326,100 @@ void TextureCacheRuntime::AccelerateImageUpload( ASSERT(false); } +void TextureCacheRuntime::TranscodeAstcToBcn( + Image& image, const StagingBufferRef& map, + std::span<const VideoCommon::SwizzleParameters> swizzles) { + const ImageInfo& info = image.info; + const u32 width = info.size.width; + const u32 height = info.size.height; + const u32 levels = static_cast<u32>(info.resources.levels); + const u32 layers = static_cast<u32>(info.resources.layers); + constexpr VkFormat DecodeFormat = VK_FORMAT_A8B8G8R8_UNORM_PACK32; + PendingTranscode pending{ + .tick = 0, + .image = memory_allocator.CreateImage(VkImageCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .imageType = VK_IMAGE_TYPE_2D, + .format = DecodeFormat, + .extent = {width, height, 1}, + .mipLevels = levels, + .arrayLayers = layers, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_STORAGE_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + }, true), + .views = {}, + }; + for (u32 level = 0; level < levels; ++level) { + pending.views.push_back( + MakeStorageView(device.GetLogical(), level, *pending.image, DecodeFormat)); + } + astc_decoder_pass->AssembleInto( + info, 0, image.guest_size_bytes, + ASTCDecoderPass::Target{ + .image = *pending.image, + .aspect_mask = VK_IMAGE_ASPECT_COLOR_BIT, + .initialized = false, + .level_view = [&pending](s32 level) { return *pending.views[level]; }, + .wait = false, + }, + map, swizzles); + + // Encode every level into one buffer, then copy it into the image. + const bool bc3 = !IsOpaqueBc1Astc(device, info) && + Settings::values.astc_recompression.GetValue() != + Settings::AstcRecompression::Bc1; + const u64 block_bytes = bc3 ? 16 : 8; + boost::container::small_vector<VideoCommon::BufferImageCopy, 16> copies; + u64 total = 0; + for (u32 level = 0; level < levels; ++level) { + const u32 level_width = (std::max)(width >> level, 1U); + const u32 level_height = (std::max)(height >> level, 1U); + const u32 blocks_x = Common::DivCeil(level_width, 4U); + const u32 blocks_y = Common::DivCeil(level_height, 4U); + const u64 size = u64{blocks_x} * blocks_y * layers * block_bytes; + copies.push_back({ + .buffer_offset = total, + .buffer_size = size, + .buffer_row_length = blocks_x * 4, + .buffer_image_height = blocks_y * 4, + .image_subresource{ + .base_level = static_cast<s32>(level), + .base_layer = 0, + .num_layers = static_cast<s32>(layers), + }, + .image_offset{0, 0, 0}, + .image_extent{level_width, level_height, 1}, + }); + total += size; + } + const auto output = staging_buffer_pool.Request(total, MemoryUsage::DeviceLocal); + for (u32 level = 0; level < levels; ++level) { + bc_encoder_pass->Encode(*pending.views[level], copies[level].image_extent.width, + copies[level].image_extent.height, layers, bc3, output.buffer, + output.offset, total, copies[level].buffer_offset); + } + scheduler.Record([](vk::CommandBuffer cmdbuf) { + const VkMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, barrier); + }); + image.UploadMemory(output.buffer, output.offset, copies); + pending.tick = scheduler.CurrentTick(); + pending_transcodes.push_back(std::move(pending)); +} + void TextureCacheRuntime::TransitionImageLayout(Image& image) { if (!image.ExchangeInitialization()) { VkImageMemoryBarrier barrier{ diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 2e9736f707..1d92d439f4 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -6,6 +6,7 @@ #pragma once +#include <deque> #include <span> #include "video_core/texture_cache/texture_cache_base.h" @@ -66,8 +67,15 @@ public: u64 GetDeviceMemoryUsage() const; + u64 GetDeviceMemoryBudget() const; + bool CanReportMemoryUsage() const; + void LogMemoryStatistics() const; + + /// True when ASTC is re-compressed to BC3 and opaque images may use BC1 instead. + bool UseOpaqueAstcFormat() const; + bool CanDownloadMsaa(const VideoCommon::ImageInfo& info) const; [[nodiscard]] VkImage AcquireMsaaScratchImage(const VkImageCreateInfo& image_ci); @@ -157,6 +165,20 @@ public: BlitImageHelper& blit_image_helper; RenderPassCache& render_pass_cache; std::optional<ASTCDecoderPass> astc_decoder_pass; + std::optional<BcEncoderPass> bc_encoder_pass; + + /// ASTC decoded on the GPU and re-compressed to BC1/BC3 there (accelerate_astc=Gpu with + /// astc_recompression=Bc1/Bc3): decode into a temporary RGBA8 image, encode, copy. + void TranscodeAstcToBcn(Image& image, const StagingBufferRef& map, + std::span<const VideoCommon::SwizzleParameters> swizzles); + + /// Temporary decode images, freed once the GPU is past `tick`. + struct PendingTranscode { + u64 tick; + vk::Image image; + std::vector<vk::ImageView> views; + }; + std::deque<PendingTranscode> pending_transcodes; std::optional<BlockLinearUnswizzle3DPass> bl3d_unswizzle_pass; const Settings::ResolutionScalingInfo& resolution; diff --git a/src/video_core/shader_cache.cpp b/src/video_core/shader_cache.cpp index f1aff2810f..dfe80abac3 100644 --- a/src/video_core/shader_cache.cpp +++ b/src/video_core/shader_cache.cpp @@ -5,12 +5,17 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <cstring> +#include <string> +#include <unordered_set> +#include <fmt/ranges.h> #include <array> #include <vector> #include <boost/container/small_vector.hpp> #include "common/assert.h" #include "shader_recompiler/frontend/maxwell/control_flow.h" +#include "shader_recompiler/exception.h" #include "shader_recompiler/object_pool.h" #include "video_core/control/channel_state.h" #include "video_core/dirty_flags.h" @@ -43,7 +48,7 @@ void ShaderCache::SyncGuestHost() { ShaderCache::ShaderCache(Tegra::MaxwellDeviceMemoryManager& device_memory_) : device_memory{device_memory_} {} -bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { +bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { auto& dirty{maxwell3d->dirty.flags}; if (!dirty[VideoCommon::Dirty::Shaders]) { return last_shaders_valid; @@ -51,6 +56,7 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { dirty[VideoCommon::Dirty::Shaders] = false; const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()}; + const bool native = maxwell3d->GetNativeMacroEngine().native_turing; for (size_t index = 0; index < Tegra::Engines::Maxwell3D::Regs::MaxShaderProgram; ++index) { if (!maxwell3d->regs.IsShaderConfigEnabled(index)) { unique_hashes[index] = 0; @@ -63,7 +69,9 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { unique_hashes[index] = 0; continue; } - const GPUVAddr shader_addr{base_addr + shader_config.offset}; + const GPUVAddr shader_addr{native + ? (static_cast<u64>(shader_config.reserved[0] & 0xff) << 32) | shader_config.reserved[1] + : base_addr + shader_config.offset}; const std::optional<VAddr> cpu_shader_addr{gpu_memory->GpuToCpuAddress(shader_addr)}; if (!cpu_shader_addr) { LOG_ERROR(HW_GPU, "Invalid GPU address for shader {:#016x}", shader_addr); @@ -71,16 +79,68 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { return false; } const ShaderInfo* shader_info{TryGet(*cpu_shader_addr)}; + if (!shader_info && native && IsRejected(*cpu_shader_addr)) { + // A rejected program fails the same way every time it is bound: skip its draws + // without rethrowing (two C++ throws each time cost ~1-2% of the GPU thread). + maxwell3d->GetNativeMacroEngine().native_execution_failed = true; + last_shaders_valid = false; + return false; + } if (!shader_info) { - const u32 start_address{shader_config.offset}; - GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, base_addr, start_address}; - shader_info = MakeShaderInfo(env, *cpu_shader_addr); + const u32 start_address{native ? 0 : shader_config.offset}; + try { + // Native programs are lowered while the environment is built. + RethrowIfRejected(*cpu_shader_addr); + GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, native ? shader_addr : base_addr, start_address}; + shader_info = MakeShaderInfo(env, *cpu_shader_addr); + } catch (const Shader::Exception&) { + RememberRejection(*cpu_shader_addr); + // Once per program: which stage slot it is bound to, its header and the slot's + // registers (to tell how a pipeline with task/mesh programs is set up). + static std::unordered_set<GPUVAddr> described; + if (native && described.insert(shader_addr).second) { + std::array<u32, 5> header{}; + gpu_memory->ReadBlockUnsafe(shader_addr, header.data(), sizeof(header)); + std::array<u32, sizeof(shader_config) / sizeof(u32)> config{}; + std::memcpy(config.data(), &shader_config, sizeof(config)); + LOG_ERROR(HW_GPU, "Rejected program {:#x} is bound to stage {}: header {:08x}, slot {:08x}", + shader_addr, index, fmt::join(header, " "), fmt::join(config, " ")); + } + throw; + } } shader_infos[index] = shader_info; unique_hashes[index] = shader_info->unique_hash; } last_shaders_valid = true; return true; +} catch (const Shader::Exception& error) { + auto& native = maxwell3d->GetNativeMacroEngine(); + if (!native.native_turing) throw; + native.native_execution_failed = true; + last_shaders_valid = false; + // Each rejected program is logged once (the draws that use it keep failing quietly). + static std::unordered_set<std::string> logged; + if (logged.insert(error.what()).second) { + LOG_ERROR(HW_GPU, "Native graphics shader rejected: {}", error.what()); + } + // Once: show what the vertex stage has bound, to find where an unbound bank's data lives. + static std::vector<std::string> dumped_errors; + if (const std::string what{error.what()}; + what.find("constant buffer") != std::string::npos && dumped_errors.size() < 12 && + std::count(dumped_errors.begin(), dumped_errors.end(), what) < 3) { + dumped_errors.push_back(what); + LOG_INFO(HW_GPU, "VS cbuf dump for: {}", what.substr(0, 40)); + for (u32 slot = 3; slot < 8; ++slot) { + const auto& buffer = maxwell3d->state.shader_stages[0].const_buffers[slot]; + if (!buffer.enabled) continue; + std::array<u32, 24> words{}; + gpu_memory->ReadBlockUnsafe(buffer.address, words.data(), sizeof(words)); + LOG_INFO(HW_GPU, "VS cbuf {} at {:#x} size {:#x}: {:08x}", slot, buffer.address, buffer.size, + fmt::join(words, " ")); + } + } + return false; } const ShaderInfo* ShaderCache::ComputeShader() { @@ -95,22 +155,54 @@ const ShaderInfo* ShaderCache::ComputeShader() { if (const ShaderInfo* const shader = TryGet(*cpu_shader_addr)) { return shader; } - ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start}; - return MakeShaderInfo(env, *cpu_shader_addr); + try { + RethrowIfRejected(*cpu_shader_addr); + ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start}; + return MakeShaderInfo(env, *cpu_shader_addr); + } catch (const Shader::Exception&) { + RememberRejection(*cpu_shader_addr); + throw; + } +} + +void ShaderCache::RethrowIfRejected(VAddr addr) { + std::scoped_lock lock{rejected_mutex}; + if (const auto it = rejected_native.find(addr); it != rejected_native.end()) { + std::rethrow_exception(it->second); + } +} + +bool ShaderCache::IsRejected(VAddr addr) { + std::scoped_lock lock{rejected_mutex}; + return rejected_native.contains(addr); +} + +void ShaderCache::RememberRejection(VAddr addr) { + std::scoped_lock lock{rejected_mutex}; + rejected_native.emplace(addr, std::current_exception()); + rejected_lowest.store((std::min)(rejected_lowest.load(std::memory_order_relaxed), addr), + std::memory_order_relaxed); + rejected_highest.store((std::max)(rejected_highest.load(std::memory_order_relaxed), addr), + std::memory_order_relaxed); } void ShaderCache::GetGraphicsEnvironments(GraphicsEnvironments& result, const std::array<u64, NUM_PROGRAMS>& unique_hashes) { size_t env_index{}; const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()}; + const bool native = maxwell3d->GetNativeMacroEngine().native_turing; for (size_t index = 0; index < NUM_PROGRAMS; ++index) { if (unique_hashes[index] == 0) { continue; } const auto program{static_cast<Tegra::Engines::Maxwell3D::Regs::ShaderType>(index)}; auto& env{result.envs[index]}; - const u32 start_address{maxwell3d->regs.pipelines[index].offset}; - env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, base_addr, start_address}; + const auto& config = maxwell3d->regs.pipelines[index]; + const u32 start_address{native ? 0 : config.offset}; + const GPUVAddr address = native + ? (static_cast<u64>(config.reserved[0] & 0xff) << 32) | config.reserved[1] + : base_addr; + env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, address, start_address}; env.SetCachedSize(shader_infos[index]->size_bytes); result.env_ptrs[env_index++] = &env; } @@ -144,6 +236,15 @@ void ShaderCache::Register(std::unique_ptr<ShaderInfo> data, VAddr addr, size_t void ShaderCache::InvalidatePagesInRegion(VAddr addr, size_t size) { const VAddr addr_end = addr + size; + // Rejected programs are few and in a few places: look at them only for writes that can + // reach them (this runs for every inline upload and pre-draw invalidation). + if (addr_end > rejected_lowest.load(std::memory_order_relaxed) && + addr <= rejected_highest.load(std::memory_order_relaxed)) { + std::scoped_lock lock{rejected_mutex}; + boost::unordered::erase_if(rejected_native, [&](const auto& rejected) { + return rejected.first >= addr && rejected.first < addr_end; + }); + } const u64 page_end = (addr_end + YUZU_PAGESIZE - 1) >> YUZU_PAGEBITS; for (u64 page = addr >> YUZU_PAGEBITS; page < page_end; ++page) { auto it = invalidation_cache.find(page); @@ -241,7 +342,7 @@ const ShaderInfo* ShaderCache::MakeShaderInfo(GenericEnvironment& env, VAddr cpu info->unique_hash = env.CalculateHash(); info->size_bytes = env.ReadSizeBytes(); } - const size_t size_bytes{info->size_bytes}; + const size_t size_bytes{env.SourceSizeBytes() ? env.SourceSizeBytes() : info->size_bytes}; const ShaderInfo* const result{info.get()}; Register(std::move(info), cpu_addr, size_bytes); return result; diff --git a/src/video_core/shader_cache.h b/src/video_core/shader_cache.h index 997f2b0371..f4179dd435 100644 --- a/src/video_core/shader_cache.h +++ b/src/video_core/shader_cache.h @@ -7,6 +7,8 @@ #pragma once #include <array> +#include <atomic> +#include <exception> #include <memory> #include <mutex> #include <span> @@ -143,6 +145,13 @@ private: /// @brief Create a new shader entry and register it const ShaderInfo* MakeShaderInfo(GenericEnvironment& env, VAddr cpu_addr); + /// Rethrows the error a native program at this address was rejected with, if any. + void RethrowIfRejected(VAddr addr); + /// Whether a native program at this address was rejected. + bool IsRejected(VAddr addr); + /// Remembers the exception being handled as this address's rejection. + void RememberRejection(VAddr addr); + Tegra::MaxwellDeviceMemoryManager& device_memory; mutable std::mutex lookup_mutex; @@ -152,6 +161,14 @@ private: ::Common::unordered_map<u64, std::vector<Entry*>> invalidation_cache; std::vector<std::unique_ptr<ShaderInfo>> storage; std::vector<Entry*> marked_for_removal; + + /// Native programs the lowering rejected, by address, so each draw or dispatch rethrows the + /// first error instead of lowering again; dropped when guest memory there is written. + std::mutex rejected_mutex; + ::Common::unordered_map<u64, std::exception_ptr> rejected_native; + /// Bounds of every address ever rejected (never shrinks): a quick test for invalidations. + std::atomic<VAddr> rejected_lowest{~VAddr{}}; + std::atomic<VAddr> rejected_highest{}; }; } // namespace VideoCommon diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp index 4d5db7e511..96306d9a27 100644 --- a/src/video_core/shader_environment.cpp +++ b/src/video_core/shader_environment.cpp @@ -5,7 +5,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <chrono> +#include <cstdlib> +#include <thread> #include <filesystem> +#include <cstring> +#include <string> #include <fstream> #include <memory> #include <optional> @@ -21,6 +26,11 @@ #include "common/settings.h" #include <ranges> #include "shader_recompiler/environment.h" +#include "shader_recompiler/exception.h" +#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h" +#include "shader_recompiler/frontend/ampere/program_override.h" +#include "shader_recompiler/frontend/ampere/program_capture.h" +#include "shader_recompiler/frontend/ampere/program_address_map.h" #include "video_core/engines/kepler_compute.h" #include "video_core/memory_manager.h" #include "video_core/shader_environment.h" @@ -149,11 +159,26 @@ u64 GenericEnvironment::ReadInstruction(u32 address) { if (address >= cached_lowest && address < cached_highest) { return code[(address - cached_lowest) / INST_SIZE]; } + if (native_source_size) { + throw Shader::RuntimeError("Lowered native instruction address out of range: {:#x}", address); + } has_unbound_instructions = true; return gpu_memory->Read<u64>(program_base + address); } +u32 GenericEnvironment::TranslateProgramAddress(u32 address) const { + if (const auto translated = Shader::Ampere::NativeProgramAddress(code, address)) return *translated; + if (native_source_size || Shader::Ampere::HasNativeAddressMap(code)) + throw Shader::RuntimeError("Invalid native indirect branch address {:#x}", address); + return address; +} + std::optional<u64> GenericEnvironment::Analyze() { + if (native_source_size) { + // Include the converted header and a version tag to isolate this experimental ABI. + return Common::CityHash64(reinterpret_cast<const char*>(code.data()), + code.size() * sizeof(u64)) ^ 0x4f554e4345000002ULL; + } const std::optional<u64> size{TryFindSize()}; if (!size) { return std::nullopt; @@ -164,6 +189,12 @@ std::optional<u64> GenericEnvironment::Analyze() { } void GenericEnvironment::SetCachedSize(size_t size_bytes) { + if (native_source_size) { + if (size_bytes != CachedSizeBytes()) { + throw Shader::RuntimeError("Native shader changed while constructing pipeline"); + } + return; + } cached_lowest = start_address; cached_highest = start_address + static_cast<u32>(size_bytes); code.resize(CachedSizeWords()); @@ -291,6 +322,146 @@ Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, return entry; } +void GenericEnvironment::InitializeNativeCode(u32 source_header_size) { + using Shader::Ampere::Instruction; + const u32 native_header_size = source_header_size; + constexpr u32 maximum_instructions = Shader::Ampere::ProgramCaptureState::MaximumInstructions; + std::vector<Instruction> native; + Shader::Ampere::ProgramCaptureState capture; + for (u32 index = 0; index < maximum_instructions; ++index) { + const GPUVAddr address = program_base + native_header_size + index * sizeof(Instruction); + if (!gpu_memory->IsFullyMappedRange(address, sizeof(Instruction))) { + // The program may end before a branch target past its last EXIT (the capture can't + // tell); stop at the mapping's end and let the lowering judge what was captured. + if (!native.empty()) break; + throw Shader::RuntimeError("Unmapped native shader instruction at {:#x}", address); + } + auto& instruction = native.emplace_back(); + gpu_memory->ReadBlock(address, &instruction, sizeof(instruction)); + const auto status = capture.Observe(instruction, index); + if (status == Shader::Ampere::ProgramCaptureState::Result::InvalidTarget) { + if (const char* dir = std::getenv("EDEN_NX2_SHADER_DUMP")) { + const auto path = fmt::format("{}/rejected-{:x}.bin", dir, program_base + native_header_size); + if (std::FILE* file = std::fopen(path.c_str(), "wb")) { + std::fwrite(native.data(), sizeof(Instruction), std::min<size_t>(native.size(), 4096), file); + std::fclose(file); + } + } + throw Shader::RuntimeError("Native shader branch outside capture bounds at {:#x} +{:#x} [{:08x} {:08x} {:08x} {:08x}]", + program_base + native_header_size, index * 16, + instruction.words[0], instruction.words[1], + instruction.words[2], instruction.words[3]); + } + if (status == Shader::Ampere::ProgramCaptureState::Result::Complete) break; + } + auto lowered = Shader::Ampere::LowerToMaxwell(native); + // EDEN_NX2_REJECT=<hex address>,...: treat these programs as rejected (to find which newly + // lowered program changes a game's behaviour without rebuilding). + if (static const std::string reject_list = [] { + const char* text = std::getenv("EDEN_NX2_REJECT"); + return text ? "," + std::string{text} + "," : std::string{}; + }(); + !reject_list.empty() && + reject_list.find(fmt::format(",{:x},", program_base)) != std::string::npos) { + throw Shader::RuntimeError("Native shader at {:#x} rejected by EDEN_NX2_REJECT", program_base); + } + // EDEN_NX2_SHADER_DUMP=<dir>: keep every captured native program (named by its key) so the + // lowering can be tested offline against all of a game's shaders. + if (static const char* const dump_directory = std::getenv("EDEN_NX2_SHADER_DUMP"); + dump_directory != nullptr) { + const auto folder = std::filesystem::path{dump_directory}; + const auto path = folder / fmt::format("{:016x}-{}-{:x}.bin", Shader::Ampere::NativeProgramKey(native), + source_header_size == 0 ? "cs" : "gfx", program_base); + if (!std::filesystem::exists(path)) { + std::filesystem::create_directories(folder); + std::ofstream dump(path, std::ios::binary); + dump.write(reinterpret_cast<const char*>(native.data()), + native.size() * sizeof(Shader::Ampere::Instruction)); + } + } + if (!lowered) { + if (const char* directory = std::getenv("EDEN_NATIVE_SHADER_OVERRIDE_DIR")) { + const auto key = fmt::format("{:016x}", Shader::Ampere::NativeProgramKey(native)); + const auto folder = std::filesystem::path{directory}; + std::filesystem::create_directories(folder); + const auto source_path = folder / (key + ".source.bin"); + const auto override_path = folder / (key + ".override.bin"); + { + std::ofstream dump(source_path, std::ios::binary); + dump.write(reinterpret_cast<const char*>(native.data()), native.size() * sizeof(Shader::Ampere::Instruction)); + } + { + // Retain mapped bytes beyond the first EXIT for investigating + // native CALL targets in compiler library subroutines. + std::ofstream mapped(folder / (key + ".mapped.bin"), std::ios::binary); + std::array<u8, 0x1000> block{}; + for (u32 offset = 0; offset < 0x10000; offset += 0x1000) { + if (!gpu_memory->IsFullyMappedRange(program_base + offset, block.size())) break; + gpu_memory->ReadBlock(program_base + offset, block.data(), block.size()); + mapped.write(reinterpret_cast<const char*>(block.data()), block.size()); + } + } + LOG_WARNING(Shader, "Native shader paused: key={}, offset={:#x}, reason={}, source={}", + key, lowered.error_offset, lowered.error, source_path.string()); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::minutes{30}; + while (!gpu_memory->IsShuttingDown() && std::chrono::steady_clock::now() < deadline) { + if (std::filesystem::exists(override_path)) { + std::ifstream file(override_path, std::ios::binary | std::ios::ate); + const auto length = file.tellg(); + if (length <= 0 || length > 16 * 1024 * 1024) { + throw Shader::RuntimeError("Invalid native shader override size"); + } + std::vector<u8> bytes(static_cast<size_t>(length)); + file.seekg(0); + file.read(reinterpret_cast<char*>(bytes.data()), bytes.size()); + if (!file) throw Shader::RuntimeError("Native shader override read failed"); + const u32 source_bytes = Shader::Ampere::ProgramOverrideSourceSize(bytes); + if (source_bytes > native.size() * sizeof(Shader::Ampere::Instruction)) { + if (!gpu_memory->IsFullyMappedRange(program_base + native_header_size, source_bytes)) + throw Shader::RuntimeError("Unmapped extended native shader source"); + native.resize(source_bytes / sizeof(Shader::Ampere::Instruction)); + gpu_memory->ReadBlock(program_base + native_header_size, native.data(), source_bytes); + } + lowered = Shader::Ampere::DecodeProgramOverride(bytes, native); + if (!lowered) throw Shader::RuntimeError("Native shader override rejected: {}", lowered.error); + LOG_WARNING(Shader, "Native shader resumed: key={}, instructions={}", key, lowered.instructions.size()); + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds{100}); + } + } + } + if (!lowered) { + const size_t failed = lowered.error_offset / sizeof(Shader::Ampere::Instruction); + const auto words = failed < native.size() ? native[failed].words : std::array<u32, 4>{}; + throw Shader::RuntimeError( + "Native shader at {:#x}, instruction {:#x}: {} [op {:03x} {:08x} {:08x} {:08x} {:08x}]", + program_base, lowered.error_offset, lowered.error, words[0] & 0xfff, words[0], + words[1], words[2], words[3]); + } + if (lowered.required_texture_buffer != ~0U) { + // Native TEX carries its cbuf index explicitly; TEXS obtains it from the environment. + texture_bound = lowered.required_texture_buffer; + } + native_source_size = native_header_size + lowered.source_size; + code.resize(sizeof(sph) / sizeof(u64)); + std::memcpy(code.data(), &sph, sizeof(sph)); + for (const u64 instruction : lowered.instructions) { + if ((code.size() * sizeof(u64)) % 32 == 0) { + code.push_back(0); // Maxwell scheduling word, absent from native instructions. + } + code.push_back(instruction); + } + // Keep the original padding ABI, then retain the native address map as + // unreachable data. Existing disk-cache serialization carries it with code. + code.push_back(0); + Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets); + cached_lowest = 0; + cached_highest = static_cast<u32>((code.size() - 1) * sizeof(u64)); + LOG_INFO(Shader, "Lowered native shader at {:#x}: {} source bytes, {} instructions", + program_base, native_source_size, lowered.instructions.size()); +} + GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, Tegra::MemoryManager& gpu_memory_, Maxwell::ShaderType program, GPUVAddr program_base_, @@ -298,6 +469,23 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, : GenericEnvironment{gpu_memory_, program_base_, start_address_}, maxwell3d{&maxwell3d_} { gpu_memory->ReadBlock(program_base + start_address, &sph, sizeof(sph)); initial_offset = sizeof(sph); + texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot; + if (maxwell3d->GetNativeMacroEngine().native_turing) { + if (program != Maxwell::ShaderType::VertexB && program != Maxwell::ShaderType::Pixel) { + throw Shader::RuntimeError("Unsupported native graphics shader stage {}", static_cast<u32>(program)); + } + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { + const u32 native_stage = program == Maxwell::ShaderType::Pixel ? 4 : 0; + const auto& table = maxwell3d->state.shader_stages[native_stage].const_buffers[1]; + if (table.enabled && table.size && table.size <= Tegra::Engines::Maxwell3D::Regs::MaxConstBufferSize) { + std::vector<u8> bytes(table.size); + gpu_memory->ReadBlock(table.address, bytes.data(), bytes.size()); + std::ofstream dump(fmt::format("native-graphics-cbuf-{}-1.bin", static_cast<u32>(program)), std::ios::binary); + dump.write(reinterpret_cast<const char*>(bytes.data()), bytes.size()); + } + } + InitializeNativeCode(); + } gp_passthrough_mask = maxwell3d->regs.post_vtg_shader_attrib_skip_mask; switch (program) { case Maxwell::ShaderType::VertexA: @@ -331,8 +519,7 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, const u64 local_size{sph.LocalMemorySize()}; ASSERT(local_size <= (std::numeric_limits<u32>::max)()); local_memory_size = static_cast<u32>(local_size) + sph.common3.shader_local_memory_crs_size; - texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot; - is_proprietary_driver = texture_bound == 2; + is_proprietary_driver = maxwell3d->GetNativeMacroEngine().native_turing || texture_bound == 2; has_hle_engine_state = maxwell3d->engine_state == Tegra::Engines::Maxwell3D::EngineHint::OnHLEMacro; } @@ -417,6 +604,12 @@ ComputeEnvironment::ComputeEnvironment(Tegra::Engines::KeplerCompute& kepler_com is_proprietary_driver = texture_bound == 2; shared_memory_size = qmd.shared_alloc; workgroup_size = {qmd.block_dim_x, qmd.block_dim_y, qmd.block_dim_z}; + if (kepler_compute->IsNativeMode()) { + initial_offset = sizeof(sph); + is_proprietary_driver = true; + InitializeNativeCode(0); + start_address = sizeof(sph); + } } u32 ComputeEnvironment::ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) { @@ -511,7 +704,7 @@ void FileEnvironment::Deserialize(std::ifstream& file) { if (stage == Shader::Stage::Compute) { file.read(reinterpret_cast<char*>(&workgroup_size), sizeof(workgroup_size)) .read(reinterpret_cast<char*>(&shared_memory_size), sizeof(shared_memory_size)); - initial_offset = 0; + initial_offset = Shader::Ampere::HasNativeAddressMap(code) ? sizeof(sph) : 0; } else { file.read(reinterpret_cast<char*>(&sph), sizeof(sph)); initial_offset = sizeof(sph); @@ -526,6 +719,13 @@ void FileEnvironment::Dump(u64 pipeline_hash, u64 shader_hash) { DumpImpl(pipeline_hash, shader_hash, code, read_highest, read_lowest, initial_offset, stage); } +u32 FileEnvironment::TranslateProgramAddress(u32 address) const { + if (const auto translated = Shader::Ampere::NativeProgramAddress(code, address)) return *translated; + if (Shader::Ampere::HasNativeAddressMap(code)) + throw Shader::RuntimeError("Invalid cached native indirect branch address {:#x}", address); + return address; +} + u64 FileEnvironment::ReadInstruction(u32 address) { if (address < read_lowest || address > read_highest) { throw Shader::LogicError("Out of bounds address {}", address); diff --git a/src/video_core/shader_environment.h b/src/video_core/shader_environment.h index 924e61ef46..e49f01d95f 100644 --- a/src/video_core/shader_environment.h +++ b/src/video_core/shader_environment.h @@ -47,6 +47,8 @@ public: [[nodiscard]] u64 ReadInstruction(u32 address) final; + [[nodiscard]] u32 TranslateProgramAddress(u32 address) const override; + [[nodiscard]] std::optional<u64> Analyze(); void SetCachedSize(size_t size_bytes); @@ -57,6 +59,10 @@ public: [[nodiscard]] size_t ReadSizeBytes() const noexcept; + [[nodiscard]] size_t SourceSizeBytes() const noexcept { + return native_source_size; + } + [[nodiscard]] bool CanBeSerialized() const noexcept; [[nodiscard]] u64 CalculateHash() const; @@ -72,6 +78,9 @@ public: protected: std::optional<u64> TryFindSize(); + void InitializeNativeCode(u32 source_header_size = 128); + size_t native_source_size{}; + Tegra::Texture::TICEntry ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw); @@ -172,6 +181,8 @@ public: [[nodiscard]] u64 ReadInstruction(u32 address) override; + [[nodiscard]] u32 TranslateProgramAddress(u32 address) const override; + [[nodiscard]] u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) override; [[nodiscard]] Shader::TextureType ReadTextureType(u32 handle) override; diff --git a/src/video_core/texture_cache/accelerated_swizzle.cpp b/src/video_core/texture_cache/accelerated_swizzle.cpp index 4c3f724d79..8ae0ed218d 100644 --- a/src/video_core/texture_cache/accelerated_swizzle.cpp +++ b/src/video_core/texture_cache/accelerated_swizzle.cpp @@ -37,6 +37,7 @@ BlockLinearSwizzle2DParams MakeBlockLinearSwizzle2DParams(const SwizzleParameter .x_shift = GOB_SIZE_SHIFT + block.height + block.depth, .block_height = block.height, .block_height_mask = (1U << block.height) - 1, + .native_tiling = info.native_tiling, }; } @@ -63,6 +64,7 @@ BlockLinearSwizzle3DParams MakeBlockLinearSwizzle3DParams(const SwizzleParameter .block_height_mask = (1U << block.height) - 1, .block_depth = block.depth, .block_depth_mask = (1U << block.depth) - 1, + .native_tiling = info.native_tiling, }; } diff --git a/src/video_core/texture_cache/accelerated_swizzle.h b/src/video_core/texture_cache/accelerated_swizzle.h index d4250da689..8296950c38 100644 --- a/src/video_core/texture_cache/accelerated_swizzle.h +++ b/src/video_core/texture_cache/accelerated_swizzle.h @@ -20,6 +20,7 @@ struct BlockLinearSwizzle2DParams { u32 x_shift; u32 block_height; u32 block_height_mask; + u32 native_tiling; }; struct BlockLinearSwizzle3DParams { @@ -33,6 +34,7 @@ struct BlockLinearSwizzle3DParams { u32 block_height_mask; u32 block_depth; u32 block_depth_mask; + u32 native_tiling; }; [[nodiscard]] BlockLinearSwizzle2DParams MakeBlockLinearSwizzle2DParams( diff --git a/src/video_core/texture_cache/format_lookup_table.cpp b/src/video_core/texture_cache/format_lookup_table.cpp index d6e019cddf..20bb96aae3 100644 --- a/src/video_core/texture_cache/format_lookup_table.cpp +++ b/src/video_core/texture_cache/format_lookup_table.cpp @@ -157,6 +157,7 @@ PixelFormat PixelFormatFromTextureInfo(TextureFormat format, ComponentType red, return PixelFormat::S8_UINT_D24_UNORM; case Hash(TextureFormat::G24R8, UINT, UNORM, UNORM, UNORM, LINEAR): return PixelFormat::S8_UINT_D24_UNORM; + case Hash(TextureFormat::S8Z24, UNORM, UINT, UNORM, UNORM, LINEAR): case Hash(TextureFormat::S8Z24, UNORM, UINT, UINT, UINT, LINEAR): return PixelFormat::D24_UNORM_S8_UINT; case Hash(TextureFormat::Z32_X24S8, FLOAT, UINT, UNORM, UNORM, LINEAR): diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h index 79fc8505d6..47bab9617a 100644 --- a/src/video_core/texture_cache/image_base.h +++ b/src/video_core/texture_cache/image_base.h @@ -45,6 +45,11 @@ enum class ImageFlagBits : u32 { AsynchronousDecode = 1 << 16, IsDecoding = 1 << 17, ///< Is currently being decoded asynchronously. + Unbacked = 1 << 18, ///< Host image has no memory (too big to back); never upload/download. + /// The host image keeps only mip levels >= host_level_offset (its decoded copy would not fit + /// in VRAM); uploads skip the rest, downloads are never done. + TrimmedLevels = 1 << 19, + ZeroFilled = 1 << 20, ///< Cleared to zero blocks before its first asynchronous decode. }; DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits) @@ -93,6 +98,7 @@ struct ImageBase { u32 unswizzled_size_bytes = 0; u32 converted_size_bytes = 0; u32 scale_rating = 0; + u32 host_level_offset = 0; ///< Guest mip level held by host mip 0 (see TrimmedLevels). u64 scale_tick = 0; bool has_scaled = false; diff --git a/src/video_core/texture_cache/image_info.cpp b/src/video_core/texture_cache/image_info.cpp index 8b59eb6841..7e86bf00cf 100644 --- a/src/video_core/texture_cache/image_info.cpp +++ b/src/video_core/texture_cache/image_info.cpp @@ -4,9 +4,13 @@ // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <algorithm> +#include <array> +#include <vector> #include <fmt/ranges.h> #include "common/assert.h" +#include "common/logging.h" #include "common/settings.h" #include "video_core/surface.h" #include "video_core/texture_cache/format_lookup_table.h" @@ -109,14 +113,23 @@ ImageInfo::ImageInfo(const TICEntry& config) noexcept { size.height = config.Height(); resources.layers = config.BaseLayer() + config.Depth() * 6; break; - case TextureType::Texture3D: + case TextureType::Texture3D: { ASSERT(config.BaseLayer() == 0); type = ImageType::e3D; size.width = config.Width(); size.height = config.Height(); size.depth = config.Depth(); resources.layers = 1; + // A few distinct 3D descriptors, raw (to check Switch 2 volume fields). + static std::vector<std::array<u64, 4>> logged; + if (logged.size() < 16 && std::find(logged.begin(), logged.end(), config.raw) == logged.end()) { + logged.push_back(config.raw); + LOG_INFO(HW_GPU, "3D TIC {:016x} {:016x} {:016x} {:016x}: {}x{}x{} block_depth {} levels {}", + config.raw[0], config.raw[1], config.raw[2], config.raw[3], size.width, size.height, + size.depth, config.block_depth.Value(), config.max_mip_level.Value() + 1); + } break; + } case TextureType::Texture1DBuffer: type = ImageType::Buffer; size.width = config.Width(); diff --git a/src/video_core/texture_cache/image_info.h b/src/video_core/texture_cache/image_info.h index eb490a6424..79237aa482 100644 --- a/src/video_core/texture_cache/image_info.h +++ b/src/video_core/texture_cache/image_info.h @@ -14,6 +14,17 @@ namespace VideoCommon { using Tegra::Texture::TICEntry; using VideoCore::Surface::PixelFormat; +// Native descriptors address the selected view directly. The legacy NVN +// layer-base bookkeeping is not an additional array prefix for this ABI. +inline TICEntry TextureViewHeader(TICEntry entry, bool native) { + if (native) { + entry.layer_base_0_2.Assign(0); + entry.layer_base_3_7.Assign(0); + entry.layer_base_8_10.Assign(0); + } + return entry; +} + struct ImageInfo { ImageInfo() = default; explicit ImageInfo(const TICEntry& config) noexcept; @@ -37,11 +48,14 @@ struct ImageInfo { u32 maybe_unaligned_layer_stride = 0; u32 num_samples = 1; u32 tile_width_spacing = 0; + bool native_tiling = false; bool rescaleable = false; bool downscaleable = false; bool forced_flushed = false; bool dma_downloaded = false; bool is_sparse = false; + // ASTC data whose blocks carry no alpha: re-compressed to BC1 instead of BC3 (half the size). + bool astc_opaque = false; }; } // namespace VideoCommon diff --git a/src/video_core/texture_cache/pending_download.h b/src/video_core/texture_cache/pending_download.h new file mode 100644 index 0000000000..52a48e82d3 --- /dev/null +++ b/src/video_core/texture_cache/pending_download.h @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include <algorithm> +#include <memory> +#include <unordered_set> +#include <cstddef> +#include <utility> +#include "common/slot_vector.h" + +namespace VideoCommon { + +template <class Image> +struct PendingImageDownload { + bool is_swizzle; + size_t async_buffer_id; + Common::SlotId object_id; + std::shared_ptr<Image> retired_image{}; + + Image& GetImage(Common::SlotVector<Image>& images) const { + return retired_image ? *retired_image : images[object_id]; + } +}; + +// Whole-image copies are recorded together at commit time. Repeated requests +// for the same live image therefore observe the same contents. Keep the first +// occurrence so reverse CPU writeback order remains unchanged for aliases. +template <class Image, class Downloads> +size_t DeduplicatePendingImageDownloads(Downloads& downloads, Common::SlotVector<Image>& images) { + std::unordered_set<const Image*> seen; + const size_t previous_size = downloads.size(); + const auto end = std::remove_if(downloads.begin(), downloads.end(), [&](const auto& download) { + return download.is_swizzle && !seen.insert(&download.GetImage(images)).second; + }); + downloads.erase(end, downloads.end()); + return previous_size - downloads.size(); +} + +// Cache slots may be recycled before queued GPU downloads are submitted or +// consumed. Keep the actual image, including its metadata, until those finish. +template <class Image, class Downloads, class Batches> +bool RetainPendingImageDownloads(Common::SlotId id, Image& image, Downloads& pending, + Batches& submitted) { + std::shared_ptr<Image> retained; + const auto retain = [&](auto& downloads) { + for (auto& download : downloads) { + if (!download.is_swizzle || download.object_id != id || download.retired_image) continue; + if (!retained) retained = std::make_shared<Image>(std::move(image)); + download.retired_image = retained; + } + }; + retain(pending); + for (auto& downloads : submitted) retain(downloads); + return retained != nullptr; +} + +} // namespace VideoCommon diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index e92e54f8ce..fb8adef161 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -6,7 +6,11 @@ #pragma once +#include <algorithm> +#include <cstdlib> #include <limits> +#include <map> +#include <vector> #include <optional> #include <bit> #include "common/container/unordered_map.h" @@ -24,6 +28,7 @@ #include "video_core/texture_cache/samples_helper.h" #include "video_core/texture_cache/texture_cache_base.h" #include "video_core/texture_cache/util.h" +#include "video_core/nx2_profile.h" #include "video_core/textures/decoders.h" namespace VideoCommon { @@ -76,6 +81,7 @@ TextureCache<P>::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag critical_memory = DEFAULT_CRITICAL_MEMORY + 1_GiB; minimum_memory = 0; } + ApplyThresholdOverrides(); const bool gpu_unswizzle_enabled = Settings::values.gpu_unswizzle_enabled.GetValue(); @@ -148,6 +154,7 @@ void TextureCache<P>::RunGarbageCollector() { if (True(image.flags & ImageFlagBits::Tracked)) { UntrackImage(image, image_id); } + ++(True(image.flags & ImageFlagBits::CostlyLoad) ? gc_costly_deletes : gc_deletes); UnregisterImage(image_id); DeleteImage(image_id, image.scale_tick > frame_tick + 5); if (aggressive_mode && total_used_memory < critical_memory) { @@ -168,11 +175,118 @@ void TextureCache<P>::RunGarbageCollector() { } template <class P> +void TextureCache<P>::ApplyThresholdOverrides(bool log) { + // EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB override the collector's thresholds. + for (const auto& [name, target] : {std::pair{"EDEN_NX2_TC_EXPECTED_MIB", &expected_memory}, + std::pair{"EDEN_NX2_TC_CRITICAL_MIB", &critical_memory}}) { + if (const char* value = std::getenv(name)) { + *target = std::strtoull(value, nullptr, 10) << 20; + } + } + if (log) { + LOG_INFO(HW_GPU, "Texture cache thresholds: expected {} MiB, critical {} MiB, minimum {} MiB", + expected_memory >> 20, critical_memory >> 20, minimum_memory >> 20); + } +} + +template <class P> void TextureCache<P>::TickFrame() { + if constexpr (HAS_DEVICE_MEMORY_INFO) { + // Switch 2 titles (native tiling) stream far more texture data than Eden's thresholds + // were tuned for: title 1's attract loop alone keeps ~5.3 GB resident, so the collector + // sat in aggressive mode evicting decoded ASTC (and decoding it again). Total use is + // measured from the driver, so collect only near the device budget instead. + if (!native_thresholds_applied && gpu_memory != nullptr && gpu_memory->UsesNativeTiling()) { + native_thresholds_applied = true; + const u64 device_local = runtime.GetDeviceLocalMemory(); + critical_memory = (std::max)(critical_memory, device_local - 256_MiB); + expected_memory = (std::max)(expected_memory, device_local - 1280_MiB); + ApplyThresholdOverrides(); + native_critical_memory = critical_memory; + native_expected_memory = expected_memory; + } + // Follow the driver's budget (re-read now and then): it shrinks when other programs use + // VRAM (another game: 7.0 -> 6.4 GB), and collecting only past it means the driver pages + // textures out. It is also the room to use: Eden's device memory figure is capped near + // 7 GB, which on a 12 GB card (11.2 GB budget) kept title 1 evicting textures with 3 GB + // unused. EDEN_NX2_TC_*_MIB overrides still win. + if constexpr (requires { runtime.GetDeviceMemoryBudget(); }) { + if (native_thresholds_applied && budget_check_frames++ % 60 == 0) { + const u64 budget = runtime.GetDeviceMemoryBudget(); + if (budget > 2_GiB) { + const u64 old_critical = critical_memory; + critical_memory = budget - 768_MiB; + expected_memory = budget - 1536_MiB; + // Log only real changes (the budget moves by a few MiB all the time). + const u64 change = critical_memory > old_critical + ? critical_memory - old_critical + : old_critical - critical_memory; + ApplyThresholdOverrides(change >= 256_MiB); + } + } + } + } // If we can obtain the memory info, use it instead of the estimate. if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); } + // Every 600 frames: device memory in use against the collector's thresholds, and what the + // images themselves account for (VRAM pressure makes the collector thrash). + if (static u64 frames{}; ++frames % 600 == 0) { + u64 image_bytes{}; + size_t images{}; + std::vector<std::pair<u64, const Image*>> largest; + std::map<u32, std::pair<u64, u32>> by_format; + for (const auto [image_id, image] : slot_images) { + const u64 bytes = GetScaledImageSizeBytes(*image); + image_bytes += bytes; + ++images; + largest.emplace_back(bytes, image); + auto& format = by_format[static_cast<u32>(image->info.format)]; + format.first += bytes; + ++format.second; + } + LOG_INFO(HW_GPU, "Texture cache: device memory {} MiB (expected {} MiB, critical {} MiB), {} images {} MiB", + total_used_memory >> 20, expected_memory >> 20, critical_memory >> 20, images, + image_bytes >> 20); + std::partial_sort(largest.begin(), largest.begin() + std::min<size_t>(12, largest.size()), + largest.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); + for (size_t i = 0; i < std::min<size_t>(12, largest.size()); ++i) { + const Image& image = *largest[i].second; + LOG_INFO(HW_GPU, " {} MiB: {:#x} {}x{}x{} layers {} levels {} format {} type {} flags {:#x}", + largest[i].first >> 20, image.gpu_addr, image.info.size.width, image.info.size.height, + image.info.size.depth, image.info.resources.layers, image.info.resources.levels, + static_cast<u32>(image.info.format), static_cast<u32>(image.info.type), + static_cast<u32>(image.flags)); + } + std::vector<std::pair<u64, u32>> formats; + for (const auto& [format, total] : by_format) formats.emplace_back(total.first, format); + std::sort(formats.rbegin(), formats.rend()); + for (size_t i = 0; i < std::min<size_t>(8, formats.size()); ++i) { + LOG_INFO(HW_GPU, " format {}: {} MiB in {} images", formats[i].second, formats[i].first >> 20, + by_format[formats[i].second].second); + } + if constexpr (requires { runtime.LogMemoryStatistics(); }) { + runtime.LogMemoryStatistics(); + } + LOG_INFO(HW_GPU, "Texture cache last 600 frames: created {} images, deleted {} (collector {}, {} costly), {} async decodes, ASTC {} opaque / {} alpha", + images_created, images_deleted, gc_deletes + gc_costly_deletes, gc_costly_deletes, + async_decodes_queued, astc_opaque_images, astc_alpha_images); + astc_opaque_images = astc_alpha_images = 0; + std::vector<std::pair<u32, u64>> shapes; + for (const auto& [key, value] : created_shapes) shapes.emplace_back(value.first, key); + std::sort(shapes.rbegin(), shapes.rend()); + for (size_t i = 0; i < std::min<size_t>(10, shapes.size()); ++i) { + const u64 key = shapes[i].second; + LOG_INFO(HW_GPU, " created {}x: format {} type {} levels {} layers {} {}x{} (last at {:#x})", + shapes[i].first, key >> 56, (key >> 52) & 0xF, (key >> 48) & 0xF, + (key >> 36) & 0xFFF, (key >> 18) & 0x3FFFF, key & 0x3FFFF, + created_shapes[key].second); + } + created_shapes.clear(); + gc_deletes = gc_costly_deletes = async_decodes_queued = 0; + images_created = images_deleted = 0; + } if (total_used_memory > minimum_memory) { RunGarbageCollector(); } @@ -597,7 +711,7 @@ void TextureCache<P>::WriteMemory(DAddr cpu_addr, size_t size) { template <class P> bool TextureCache<P>::IsDownloadable(const ImageBase& image) const noexcept { - if (!image.IsSafeGpuCopy()) { + if (!image.IsSafeGpuCopy() || True(image.flags & (ImageFlagBits::Unbacked | ImageFlagBits::TrimmedLevels))) { return false; } if (image.info.num_samples == 1) { @@ -714,6 +828,14 @@ bool TextureCache<P>::BlitImage(const Tegra::Engines::Fermi2D::Surface& dst, Image& dst_image = slot_images[dst_id]; Image& src_image = slot_images[src_id]; + if (True(dst_image.flags & ImageFlagBits::Unbacked)) { + static u32 logged{}; + if (logged++ < 64) { + LOG_INFO(HW_GPU, "2D blit into unbacked image {:#x}: dst {:#x} {}x{} from {:#x} {}x{}", + dst_image.gpu_addr, dst.Address(), dst.width, dst.height, src.Address(), + src.width, src.height); + } + } bool is_src_rescaled = True(src_image.flags & ImageFlagBits::Rescaled); bool is_dst_rescaled = True(dst_image.flags & ImageFlagBits::Rescaled); @@ -803,6 +925,17 @@ std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImage if (it == page_table.end()) { return {}; } + const auto view_format = [&]() { + switch (config.pixel_format) { + case Service::android::PixelFormat::Rgb565: + return PixelFormat::R5G6B5_UNORM; + case Service::android::PixelFormat::Bgra8888: + return PixelFormat::B8G8R8A8_UNORM; + default: + return PixelFormat::A8B8G8R8_UNORM; + } + }(); + const auto& image_map_ids = it->second; boost::container::small_vector<ImageId, 4> valid_image_ids; for (const ImageMapId map_id : image_map_ids) { @@ -814,20 +947,17 @@ std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImage if (image.image_view_ids.empty()) { continue; } + if (!VideoCore::Surface::IsViewCompatible(image.info.format, view_format, false, true)) { + static u32 rejected_alias_count{}; + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && rejected_alias_count++ < 8) { + LOG_WARNING(HW_GPU, "Ignoring incompatible framebuffer alias: address={:#x}, image={}, view={}", + cpu_addr, image.info.format, view_format); + } + continue; + } valid_image_ids.push_back(map.image_id); } - const auto view_format = [&]() { - switch (config.pixel_format) { - case Service::android::PixelFormat::Rgb565: - return PixelFormat::R5G6B5_UNORM; - case Service::android::PixelFormat::Bgra8888: - return PixelFormat::B8G8R8A8_UNORM; - default: - return PixelFormat::A8B8G8R8_UNORM; - } - }(); - const auto GetImageViewForFramebuffer = [&](ImageId image_id) { ImageViewInfo info{ImageViewType::e2D, view_format}; if (config.blending == Tegra::BlendMode::Opaque) { @@ -860,6 +990,11 @@ bool TextureCache<P>::HasUncommittedFlushes() const noexcept { } template <class P> +bool TextureCache<P>::HasAsyncFlushWork() const noexcept { + return !uncommitted_downloads.empty() || !uncommitted_async_buffers.empty(); +} + +template <class P> bool TextureCache<P>::ShouldWaitAsyncFlushes() const noexcept { return !committed_downloads.empty() && !committed_downloads.front().empty(); } @@ -869,6 +1004,7 @@ void TextureCache<P>::CommitAsyncFlushes() { // This is intentionally passing the value by copy if constexpr (IMPLEMENTS_ASYNC_DOWNLOADS) { auto& download_ids = uncommitted_downloads; + const size_t redundant_images = DeduplicatePendingImageDownloads(download_ids, slot_images); if (download_ids.empty()) { committed_downloads.emplace_back(std::move(uncommitted_downloads)); uncommitted_downloads.clear(); @@ -882,17 +1018,32 @@ void TextureCache<P>::CommitAsyncFlushes() { for (PendingDownload& download_info : download_ids) { if (download_info.is_swizzle) { total_size_bytes += - Common::AlignUp(slot_images[download_info.object_id].unswizzled_size_bytes, 64); + Common::AlignUp(download_info.GetImage(slot_images).unswizzled_size_bytes, 64); any_none_dma = true; download_info.async_buffer_id = last_async_buffer_id; } } if (any_none_dma) { + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && total_size_bytes >= 64_MiB) { + LOG_WARNING(HW_GPU, "Large image readback batch: bytes={}, entries={}, duplicates_removed={}, submitted_batches={}, retained_buffers={}", + total_size_bytes, download_ids.size(), redundant_images, + committed_downloads.size(), async_buffers_death_ring.size()); + size_t traced_images = 0; + for (const auto& download : download_ids) { + if (!download.is_swizzle || traced_images++ >= 8) continue; + const auto& image = download.GetImage(slot_images); + LOG_WARNING(HW_GPU, "Readback image: gpu={:#x}, cpu={:#x}, format={}, size={}x{}x{}, levels={}, layers={}, layer_stride={}, guest_bytes={}, download_bytes={}", + image.gpu_addr, image.cpu_addr, image.info.format, + image.info.size.width, image.info.size.height, image.info.size.depth, + image.info.resources.levels, image.info.resources.layers, + image.info.layer_stride, image.guest_size_bytes, image.unswizzled_size_bytes); + } + } auto download_map = runtime.DownloadStagingBuffer(total_size_bytes, true); for (const PendingDownload& download_info : download_ids) { if (download_info.is_swizzle) { - Image& image = slot_images[download_info.object_id]; + Image& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); image.DownloadMemory(download_map, copies); download_map.offset += Common::AlignUp(image.unswizzled_size_bytes, 64); @@ -925,7 +1076,7 @@ void TextureCache<P>::PopAsyncFlushes() { auto& download_info = download_ids[i - 1]; auto& download_buffer = download_map[download_info.async_buffer_id]; if (download_info.is_swizzle) { - const ImageBase& image = slot_images[download_info.object_id]; + const ImageBase& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); download_buffer.offset -= Common::AlignUp(image.unswizzled_size_bytes, 64); std::span<u8> download_span = @@ -954,7 +1105,7 @@ void TextureCache<P>::PopAsyncFlushes() { size_t total_size_bytes = 0; for (const PendingDownload& download_info : download_ids) { if (download_info.is_swizzle) { - total_size_bytes += slot_images[download_info.object_id].unswizzled_size_bytes; + total_size_bytes += download_info.GetImage(slot_images).unswizzled_size_bytes; } } auto download_map = runtime.DownloadStagingBuffer(total_size_bytes); @@ -963,7 +1114,7 @@ void TextureCache<P>::PopAsyncFlushes() { if (!download_info.is_swizzle) { continue; } - Image& image = slot_images[download_info.object_id]; + Image& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); image.DownloadMemory(download_map, copies); download_map.offset += image.unswizzled_size_bytes; @@ -976,7 +1127,7 @@ void TextureCache<P>::PopAsyncFlushes() { if (!download_info.is_swizzle) { continue; } - const ImageBase& image = slot_images[download_info.object_id]; + const ImageBase& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, download_span, swizzle_data_buffer); download_map.offset += image.unswizzled_size_bytes; @@ -1117,11 +1268,18 @@ void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) { // Only upload modified images return; } + VideoCommon::Nx2Profile::Scope profile_scope{VideoCommon::Nx2Profile::Zone::TextureUpload}; image.flags &= ~ImageFlagBits::CpuModified; TrackImage(image, image_id); + if (True(image.flags & ImageFlagBits::Unbacked)) { + // Nothing to upload into (and staging it could take tens of GB). + runtime.TransitionImageLayout(image); + return; + } + if (image.info.num_samples > 1 && !runtime.CanUploadMSAA()) { LOG_WARNING(HW_GPU, "MSAA image uploads are not implemented"); runtime.TransitionImageLayout(image); @@ -1156,6 +1314,24 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging) const std::span<u8> mapped_span = staging.mapped_span; const GPUVAddr gpu_addr = image.gpu_addr; + if (True(image.flags & ImageFlagBits::AcceleratedUpload) && image.host_level_offset != 0) { + // Only levels >= host_level_offset exist on the host: read each layer's tail of the mip + // chain into a compact buffer (layer stride = that tail's size) and decode those levels. + auto uploads = FullUploadSwizzles(image.info); + const size_t level_offset = uploads[image.host_level_offset].buffer_offset; + const size_t tail = image.info.layer_stride - level_offset; + for (s32 layer = 0; layer < image.info.resources.layers; ++layer) { + gpu_memory->ReadBlock(gpu_addr + layer * image.info.layer_stride + level_offset, + mapped_span.data() + layer * tail, tail, + VideoCommon::CacheType::NoTextureCache); + } + uploads.erase(uploads.begin(), uploads.begin() + image.host_level_offset); + for (auto& upload : uploads) { + upload.buffer_offset -= level_offset; + } + runtime.AccelerateImageUpload(image, staging, uploads, 0, 0); + return; + } if (True(image.flags & ImageFlagBits::AcceleratedUpload)) { gpu_memory->ReadBlock(gpu_addr, mapped_span.data(), mapped_span.size_bytes(), VideoCommon::CacheType::NoTextureCache); @@ -1178,7 +1354,18 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging) } template <class P> -ImageViewId TextureCache<P>::CreateImageView(const TICEntry& config) { +ImageViewId TextureCache<P>::CreateImageView(const TICEntry& raw_config) { + const bool native = gpu_memory->UsesNativeTiling(); + if (native && raw_config.BaseLayer() && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { + static u32 texture_view_traces{}; + if (texture_view_traces++ < 8) { + LOG_WARNING(HW_GPU, "Native texture view header: address={:#x}, dimensions={}x{}x{}, type={}, legacy_layer={}, raw={:016x}/{:016x}/{:016x}/{:016x}", + raw_config.Address(), raw_config.Width(), raw_config.Height(), raw_config.Depth(), + static_cast<u32>(raw_config.texture_type.Value()), raw_config.BaseLayer(), + raw_config.raw[0], raw_config.raw[1], raw_config.raw[2], raw_config.raw[3]); + } + } + const TICEntry config = TextureViewHeader(raw_config, native); const ImageInfo info(config); if (info.type == ImageType::Buffer) { const ImageViewInfo view_info(config, 0); @@ -1213,12 +1400,9 @@ ImageId TextureCache<P>::FindOrInsertImage(const ImageInfo& info, GPUVAddr gpu_a template <class P> ImageId TextureCache<P>::FindImage(const ImageInfo& info, GPUVAddr gpu_addr, RelaxedOptions options) { - std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + const std::optional<DAddr> cpu_addr = ImageCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info)); if (!cpu_addr) { - cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info)); - if (!cpu_addr) { - return ImageId{}; - } + return ImageId{}; } const bool broken_views = runtime.HasBrokenTextureViewFormats() || True(options & RelaxedOptions::ForceBrokenViews); @@ -1344,9 +1528,10 @@ u64 TextureCache<P>::GetScaledImageSizeBytes(const ImageBase& image) { template <class P> void TextureCache<P>::QueueAsyncDecode(Image& image, ImageId image_id) { UNIMPLEMENTED_IF(False(image.flags & ImageFlagBits::Converted)); - LOG_INFO(HW_GPU, "Queuing async texture decode"); + LOG_DEBUG(HW_GPU, "Queuing async texture decode {:#x}", image.gpu_addr); image.flags |= ImageFlagBits::IsDecoding; + ++async_decodes_queued; auto decode = std::make_unique<AsyncDecodeContext>(); auto* decode_ptr = decode.get(); decode->image_id = image_id; @@ -1356,6 +1541,18 @@ void TextureCache<P>::QueueAsyncDecode(Image& image, ImageId image_id) { Tegra::Memory::GpuGuestMemory<u8, Tegra::Memory::GuestMemoryFlags::UnsafeRead> swizzle_data(*gpu_memory, image.gpu_addr, image.guest_size_bytes, &swizzle_data_buffer); auto copies = UnswizzleImage(*gpu_memory, image.gpu_addr, image.info, swizzle_data, local_unswizzle_data_buffer); const size_t out_size = MapSizeBytes(image); + if (False(image.flags & ImageFlagBits::ZeroFilled)) { + // Until the decode lands (a second or more while title 1 streams ~40 textures a second) + // a new image showed whatever its memory last held, often other textures. Zero blocks + // read as opaque black (BC1) or transparent (BC3) instead. + image.flags |= ImageFlagBits::ZeroFilled; + auto zero_copies = copies; + ConvertedLayout(image.info, zero_copies); + auto staging = runtime.UploadStagingBuffer(out_size); + std::memset(staging.mapped_span.data(), 0, out_size); + image.UploadMemory(staging, zero_copies); + runtime.InsertUploadMemoryBarrier(); + } auto func = [out_size, copies, info = image.info, input = std::move(local_unswizzle_data_buffer), @@ -1392,12 +1589,26 @@ void TextureCache<P>::TickAsyncDecode() { auto i = async_decodes.begin(); while (i != async_decodes.end()) { auto* async_decode = i->get(); - std::unique_lock lock{async_decode->mutex}; - if (!async_decode->complete) { - ++i; + { + // Release the lock before the context is erased below: unlocking it afterwards wrote + // into freed memory (heap corruption, crashes minutes into title 1). + std::unique_lock lock{async_decode->mutex}; + if (!async_decode->complete) { + ++i; + continue; + } + } + if (async_decode->cancelled) { + i = async_decodes.erase(i); continue; } Image& image = slot_images[async_decode->image_id]; + if (async_decode->decoded_data.size() > MapSizeBytes(image)) { + LOG_ERROR(HW_GPU, "Async decode of {} bytes for a {}-byte image at {:#x}, dropped", + async_decode->decoded_data.size(), MapSizeBytes(image), image.gpu_addr); + i = async_decodes.erase(i); + continue; + } auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image)); std::memcpy(staging.mapped_span.data(), async_decode->decoded_data.data(), async_decode->decoded_data.size()); @@ -1513,20 +1724,42 @@ bool TextureCache<P>::ScaleDown(Image& image) { } template <class P> +std::optional<DAddr> TextureCache<P>::ImageCpuAddress(GPUVAddr gpu_addr, size_t size) const { + if (const std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr)) { + return cpu_addr; + } + if (const std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, size)) { + return cpu_addr; + } + // No CPU backing (title 1 renders a bloom chain at GPU 0xf0000-0x15b600): a stand-in address + // fixed by the GPU address, at or above ~(1 << 40) so TrackImage skips it. A fresh address per + // lookup made every use create another image (thousands a minute, never found again). + constexpr GPUVAddr StandInRange = 1ULL << 40; + if (gpu_addr >= StandInRange || size > StandInRange - gpu_addr - 1) { + return std::nullopt; + } + return ~(1ULL << 40ULL) + gpu_addr; +} + +template <class P> ImageId TextureCache<P>::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr, RelaxedOptions options) { - std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + std::optional<DAddr> cpu_addr = ImageCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info)); if (!cpu_addr) { - const auto size = CalculateGuestSizeInBytes(info); - cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, size); - if (!cpu_addr) { - const DAddr fake_addr = ~(1ULL << 40ULL) + virtual_invalid_space; - virtual_invalid_space += Common::AlignUp(size, 32); - cpu_addr = std::optional<DAddr>(fake_addr); - } + const DAddr fake_addr = ~(1ULL << 40ULL) + virtual_invalid_space; + virtual_invalid_space += Common::AlignUp(CalculateGuestSizeInBytes(info), 32); + cpu_addr = std::optional<DAddr>(fake_addr); } ASSERT_MSG(cpu_addr, "Tried to insert an image to an invalid gpu_addr={:#x}", gpu_addr); - const ImageId image_id = JoinImages(info, gpu_addr, *cpu_addr); + ImageInfo storage_info = info; + storage_info.native_tiling = gpu_memory->UsesNativeTiling(); + if (storage_info.native_tiling && info.type == ImageType::e3D) { + // Switch 2 3D images (lighting/fog volumes) are rare: log each one's shape. + LOG_INFO(HW_GPU, "3D image at {:#x}: {}x{}x{} levels {} format {} block {}x{}x{}", + gpu_addr, info.size.width, info.size.height, info.size.depth, info.resources.levels, + static_cast<u32>(info.format), info.block.width, info.block.height, info.block.depth); + } + const ImageId image_id = JoinImages(storage_info, gpu_addr, *cpu_addr); const Image& image = slot_images[image_id]; // Using "image.gpu_addr" instead of "gpu_addr" is important because it might be different const auto [it, is_new] = image_allocs_table.try_emplace(image.gpu_addr); @@ -1634,10 +1867,36 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA } } + if constexpr (requires { runtime.UseOpaqueAstcFormat(); }) { + // Opaque ASTC goes to BC1 rather than BC3 when it is re-compressed (the Vulkan runtime + // decides). The guest data is already there when an image is first used. + constexpr size_t MaxScanBytes = 64_MiB; + if (IsPixelFormatASTC(new_info.format) && runtime.UseOpaqueAstcFormat()) { + const size_t scan_bytes = CalculateGuestSizeInBytes(new_info); + if (scan_bytes <= MaxScanBytes) { + astc_scan_buffer.resize(scan_bytes); + gpu_memory->ReadBlockUnsafe(gpu_addr, astc_scan_buffer.data(), scan_bytes); + new_info.astc_opaque = AstcDataLooksOpaque(astc_scan_buffer); + ++(new_info.astc_opaque ? astc_opaque_images : astc_alpha_images); + } + } + } const ImageId new_image_id = slot_images.insert(runtime, new_info, gpu_addr, cpu_addr); Image& new_image = slot_images[new_image_id]; new_image.allocation_tick = frame_tick; + { + // Which shapes are being created (periodic log): format, type, levels, layers, size. + const u64 key = (u64{static_cast<u32>(new_info.format)} << 56) | + (u64{static_cast<u32>(new_info.type)} << 52) | + (u64{static_cast<u32>(new_info.resources.levels) & 0xF} << 48) | + (u64{static_cast<u32>(new_info.resources.layers) & 0xFFF} << 36) | + (u64{new_info.size.width & 0x3FFFF} << 18) | (new_info.size.height & 0x3FFFF); + auto& shape = created_shapes[key]; + ++shape.first; + shape.second = gpu_addr; + ++images_created; + } if (!gpu_memory->IsContinuousRange(new_image.gpu_addr, new_image.guest_size_bytes) && new_info.is_sparse) { @@ -1833,12 +2092,9 @@ std::optional<typename TextureCache<P>::BlitImages> TextureCache<P>::GetBlitImag template <class P> ImageId TextureCache<P>::FindDMAImage(const ImageInfo& info, GPUVAddr gpu_addr) { - std::optional<DAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + const std::optional<DAddr> cpu_addr = ImageCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info)); if (!cpu_addr) { - cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info)); - if (!cpu_addr) { - return ImageId{}; - } + return ImageId{}; } ImageId image_id{}; boost::container::small_vector<ImageId, 8> image_ids; @@ -1859,6 +2115,13 @@ ImageId TextureCache<P>::FindDMAImage(const ImageInfo& info, GPUVAddr gpu_addr) return true; } } else if (IsSubCopy(info, existing_image, gpu_addr)) { + if (True(existing_image.flags & ImageFlagBits::Unbacked)) { + static u32 logged{}; + if (logged++ < 64) { + LOG_INFO(HW_GPU, "DMA copy at {:#x} ({}x{}) into unbacked image {:#x}", gpu_addr, + info.size.width, info.size.height, existing_image.gpu_addr); + } + } image_id = existing_image_id; image_ids.push_back(existing_image_id); return true; @@ -1954,6 +2217,13 @@ ImageViewId TextureCache<P>::FindRenderTargetView(const ImageInfo& info, GPUVAdd base = image.TryFindBase(gpu_addr).value(); } const s32 layers = image.info.type == ImageType::e3D ? info.size.depth : info.resources.layers; + if (True(image.flags & ImageFlagBits::Unbacked)) { + static u32 logged{}; + if (logged++ < 64) { + LOG_INFO(HW_GPU, "Render target in unbacked image {:#x} at {:#x}: {}x{} level {} layer {}", + image.gpu_addr, gpu_addr, info.size.width, info.size.height, base.level, base.layer); + } + } const SubresourceRange range{ .base = base, .extent = {.levels = 1, .layers = layers}, @@ -2136,6 +2406,13 @@ ImageViewId TextureCache<P>::FindOrEmplaceImageView(ImageId image_id, const Imag if (const ImageViewId image_view_id = image.FindView(info); image_view_id) { return image_view_id; } + if (True(image.flags & ImageFlagBits::Unbacked)) { + // How the game uses an image too large to back (e.g. a sparse virtual texture). + LOG_INFO(HW_GPU, "View of unbacked image {:#x}: type {} format {} levels {}+{} layers {}+{}", + image.gpu_addr, static_cast<u32>(info.type), static_cast<u32>(info.format), + info.range.base.level, info.range.extent.levels, info.range.base.layer, + info.range.extent.layers); + } const ImageViewId image_view_id = slot_image_views.insert(runtime, info, image_id, image, slot_images); image.InsertView(info, image_view_id); return image_view_id; @@ -2316,6 +2593,16 @@ void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) { template <class P> void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { ImageBase& image = slot_images[image_id]; + ++images_deleted; + // Drop every decode still queued for it, whatever IsDecoding says: an image re-decoded while + // a decode was in flight has two, and the first to finish clears the flag. A decode left + // behind was uploaded into the next image to take this slot (another texture's data: garbage + // textures, and a staging overrun when that image was smaller). + for (auto& async_decode : async_decodes) { + if (async_decode->image_id == image_id) { + async_decode->cancelled = true; + } + } if (image.HasScaled()) { total_used_memory -= GetScaledImageSizeBytes(image); } @@ -2385,7 +2672,11 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { } slot_image_views.erase(image_view_id); } - if (!immediate_delete) { + const bool retained = RetainPendingImageDownloads(image_id, slot_images[image_id], + uncommitted_downloads, committed_downloads); + if (retained) { + LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value()); + } else if (!immediate_delete) { sentenced_images.Push(std::move(slot_images[image_id])); } slot_images.erase(image_id); @@ -2537,6 +2828,11 @@ template <class P> void TextureCache<P>::CopyImage(ImageId dst_id, ImageId src_id, std::vector<ImageCopy> copies) { Image& dst = slot_images[dst_id]; Image& src = slot_images[src_id]; + if (dst.info.astc_opaque != src.info.astc_opaque) { + // BC1 and BC3 blocks differ in size, so the host can't copy between them; both hold the + // same guest data, which is uploaded from guest memory instead. + return; + } const bool is_rescaled = True(src.flags & ImageFlagBits::Rescaled); if (is_rescaled) { ASSERT(True(dst.flags & ImageFlagBits::Rescaled)); @@ -2628,7 +2924,8 @@ void TextureCache<P>::BindRenderTarget(ImageViewId* old_id, ImageViewId new_id) } if (new_id) { const ImageViewBase& old_view = slot_image_views[new_id]; - if (True(old_view.flags & ImageViewFlagBits::PreemtiveDownload)) { + if (True(old_view.flags & ImageViewFlagBits::PreemtiveDownload) && + False(slot_images[old_view.image_id].flags & (ImageFlagBits::Unbacked | ImageFlagBits::TrimmedLevels))) { const PendingDownload new_download{true, 0, old_view.image_id}; uncommitted_downloads.emplace_back(new_download); } diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 8a2318cc72..11ac8a2421 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -9,6 +9,7 @@ #include <atomic> #include <deque> #include <limits> +#include <map> #include <mutex> #include <span> #include <type_traits> @@ -22,12 +23,14 @@ #include <queue> #include "common/common_types.h" +#include "common/spin_recursive_mutex.h" #include "common/hash.h" #include "common/literals.h" #include "common/lru_cache.h" #include <ranges> #include "common/scratch_buffer.h" #include "common/slot_vector.h" +#include "video_core/texture_cache/pending_download.h" #include "common/thread_worker.h" #include "video_core/compatible_formats.h" #include "video_core/control/channel_state_cache.h" @@ -67,6 +70,7 @@ struct AsyncDecodeContext { boost::container::small_vector<BufferImageCopy, 16> copies; std::mutex mutex; std::atomic_bool complete; + bool cancelled{}; ///< The image was deleted meanwhile: drop the result (GPU thread only). }; using TextureCacheGPUMap = ::Common::unordered_map<u64, std::vector<ImageId>, Common::IdentityHash<u64>>; @@ -118,6 +122,21 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI static constexpr s64 TARGET_THRESHOLD = 4_GiB; #endif + // Periodic statistics (TickFrame). + u64 gc_deletes{}; + u64 gc_costly_deletes{}; + u64 async_decodes_queued{}; + u64 images_created{}; + bool native_thresholds_applied{}; + u64 native_critical_memory{}; + u64 native_expected_memory{}; + u64 budget_check_frames{}; + u64 astc_opaque_images{}; + u64 astc_alpha_images{}; + std::vector<u8> astc_scan_buffer; + u64 images_deleted{}; + std::map<u64, std::pair<u32, GPUVAddr>> created_shapes; + static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB; static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB; static constexpr size_t GC_EMERGENCY_COUNTS = 2; @@ -226,6 +245,9 @@ public: /// Return true when there are uncommitted images to be downloaded [[nodiscard]] bool HasUncommittedFlushes() const noexcept; + /// Whether CommitAsyncFlushes would commit anything (downloads or their buffers). + [[nodiscard]] bool HasAsyncFlushWork() const noexcept; + /// Return true when the caller should wait for async downloads [[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept; @@ -258,7 +280,7 @@ public: /// Prepare an image to be used void PrepareImage(ImageId image_id, bool is_modification, bool invalidate); - std::recursive_mutex mutex; + Common::SpinRecursiveMutex mutex; private: /// Iterate over all page indices in a range @@ -448,6 +470,12 @@ private: DAddr virtual_invalid_space{}; + /// Applies EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB and logs the thresholds. + void ApplyThresholdOverrides(bool log = true); + + /// The CPU address an image at gpu_addr is keyed by: its backing, or a fixed stand-in. + [[nodiscard]] std::optional<DAddr> ImageCpuAddress(GPUVAddr gpu_addr, size_t size) const; + bool has_deleted_images = false; bool is_rescaling = false; u64 total_used_memory = 0; @@ -463,11 +491,7 @@ private: size_t size; }; - struct PendingDownload { - bool is_swizzle; - size_t async_buffer_id; - Common::SlotId object_id; - }; + using PendingDownload = PendingImageDownload<Image>; Common::SlotVector<Image> slot_images; Common::SlotVector<ImageMapView> slot_map_views; diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp index e55d0752ec..ffc1d59ec3 100644 --- a/src/video_core/texture_cache/util.cpp +++ b/src/video_core/texture_cache/util.cpp @@ -11,6 +11,7 @@ #include <algorithm> #include <array> +#include <cstring> #include <numeric> #include <optional> #include <span> @@ -439,7 +440,7 @@ template <u32 GOB_EXTENT> } layers = 1; } else { - layers = (std::max)(resources.layers, info.resources.layers + base->layer); + layers = (std::max)(info.resources.layers, resources.layers + base->layer); } return OverlapResult{ .gpu_addr = overlap.gpu_addr, @@ -566,7 +567,7 @@ void SwizzleBlockLinearImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr dst(gpu_memory, gpu_addr + guest_offset, subresource_size, &tmp_buffer); SwizzleTexture(dst, src, bytes_per_block, num_tiles.width, num_tiles.height, - num_tiles.depth, block.height, block.depth); + num_tiles.depth, block.height, block.depth, 1, gpu_memory.UsesNativeTiling()); } host_offset += host_bytes_per_layer; @@ -603,6 +604,10 @@ u32 CalculateUnswizzledSizeBytes(const ImageInfo& info) noexcept { return NumBlocksPerLayer(info, tile_size) * info.resources.layers * BytesPerBlock(info.format); } +bool AstcDataLooksOpaque(std::span<const u8> data) noexcept { + return Tegra::Texture::ASTC::IsOpaque(data); +} + u32 CalculateConvertedSizeBytes(const ImageInfo& info) noexcept { if (info.type == ImageType::Buffer) { return info.size.width * BytesPerBlock(info.format); @@ -611,8 +616,10 @@ u32 CalculateConvertedSizeBytes(const ImageInfo& info) noexcept { if (IsPixelFormatASTC(info.format) && Settings::values.astc_recompression.GetValue() != Settings::AstcRecompression::Uncompressed) { const u32 bpp_div = - Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Bc1 ? 2 - : 1; + Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Bc1 || + info.astc_opaque + ? 2 + : 1; // NumBlocksPerLayer doesn't account for this correctly, so we have to do it manually. u32 output_size = 0; for (s32 i = 0; i < info.resources.levels; i++) { @@ -913,7 +920,7 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory const std::span<u8> dst = output.subspan(host_offset); const std::span<const u8> src = input.subspan(guest_offset + guest_layer_offset); UnswizzleTexture(dst, src, 1U << bpp_log2, num_tiles.width, num_tiles.height, - num_tiles.depth, block.height, block.depth, stride_alignment); + num_tiles.depth, block.height, block.depth, stride_alignment, gpu_memory.UsesNativeTiling()); guest_layer_offset += layer_stride; host_offset += host_bytes_per_layer; } @@ -922,6 +929,35 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory return copies; } +void ConvertedLayout(const ImageInfo& info, std::span<BufferImageCopy> copies) { + // Same offsets and sizes as ConvertImage below. + const auto recompression = Settings::values.astc_recompression.GetValue(); + const bool astc = IsPixelFormatASTC(info.format); + u64 output_offset = 0; + for (BufferImageCopy& copy : copies) { + const Extent3D mip_size = AdjustMipSize(info.size, copy.image_subresource.base_level); + const u64 layers = static_cast<u64>(copy.image_subresource.num_layers) * copy.image_extent.depth; + u64 size; + if (astc && recompression == Settings::AstcRecompression::Uncompressed) { + size = u64{copy.image_extent.width} * copy.image_extent.height * layers * + BytesPerBlock(PixelFormat::A8B8G8R8_UNORM); + } else if (astc) { + const u64 bpp_div = + recompression == Settings::AstcRecompression::Bc1 || info.astc_opaque ? 2 : 1; + size = u64{Common::AlignUp(copy.image_extent.width, 4U)} * + Common::AlignUp(copy.image_extent.height, 4U) * layers / bpp_div; + } else { + size = u64{copy.image_extent.width} * copy.image_extent.height * layers * + ConvertedBytesPerBlock(info.format); + } + copy.buffer_offset = output_offset; + copy.buffer_size = size; + copy.buffer_row_length = mip_size.width; + copy.buffer_image_height = mip_size.height; + output_offset += size; + } +} + void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8> output, std::span<BufferImageCopy> copies) { u32 output_offset = 0; @@ -955,10 +991,11 @@ void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8 } else if (astc) { // BC1 uses 0.5 bytes per texel // BC3 uses 1 byte per texel - const auto compress = recompression_setting == Settings::AstcRecompression::Bc1 - ? Tegra::Texture::BCN::CompressBC1 - : Tegra::Texture::BCN::CompressBC3; - const auto bpp_div = recompression_setting == Settings::AstcRecompression::Bc1 ? 2 : 1; + const bool bc1 = + recompression_setting == Settings::AstcRecompression::Bc1 || info.astc_opaque; + const auto compress = + bc1 ? Tegra::Texture::BCN::CompressBC1 : Tegra::Texture::BCN::CompressBC3; + const auto bpp_div = bc1 ? 2 : 1; const u32 plane_dim = copy.image_extent.width * copy.image_extent.height; const u32 level_size = plane_dim * copy.image_extent.depth * @@ -1298,6 +1335,13 @@ void DeduceBlitImages(ImageInfo& dst_info, ImageInfo& src_info, const ImageBase* } u32 MapSizeBytes(const ImageBase& image) { + if (True(image.flags & ImageFlagBits::AcceleratedUpload) && image.host_level_offset != 0) { + // Each layer's mip chain from host_level_offset on (see UploadImageContents). + const auto uploads = FullUploadSwizzles(image.info); + const u32 tail = image.info.layer_stride - + static_cast<u32>(uploads[image.host_level_offset].buffer_offset); + return tail * image.info.resources.layers; + } if (True(image.flags & ImageFlagBits::AcceleratedUpload)) { return image.guest_size_bytes; } else if (True(image.flags & ImageFlagBits::Converted)) { diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h index 590d85b042..0381d2d24b 100644 --- a/src/video_core/texture_cache/util.h +++ b/src/video_core/texture_cache/util.h @@ -36,6 +36,9 @@ struct OverlapResult { [[nodiscard]] u32 CalculateConvertedSizeBytes(const ImageInfo& info) noexcept; +/// True if no 16-byte ASTC block in `data` decodes to alpha below 1. +[[nodiscard]] bool AstcDataLooksOpaque(std::span<const u8> data) noexcept; + [[nodiscard]] u32 CalculateLayerStride(const ImageInfo& info) noexcept; [[nodiscard]] u32 CalculateLayerSize(const ImageInfo& info) noexcept; @@ -69,6 +72,9 @@ struct OverlapResult { Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info, std::span<const u8> input, std::span<u8> output); +/// Rewrites `copies` (from UnswizzleImage) to the layout ConvertImage produces, without converting. +void ConvertedLayout(const ImageInfo& info, std::span<BufferImageCopy> copies); + void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8> output, std::span<BufferImageCopy> copies); diff --git a/src/video_core/textures/astc.cpp b/src/video_core/textures/astc.cpp index 0b6d06c49a..950ce28d6b 100644 --- a/src/video_core/textures/astc.cpp +++ b/src/video_core/textures/astc.cpp @@ -2038,4 +2038,69 @@ void Decompress(std::span<const uint8_t> data, uint32_t width, uint32_t height, } } +bool IsOpaque(std::span<const uint8_t> data) { + // Endpoint modes with an alpha channel: LDR luminance+alpha (4, 5), RGB scale + two alpha + // (10), LDR RGBA (12, 13) and HDR RGBA (14, 15). + constexpr u32 AlphaModes = (1U << 4) | (1U << 5) | (1U << 10) | (1U << 12) | (1U << 13) | + (1U << 14) | (1U << 15); + for (size_t offset = 0; offset + 16 <= data.size(); offset += 16) { + const std::span<const u8> block = data.subspan(offset, 16); + InputBitStream strm(block); + const TexelWeightParams params = DecodeBlockInfo(strm); + if (params.m_bError) { + continue; // decodes to the opaque error colour + } + if (params.m_bVoidExtentLDR || params.m_bVoidExtentHDR) { + // One RGBA16 colour in bits 64-127: UNORM16, or FP16 for HDR. + const u32 alpha = block[14] | (u32{block[15]} << 8); + if (alpha != (params.m_bVoidExtentHDR ? 0x3C00U : 0xFFFFU)) { + return false; + } + continue; + } + const u32 partitions = strm.ReadBits<2>() + 1; + if (partitions == 4 && params.m_bDualPlane) { + continue; + } + if (partitions == 1) { + if ((AlphaModes >> strm.ReadBits<4>()) & 1) { + return false; + } + continue; + } + strm.ReadBits<10>(); // partition index + const u32 base_cem = strm.ReadBits<6>(); + const u32 base_mode = base_cem & 3; + if (base_mode == 0) { + if ((AlphaModes >> (base_cem >> 2)) & 1) { + return false; + } + continue; + } + // The rest of the per-partition modes sit just below the weights (DecompressBlock). + const u32 weight_bits = params.GetPackedBitSize(); + if (weight_bits < 24 || weight_bits > 96) { + continue; + } + const u32 extra_bits = partitions == 2 ? 2 : partitions == 3 ? 5 : 8; + const u32 extra_offset = 128 - weight_bits - extra_bits; + u32 extra = 0; + for (u32 bit = 0; bit < extra_bits; ++bit) { + const u32 position = extra_offset + bit; + extra |= ((block[position / 8] >> (position % 8)) & 1U) << bit; + } + u32 cem = ((extra << 6) | base_cem) >> 2; + const u32 classes = cem; + cem >>= partitions; + for (u32 i = 0; i < partitions; ++i) { + const u32 mode = ((base_mode - (((classes >> i) & 1) != 0 ? 0 : 1)) << 2) | + ((cem >> (2 * i)) & 3); + if ((AlphaModes >> mode) & 1) { + return false; + } + } + } + return true; +} + } // namespace Tegra::Texture::ASTC diff --git a/src/video_core/textures/astc.h b/src/video_core/textures/astc.h index afd3933c3e..95c2c7ffea 100644 --- a/src/video_core/textures/astc.h +++ b/src/video_core/textures/astc.h @@ -8,4 +8,7 @@ namespace Tegra::Texture::ASTC { void Decompress(std::span<const uint8_t> data, uint32_t width, uint32_t height, uint32_t depth, uint32_t block_width, uint32_t block_height, std::span<uint8_t> output); +/// True if no 16-byte block in `data` decodes to alpha below 1. +bool IsOpaque(std::span<const uint8_t> data); + } // namespace Tegra::Texture::ASTC diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index 25b0860b0f..ef50be156b 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -18,6 +18,11 @@ namespace Tegra::Texture { namespace { +constexpr u32 NativeGobOffset(u32 offset, bool native_tiling) { + if (!native_tiling) return offset; + const u32 difference = ((offset >> 5) ^ (offset >> 6)) & 1; + return offset ^ (difference << 5) ^ (difference << 6); +} template <u32 mask> constexpr u32 pdep(u32 value) { u32 result = 0; @@ -38,7 +43,7 @@ void incrpdep(u32& value) { template <bool TO_LINEAR, u32 BYTES_PER_PIXEL> void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 height, u32 depth, - u32 block_height, u32 block_depth, u32 stride) { + u32 block_height, u32 block_depth, u32 stride, bool native_tiling) { // The origin of the transformation can be configured here, leave it as zero as the current API // doesn't expose it. static constexpr u32 origin_x = 0; @@ -77,7 +82,7 @@ void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift; const u32 base_swizzled_offset = offset_z + offset_y + offset_x; - const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y); + const u32 swizzled_offset = NativeGobOffset(base_swizzled_offset + (swizzled_x | swizzled_y), native_tiling); const u32 unswizzled_offset = slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL; @@ -94,7 +99,7 @@ void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 template <bool TO_LINEAR, u32 BYTES_PER_PIXEL> void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 num_lines, - u32 block_height, u32 block_depth, u32 pitch_linear) { + u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) { // The origin of the transformation can be configured here, leave it as zero as the current API // doesn't expose it. static constexpr u32 origin_z = 0; @@ -136,7 +141,7 @@ void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 wid const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift; const u32 base_swizzled_offset = offset_z + offset_y + offset_x; - const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y); + const u32 swizzled_offset = NativeGobOffset(base_swizzled_offset + (swizzled_x | swizzled_y), native_tiling); const u32 unswizzled_offset = slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL; @@ -156,12 +161,12 @@ void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 wid template <bool TO_LINEAR> void Swizzle(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, - u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment) { + u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment, bool native_tiling) { switch (bytes_per_pixel) { #define BPP_CASE(x) \ case x: \ return SwizzleImpl<TO_LINEAR, x>(output, input, width, height, depth, block_height, \ - block_depth, stride_alignment); + block_depth, stride_alignment, native_tiling); BPP_CASE(1) BPP_CASE(2) BPP_CASE(3) @@ -181,35 +186,35 @@ void Swizzle(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixe void UnswizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment) { + u32 stride_alignment, bool native_tiling) { const u32 stride = Common::AlignUpLog2(width, stride_alignment) * bytes_per_pixel; const u32 new_bpp = (std::min)(4U, static_cast<u32>(std::countr_zero(width * bytes_per_pixel))); width = (width * bytes_per_pixel) >> new_bpp; bytes_per_pixel = 1U << new_bpp; Swizzle<false>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth, - stride); + stride, native_tiling); } void SwizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment) { + u32 stride_alignment, bool native_tiling) { const u32 stride = Common::AlignUpLog2(width, stride_alignment) * bytes_per_pixel; const u32 new_bpp = (std::min)(4U, static_cast<u32>(std::countr_zero(width * bytes_per_pixel))); width = (width * bytes_per_pixel) >> new_bpp; bytes_per_pixel = 1U << new_bpp; Swizzle<true>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth, - stride); + stride, native_tiling); } void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 extent_y, - u32 block_height, u32 block_depth, u32 pitch_linear) { + u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) { switch (bytes_per_pixel) { #define BPP_CASE(x) \ case x: \ return SwizzleSubrectImpl<true, x>(output, input, width, height, depth, origin_x, \ origin_y, extent_x, extent_y, block_height, \ - block_depth, pitch_linear); + block_depth, pitch_linear, native_tiling); BPP_CASE(1) BPP_CASE(2) BPP_CASE(3) @@ -227,13 +232,13 @@ void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_p void UnswizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, - u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear) { + u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) { switch (bytes_per_pixel) { #define BPP_CASE(x) \ case x: \ return SwizzleSubrectImpl<false, x>(output, input, width, height, depth, origin_x, \ origin_y, extent_x, extent_y, block_height, \ - block_depth, pitch_linear); + block_depth, pitch_linear, native_tiling); BPP_CASE(1) BPP_CASE(2) BPP_CASE(3) diff --git a/src/video_core/textures/decoders.h b/src/video_core/textures/decoders.h index 1e14a38753..6151967869 100644 --- a/src/video_core/textures/decoders.h +++ b/src/video_core/textures/decoders.h @@ -29,12 +29,12 @@ constexpr u32 SWIZZLE_Y_BITS = 0b011010000; /// Unswizzles a block linear texture into linear memory. void UnswizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment = 1); + u32 stride_alignment = 1, bool native_tiling = false); /// Swizzles linear memory into a block linear texture. void SwizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment = 1); + u32 stride_alignment = 1, bool native_tiling = false); /// This function calculates the correct size of a texture depending if it's tiled or not. std::size_t CalculateSize(bool tiled, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, @@ -43,12 +43,12 @@ std::size_t CalculateSize(bool tiled, u32 bytes_per_pixel, u32 width, u32 height /// Copies an untiled subrectangle into a tiled surface. void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 extent_y, - u32 block_height, u32 block_depth, u32 pitch_linear); + u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling = false); /// Copies a tiled subrectangle into a linear surface. void UnswizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, - u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear); + u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling = false); /// Obtains the offset of the gob for positions 'dst_x' & 'dst_y' u64 GetGOBOffset(u32 width, u32 height, u32 dst_x, u32 dst_y, u32 block_height, diff --git a/src/video_core/textures/workers.cpp b/src/video_core/textures/workers.cpp index a1953c4bf4..e3a0f26cf2 100644 --- a/src/video_core/textures/workers.cpp +++ b/src/video_core/textures/workers.cpp @@ -4,13 +4,23 @@ // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> + #include "video_core/textures/workers.h" namespace Tegra::Texture { Common::ThreadWorker& GetThreadWorkers() { - static Common::ThreadWorker workers{(std::max)(std::thread::hardware_concurrency(), 2U) / 2, - "ImageTranscode"}; + // A quarter of the hardware threads (at least 2). Half took ~3.5 cores from the emulated + // CPU cores and the GPU thread on an 8-core laptop when textures were thrashing; with an + // eighth (2) both threads stayed saturated and streamed textures arrived seconds late (shown + // as placeholders), while 4 cost no frame rate. EDEN_NX2_TRANSCODE_THREADS=<n> overrides. + static Common::ThreadWorker workers{[] { + if (const char* text = std::getenv("EDEN_NX2_TRANSCODE_THREADS")) { + return (std::max)(static_cast<u32>(std::strtoul(text, nullptr, 10)), 1U); + } + return (std::max)(std::thread::hardware_concurrency() / 4, 2U); + }(), "ImageTranscode"}; return workers; } diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index be09bca8d3..69c3ba14fa 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <array> #include <bitset> #include <chrono> #include <filesystem> @@ -752,9 +753,10 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR .flags = flags, .physicalDevice = physical, .device = *logical, - .preferredLargeHeapBlockSize = is_integrated - ? (64u * 1024u * 1024u) - : (256u * 1024u * 1024u), + // 64 MiB blocks on discrete GPUs too: with 256 MiB ones, Switch 2 titles' mix of + // texture sizes left ~38% of the image blocks empty (2 GB of an 8 GB card in title + // 1), which the texture cache counted as used and kept evicting textures for. + .preferredLargeHeapBlockSize = 64u * 1024u * 1024u, .pAllocationCallbacks = nullptr, .pDeviceMemoryCallbacks = nullptr, .pHeapSizeLimit = nullptr, @@ -1609,6 +1611,33 @@ u64 Device::GetDeviceMemoryUsage() const { for (const size_t heap : valid_heap_memory) { result += budget.heapUsage[heap]; } + // Space in VMA's blocks that no allocation uses counts as used, but freeing an image gives + // its share back only once its whole block is empty: the caches, collecting until this drops, + // evicted thousands of textures in a row without lowering it (title 1). Count only what + // allocations use. + if (allocator != nullptr) { + std::array<VmaBudget, VK_MAX_MEMORY_HEAPS> heap_budgets{}; + vmaGetHeapBudgets(allocator, heap_budgets.data()); + for (const size_t heap : valid_heap_memory) { + const VmaStatistics& statistics = heap_budgets[heap].statistics; + const u64 unused = statistics.blockBytes - statistics.allocationBytes; + result -= (std::min)(result, unused); + } + } + return result; +} + +u64 Device::GetDeviceMemoryBudget() const { + if (!extensions.memory_budget) { + return 0; + } + VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{}; + budget.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT; + physical.GetMemoryProperties(&budget); + u64 result{}; + for (const size_t heap : valid_heap_memory) { + result += budget.heapBudget[heap]; + } return result; } diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index bbf3754e39..b1fa8db183 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -963,6 +963,9 @@ FN_MAX_LIMIT_LIST u64 GetDeviceMemoryUsage() const; + /// The driver's current memory budget for this process (VK_EXT_memory_budget), 0 without it. + u64 GetDeviceMemoryBudget() const; + u32 GetSetsPerPool() const { return sets_per_pool; } @@ -1145,7 +1148,7 @@ private: private: VkInstance instance; ///< Vulkan instance. - VmaAllocator allocator; ///< VMA allocator. + VmaAllocator allocator{}; ///< VMA allocator. vk::DeviceDispatch dld; ///< Device function pointers. vk::PhysicalDevice physical; ///< Physical device. vk::Device logical; ///< Logical device. diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index f4721543ae..27f8fa6b4c 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <array> #include <bit> #include <limits> #include <optional> @@ -205,10 +206,12 @@ MemoryAllocator::MemoryAllocator(const Device &device_) MemoryAllocator::~MemoryAllocator() = default; -vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const +vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicated) const { const VmaAllocationCreateInfo alloc_ci = { - .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT, + .flags = static_cast<VmaAllocationCreateFlags>(VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) | + (dedicated ? VmaAllocationCreateFlags{VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT} + : VmaAllocationCreateFlags{0}), .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, .requiredFlags = 0, .preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, @@ -221,7 +224,53 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const VkImage handle{}; VmaAllocation allocation{}; VmaAllocationInfo alloc_info{}; - vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + // Images this large (title 1's sparse texture arrays reach 4.5G texels, ~23 GB) are never + // backed: VMA would quietly place them in system memory once VRAM is full. They take the + // unbacked sparse fallback below. + const u64 texels = u64{ci.extent.width} * ci.extent.height * ci.extent.depth * ci.arrayLayers; + const bool huge = texels > (u64{1} << 28); + VkResult result = huge ? VK_ERROR_OUT_OF_DEVICE_MEMORY + : vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && !huge) { + // Over the device memory budget (Switch 2 titles fill 8 GB cards): let the driver page + // it out rather than failing the draw and aborting. + static u32 logged{}; + if (logged++ < 32) { + LOG_WARNING(Render_Vulkan, + "Image {}x{}x{} ({} levels, {} layers, format {}) is over the memory " + "budget; allocating it anyway", + ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, + ci.arrayLayers, static_cast<u32>(ci.format)); + } + VmaAllocationCreateInfo relaxed = alloc_ci; + relaxed.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + result = vmaCreateImage(allocator, &ci, &relaxed, &handle, &allocation, &alloc_info); + } + if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && ci.imageType != VK_IMAGE_TYPE_1D) { + // Too big to back at all, e.g. title 1's sparse virtual texture arrays (8192x8192, + // 68 layers, ~23 GB): on the console only resident tiles have memory. Create a sparse + // resident image with nothing bound, so views and copies still work; reads of it return + // zero and writes are dropped, instead of aborting. + VkImageCreateInfo sparse_ci = ci; + sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT; + VkImage sparse{}; + if (device.GetDispatchLoader().vkCreateImage(*device.GetLogical(), &sparse_ci, nullptr, + &sparse) == VK_SUCCESS) { + LOG_WARNING(Render_Vulkan, + "Image {}x{}x{} ({} levels, {} layers, format {}) does not fit; using an " + "unbacked sparse image", + ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, + ci.arrayLayers, static_cast<u32>(ci.format)); + return vk::Image(sparse, ci.usage, *device.GetLogical(), allocator, VK_NULL_HANDLE, + device.GetDispatchLoader()); + } + } + if (result != VK_SUCCESS) { + LOG_CRITICAL(Render_Vulkan, "Image {}x{}x{} ({} levels, {} layers, format {}) failed: {}", + ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, + ci.arrayLayers, static_cast<u32>(ci.format), static_cast<int>(result)); + } + vk::Check(result); // Log GPU memory allocation for images if (GPU::Logging::IsActive() && @@ -405,4 +454,29 @@ MemoryCommit MemoryAllocator::Commit(const vk::Buffer &buffer, MemoryUsage usage return MemoryCommit(allocator, a, info); } +void LogMemoryStatistics(const Device& device) { + const VmaAllocator allocator = device.GetAllocator(); + const VkPhysicalDeviceMemoryProperties* properties{}; + vmaGetMemoryProperties(allocator, &properties); + VmaTotalStatistics stats{}; + vmaCalculateStatistics(allocator, &stats); + for (u32 type = 0; type < properties->memoryTypeCount; ++type) { + const VmaStatistics& s = stats.memoryType[type].statistics; + if (s.blockCount == 0) { + continue; + } + LOG_INFO(Render_Vulkan, "VMA type {} (heap {}, flags {:#x}): {} blocks {} MiB, {} allocations {} MiB", + type, properties->memoryTypes[type].heapIndex, + static_cast<u32>(properties->memoryTypes[type].propertyFlags), s.blockCount, + s.blockBytes >> 20, s.allocationCount, s.allocationBytes >> 20); + } + std::array<VmaBudget, VK_MAX_MEMORY_HEAPS> budgets{}; + vmaGetHeapBudgets(allocator, budgets.data()); + for (u32 heap = 0; heap < properties->memoryHeapCount; ++heap) { + LOG_INFO(Render_Vulkan, "VMA heap {}: usage {} MiB of budget {} MiB, VMA blocks {} MiB", + heap, budgets[heap].usage >> 20, budgets[heap].budget >> 20, + budgets[heap].statistics.blockBytes >> 20); + } +} + } // namespace Vulkan diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index a0f9081fc0..fa7d4b41c3 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -103,7 +103,9 @@ namespace Vulkan { MemoryAllocator(const MemoryAllocator &) = delete; - vk::Image CreateImage(const VkImageCreateInfo &ci) const; + /// `dedicated`: own device memory, returned to the driver on destruction (short-lived + /// images would otherwise leave holes in the shared blocks). + vk::Image CreateImage(const VkImageCreateInfo &ci, bool dedicated = false) const; vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const; @@ -142,4 +144,7 @@ namespace Vulkan { u32 valid_memory_types{~0u}; }; + /// Logs VMA's blocks and allocations per memory type, and each heap's budget. + void LogMemoryStatistics(const Device& device); + } // namespace Vulkan diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index 97d2ba3700..947150648d 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -689,6 +689,11 @@ public: allocation{allocation_}, dld{&dld_} {} Image() = default; + /// True for an image created without memory (an unbacked sparse image). + bool IsUnbacked() const noexcept { + return handle != VK_NULL_HANDLE && allocation == VK_NULL_HANDLE; + } + Image(const Image&) = delete; Image& operator=(const Image&) = delete; diff --git a/src/yuzu_cmd/CMakeLists.txt b/src/yuzu_cmd/CMakeLists.txt index 17a1545167..a87b4e5288 100644 --- a/src/yuzu_cmd/CMakeLists.txt +++ b/src/yuzu_cmd/CMakeLists.txt @@ -31,6 +31,8 @@ add_executable(yuzu-cmd emu_window/emu_window_sdl3_null.h emu_window/emu_window_sdl3_vk.cpp emu_window/emu_window_sdl3_vk.h + nx2_sampler.cpp + nx2_sampler.h sdl_config.cpp sdl_config.h yuzu.cpp diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp index 950e001f81..174f8b5328 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp +++ b/src/yuzu_cmd/emu_window/emu_window_sdl3.cpp @@ -4,6 +4,10 @@ // SPDX-FileCopyrightText: 2016 Citra Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> +#include <cstdio> +#include <chrono> +#include <array> #include <SDL3/SDL.h> #include <SDL3/SDL_timer.h> #ifdef __EMSCRIPTEN__ @@ -39,7 +43,10 @@ EmuWindow_SDL3::EmuWindow_SDL3(InputCommon::InputSubsystem* input_subsystem_, Co }, this); mouse_timer = SDL_AddTimer(100, [](void *userdata, SDL_TimerID, Uint32) -> Uint32 { auto* this_ = (EmuWindow_SDL3*)userdata; - this_->input_subsystem->GetMouse()->NotifyChanged(); + // The timer can fire before the input subsystem has created its mouse driver. + if (auto* const mouse = this_->input_subsystem->GetMouse()) { + mouse->NotifyChanged(); + } return 100; }, this); } @@ -115,7 +122,26 @@ void EmuWindow_SDL3::OnFingerUp() { input_subsystem->GetTouchScreen()->ReleaseAllTouch(); } +// Seconds since the first frame; shared by the key script (yuzu.cpp) and key recording. +double Nx2KeyClockSeconds() { + static const auto begin = std::chrono::steady_clock::now(); + return std::chrono::duration<double>(std::chrono::steady_clock::now() - begin).count(); +} + void EmuWindow_SDL3::OnKeyEvent(int key, u8 state) { + // EDEN_NX2_KEY_RECORD=<file>: append each press/release ("seconds:code", "seconds:-code"). + static std::FILE* const record = [] { + const char* path = std::getenv("EDEN_NX2_KEY_RECORD"); + return path ? std::fopen(path, "w") : nullptr; + }(); + static std::array<bool, 512> held{}; + if (record && key >= 0 && key < static_cast<int>(held.size()) && held[key] != (state != 0)) { + held[key] = state != 0; + std::fprintf(record, "f%llu:%s%d,", + static_cast<unsigned long long>(Core::g_nx2_present_count.load()), + state != 0 ? "" : "-", key); + std::fflush(record); + } if (state != 0) { input_subsystem->GetKeyboard()->PressKey(static_cast<std::size_t>(key)); } else { @@ -201,6 +227,7 @@ void EmuWindow_SDL3::OnEvent(SDL_Event& event) { is_shown = true; return OnResize(); case SDL_EVENT_WINDOW_CLOSE_REQUESTED: + LOG_INFO(Frontend, "Window close requested"); is_open = false; return; case SDL_EVENT_KEY_DOWN: @@ -224,6 +251,7 @@ void EmuWindow_SDL3::OnEvent(SDL_Event& event) { case SDL_EVENT_FINGER_UP: return OnFingerUp(); case SDL_EVENT_QUIT: + LOG_INFO(Frontend, "Quit requested"); is_open = false; return; default: diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3.h b/src/yuzu_cmd/emu_window/emu_window_sdl3.h index ffabaeb143..0dc2a4983a 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl3.h +++ b/src/yuzu_cmd/emu_window/emu_window_sdl3.h @@ -6,6 +6,7 @@ #pragma once +#include <cstdlib> #include <tuple> #include <utility> @@ -24,6 +25,14 @@ class InputSubsystem; enum class MouseButton; } // namespace InputCommon +// EDEN_NX2_HIDE_WINDOW=1 (anything but 0) creates the render window hidden and without focus, for unattended +// test runs that must not cover or interrupt the desktop. +inline SDL_WindowFlags Nx2HiddenWindowFlags() { + const char* hide = std::getenv("EDEN_NX2_HIDE_WINDOW"); + return hide != nullptr && hide[0] != '0' ? SDL_WINDOW_HIDDEN | SDL_WINDOW_NOT_FOCUSABLE + : SDL_WindowFlags{0}; +} + class EmuWindow_SDL3 : public Core::Frontend::EmuWindow { public: explicit EmuWindow_SDL3(InputCommon::InputSubsystem* input_subsystem_, Core::System& system_); diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp index e86858a232..38cc340426 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp +++ b/src/yuzu_cmd/emu_window/emu_window_sdl3_gl.cpp @@ -101,7 +101,7 @@ EmuWindow_SDL3_GL::EmuWindow_SDL3_GL(InputCommon::InputSubsystem* input_subsyste SDL_CreateWindow(window_title.c_str(), Layout::ScreenUndocked::Width, Layout::ScreenUndocked::Height, SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE | - SDL_WINDOW_HIGH_PIXEL_DENSITY); + SDL_WINDOW_HIGH_PIXEL_DENSITY | Nx2HiddenWindowFlags()); if (render_window == nullptr) { LOG_CRITICAL(Frontend, "Failed to create SDL3 window! {}", SDL_GetError()); diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp index f5e4e19a30..45ff657f13 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp +++ b/src/yuzu_cmd/emu_window/emu_window_sdl3_null.cpp @@ -31,7 +31,7 @@ EmuWindow_SDL3_Null::EmuWindow_SDL3_Null(InputCommon::InputSubsystem* input_subs render_window = SDL_CreateWindow(window_title.c_str(), Layout::ScreenUndocked::Width, Layout::ScreenUndocked::Height, - SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY); + SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY | Nx2HiddenWindowFlags()); SetWindowIcon(); diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp index 9019cbbba0..afbe67716f 100644 --- a/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp +++ b/src/yuzu_cmd/emu_window/emu_window_sdl3_vk.cpp @@ -29,7 +29,7 @@ EmuWindow_SDL3_VK::EmuWindow_SDL3_VK(InputCommon::InputSubsystem* input_subsyste render_window = SDL_CreateWindow(window_title.c_str(), Layout::ScreenUndocked::Width, Layout::ScreenUndocked::Height, - SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY); + SDL_WINDOW_RESIZABLE | SDL_WINDOW_HIGH_PIXEL_DENSITY | Nx2HiddenWindowFlags()); const SDL_PropertiesID window_props = SDL_GetWindowProperties(render_window); diff --git a/src/yuzu_cmd/nx2_sampler.cpp b/src/yuzu_cmd/nx2_sampler.cpp new file mode 100644 index 0000000000..c01b812650 --- /dev/null +++ b/src/yuzu_cmd/nx2_sampler.cpp @@ -0,0 +1,333 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "yuzu_cmd/nx2_sampler.h" + +#ifdef _WIN32 + +#include <algorithm> +#include <array> +#include <chrono> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <map> +#include <set> +#include <string> +#include <thread> +#include <unordered_map> +#include <vector> + +#include <windows.h> +#include <tlhelp32.h> +#include <dbghelp.h> + +namespace Nx2Sampler { +namespace { + +constexpr size_t MaxDepth = 40; + +struct Sample { + unsigned thread; + unsigned depth; + std::array<DWORD64, MaxDepth> frames; +}; + +struct Target { + HANDLE handle; + std::string name; +}; + +double EnvSeconds(const char* name, double fallback) { + const char* text = std::getenv(name); + return text != nullptr ? std::strtod(text, nullptr) : fallback; +} + +std::vector<std::string> SplitNames(const char* text) { + std::vector<std::string> names; + std::string current; + for (const char* c = text; *c != '\0'; ++c) { + if (*c == ',') { + names.push_back(current); + current.clear(); + } else { + current += *c; + } + } + if (!current.empty()) { + names.push_back(current); + } + return names; +} + +bool Matches(const std::string& thread_name, const std::vector<std::string>& patterns) { + for (const auto& pattern : patterns) { + if (!pattern.empty() && pattern.back() == '*') { + if (thread_name.compare(0, pattern.size() - 1, pattern, 0, pattern.size() - 1) == 0) { + return true; + } + } else if (thread_name == pattern) { + return true; + } + } + return false; +} + +std::vector<Target> FindThreads(const std::vector<std::string>& patterns) { + using GetThreadDescriptionFn = HRESULT(WINAPI*)(HANDLE, PWSTR*); + const auto get_description = reinterpret_cast<GetThreadDescriptionFn>( + GetProcAddress(GetModuleHandleA("KernelBase.dll"), "GetThreadDescription")); + std::vector<Target> targets; + if (get_description == nullptr) { + return targets; + } + const HANDLE snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); + if (snapshot == INVALID_HANDLE_VALUE) { + return targets; + } + THREADENTRY32 entry{}; + entry.dwSize = sizeof(entry); + const DWORD self = GetCurrentThreadId(); + for (BOOL ok = Thread32First(snapshot, &entry); ok; ok = Thread32Next(snapshot, &entry)) { + if (entry.th32OwnerProcessID != GetCurrentProcessId() || entry.th32ThreadID == self) { + continue; + } + const HANDLE handle = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT | + THREAD_QUERY_LIMITED_INFORMATION, + FALSE, entry.th32ThreadID); + if (handle == nullptr) { + continue; + } + PWSTR wide{}; + std::string name; + if (SUCCEEDED(get_description(handle, &wide)) && wide != nullptr) { + char buffer[256]{}; + WideCharToMultiByte(CP_UTF8, 0, wide, -1, buffer, sizeof(buffer) - 1, nullptr, + nullptr); + name = buffer; + LocalFree(wide); + } + if (Matches(name, patterns)) { + targets.push_back({handle, name}); + } else { + CloseHandle(handle); + } + } + CloseHandle(snapshot); + return targets; +} + +// Walks a suspended thread's stack from its context. Nothing here may allocate: the thread +// could be paused holding the heap lock. +unsigned Walk(CONTEXT context, std::array<DWORD64, MaxDepth>& frames) { + unsigned depth = 0; + while (depth < MaxDepth && context.Rip != 0) { + frames[depth++] = context.Rip; + DWORD64 image_base{}; + const PRUNTIME_FUNCTION function = RtlLookupFunctionEntry(context.Rip, &image_base, nullptr); + if (function == nullptr) { + // A leaf function (no unwind data): its return address is on top of the stack. JIT + // code has none either; past the first frame, stop there. + if (depth != 1) { + break; + } + context.Rip = *reinterpret_cast<const DWORD64*>(context.Rsp); + context.Rsp += 8; + continue; + } + PVOID handler_data{}; + DWORD64 establisher_frame{}; + RtlVirtualUnwind(UNW_FLAG_NHANDLER, image_base, context.Rip, function, &context, + &handler_data, &establisher_frame, nullptr); + } + return depth; +} + +struct Symbolizer { + using SymInitializeFn = BOOL(WINAPI*)(HANDLE, PCSTR, BOOL); + using SymSetOptionsFn = DWORD(WINAPI*)(DWORD); + using SymFromAddrFn = BOOL(WINAPI*)(HANDLE, DWORD64, PDWORD64, PSYMBOL_INFO); + + Symbolizer() { + const HMODULE dbghelp = LoadLibraryA("dbghelp.dll"); + if (dbghelp == nullptr) { + return; + } + const auto set_options = + reinterpret_cast<SymSetOptionsFn>(GetProcAddress(dbghelp, "SymSetOptions")); + const auto initialize = + reinterpret_cast<SymInitializeFn>(GetProcAddress(dbghelp, "SymInitialize")); + from_address = reinterpret_cast<SymFromAddrFn>(GetProcAddress(dbghelp, "SymFromAddr")); + if (set_options == nullptr || initialize == nullptr || from_address == nullptr) { + from_address = nullptr; + return; + } + set_options(SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS); + if (!initialize(GetCurrentProcess(), nullptr, TRUE)) { + from_address = nullptr; + } + } + + const std::string& Name(DWORD64 address) { + const auto [it, is_new] = cache.try_emplace(address); + if (!is_new) { + return it->second; + } + if (from_address != nullptr) { + alignas(SYMBOL_INFO) char buffer[sizeof(SYMBOL_INFO) + 1024]{}; + auto* const symbol = reinterpret_cast<SYMBOL_INFO*>(buffer); + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); + symbol->MaxNameLen = 1024; + DWORD64 displacement{}; + if (from_address(GetCurrentProcess(), address, &displacement, symbol)) { + it->second.assign(symbol->Name, std::min<size_t>(symbol->NameLen, 160)); + return it->second; + } + } + HMODULE module{}; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast<LPCSTR>(address), &module)) { + char path[MAX_PATH] = "?"; + GetModuleFileNameA(module, path, sizeof(path)); + const char* base = std::strrchr(path, '\\'); + it->second = std::string{base != nullptr ? base + 1 : path} + "!?"; + } else { + it->second = "(jit)"; + } + return it->second; + } + + SymFromAddrFn from_address{}; + std::unordered_map<DWORD64, std::string> cache; +}; + +void Report(const std::vector<Target>& targets, const std::vector<Sample>& samples, + const char* path) { + Symbolizer symbolizer; + std::FILE* const file = std::fopen(path, "w"); + if (file == nullptr) { + return; + } + for (unsigned thread = 0; thread < targets.size(); ++thread) { + std::map<std::string, unsigned> self; + std::map<std::string, unsigned> inclusive; + std::map<std::string, unsigned> self_caller; + unsigned total = 0; + for (const Sample& sample : samples) { + if (sample.thread != thread || sample.depth == 0) { + continue; + } + ++total; + const std::string& top = symbolizer.Name(sample.frames[0]); + ++self[top]; + if (sample.depth > 1) { + ++self_caller[top + " <- " + symbolizer.Name(sample.frames[1])]; + } + std::set<std::string> seen; + for (unsigned i = 0; i < sample.depth; ++i) { + const std::string& name = symbolizer.Name(sample.frames[i]); + if (seen.insert(name).second) { + ++inclusive[name]; + } + } + } + const auto print = [&](const char* title, const std::map<std::string, unsigned>& counts, + size_t limit) { + std::vector<std::pair<unsigned, std::string>> sorted; + for (const auto& [name, count] : counts) { + sorted.emplace_back(count, name); + } + std::sort(sorted.rbegin(), sorted.rend()); + std::fprintf(file, "-- %s\n", title); + for (size_t i = 0; i < std::min(limit, sorted.size()); ++i) { + std::fprintf(file, "%6u %5.1f%% %s\n", sorted[i].first, + 100.0 * sorted[i].first / std::max(total, 1U), + sorted[i].second.c_str()); + } + }; + std::fprintf(file, "=== thread \"%s\": %u samples\n", targets[thread].name.c_str(), total); + print("self (where the thread is)", self, 60); + print("self <- caller", self_caller, 50); + print("inclusive (on the stack)", inclusive, 90); + std::fprintf(file, "\n"); + } + std::fclose(file); + + // <out>.folded: every distinct stack with its count, root first ("thread;a;b;c count"). + std::map<std::string, unsigned> folded; + for (const Sample& sample : samples) { + std::string line = targets[sample.thread].name; + for (unsigned i = sample.depth; i-- > 0;) { + line += ';'; + line += symbolizer.Name(sample.frames[i]); + } + ++folded[line]; + } + if (std::FILE* const out = std::fopen((std::string{path} + ".folded").c_str(), "w")) { + for (const auto& [line, count] : folded) { + std::fprintf(out, "%s %u\n", line.c_str(), count); + } + std::fclose(out); + } +} + +void Run(std::vector<std::string> patterns, double start, double seconds, std::string out) { + std::this_thread::sleep_for(std::chrono::duration<double>(start)); + const std::vector<Target> targets = FindThreads(patterns); + std::fprintf(stderr, "nx2 sampler: %zu threads match, sampling %.0f s\n", targets.size(), + seconds); + if (targets.empty()) { + return; + } + std::vector<Sample> samples; + samples.reserve(static_cast<size_t>(seconds * 1100.0) * targets.size() + 16); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::duration<double>(seconds); + while (std::chrono::steady_clock::now() < deadline && + samples.size() + targets.size() <= samples.capacity()) { + for (unsigned i = 0; i < targets.size(); ++i) { + const HANDLE handle = targets[i].handle; + if (SuspendThread(handle) == static_cast<DWORD>(-1)) { + continue; + } + CONTEXT context{}; + context.ContextFlags = CONTEXT_FULL; + if (GetThreadContext(handle, &context)) { + Sample& sample = samples.emplace_back(); + sample.thread = i; + sample.depth = Walk(context, sample.frames); + } + ResumeThread(handle); + } + Sleep(1); + } + Report(targets, samples, out.c_str()); + std::fprintf(stderr, "nx2 sampler: %zu samples written to %s\n", samples.size(), out.c_str()); + for (const auto& target : targets) { + CloseHandle(target.handle); + } +} + +} // Anonymous namespace + +void StartFromEnvironment() { + const char* names = std::getenv("EDEN_NX2_SAMPLE"); + if (names == nullptr || *names == '\0') { + return; + } + const char* out = std::getenv("EDEN_NX2_SAMPLE_OUT"); + std::thread{Run, SplitNames(names), EnvSeconds("EDEN_NX2_SAMPLE_START", 60.0), + EnvSeconds("EDEN_NX2_SAMPLE_SECONDS", 30.0), + std::string{out != nullptr ? out : "nx2_samples.txt"}} + .detach(); +} + +} // namespace Nx2Sampler + +#else + +namespace Nx2Sampler { +void StartFromEnvironment() {} +} // namespace Nx2Sampler + +#endif diff --git a/src/yuzu_cmd/nx2_sampler.h b/src/yuzu_cmd/nx2_sampler.h new file mode 100644 index 0000000000..a73c8fa302 --- /dev/null +++ b/src/yuzu_cmd/nx2_sampler.h @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +namespace Nx2Sampler { + +// EDEN_NX2_SAMPLE=<thread names>: a sampling profiler for named host threads (Windows only). +// Names are comma separated; a name ending in '*' matches by prefix (CPUCore_*). From +// EDEN_NX2_SAMPLE_START seconds (default 60) for EDEN_NX2_SAMPLE_SECONDS (default 30), each +// thread is paused about 1000 times a second and its call stack recorded; the counts per +// function (where the thread is, and everything on its stack) go to EDEN_NX2_SAMPLE_OUT +// (default nx2_samples.txt). +void StartFromEnvironment(); + +} // namespace Nx2Sampler diff --git a/src/yuzu_cmd/yuzu.cpp b/src/yuzu_cmd/yuzu.cpp index 473a5b1014..ffa1876c4f 100644 --- a/src/yuzu_cmd/yuzu.cpp +++ b/src/yuzu_cmd/yuzu.cpp @@ -4,6 +4,14 @@ // SPDX-FileCopyrightText: 2014 Citra Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <atomic> +#include <iterator> +#include <fstream> +#include <vector> +#include <cstdlib> +#include <chrono> +#include <cstdio> +#include <cstring> #include <iostream> #include <memory> #include <mutex> @@ -33,6 +41,7 @@ #include "core/hle/service/filesystem/filesystem.h" #include "core/loader/loader.h" #include "frontend_common/config.h" +#include "input_common/drivers/keyboard.h" #include "input_common/main.h" #include "network/network.h" #include "sdl_config.h" @@ -45,6 +54,7 @@ #endif #include "yuzu_cmd/emu_window/emu_window_sdl3_null.h" #include "yuzu_cmd/emu_window/emu_window_sdl3_vk.h" +#include "yuzu_cmd/nx2_sampler.h" #ifdef _WIN32 // windows.h needs to be included before shellapi.h @@ -60,6 +70,75 @@ extern "C" { __declspec(dllexport) unsigned long NvOptimusEnablement = 0x00000001; __declspec(dllexport) int AmdPowerXpressRequestHighPerformance = 1; } + +// EDEN_NX2_CRASH_DUMP=<dir>: on an unhandled exception (a real crash; fastmem faults are handled +// before this), write <dir>/eden-crash-<pid>.dmp. A debugger changes the timing enough to hide +// some crashes, so this catches them at full speed. dbghelp.dll is loaded only then. +// Prints a code address as module+offset (resolve with llvm-symbolizer and the module's PDB). +static void PrintCodeAddress(const char* label, const void* address) { + HMODULE module{}; + char name[MAX_PATH] = "?"; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + static_cast<LPCSTR>(address), &module)) { + GetModuleFileNameA(module, name, sizeof(name)); + } + const char* base_name = std::strrchr(name, '\\'); + std::fprintf(stderr, " %s %p %s+%#llx\n", label, address, base_name ? base_name + 1 : name, + static_cast<unsigned long long>(reinterpret_cast<uintptr_t>(address) - + reinterpret_cast<uintptr_t>(module))); +} + +static LONG WINAPI WriteCrashDump(EXCEPTION_POINTERS* exception) { + static std::atomic<bool> written{}; + const char* directory = std::getenv("EDEN_NX2_CRASH_DUMP"); + if (directory == nullptr || written.exchange(true)) { + return EXCEPTION_CONTINUE_SEARCH; + } + // The faulting address and stack first: they survive even if the dump fails. + const EXCEPTION_RECORD& record = *exception->ExceptionRecord; + std::fprintf(stderr, "Unhandled exception %#lx on thread %lu, info %#llx %#llx\n", + record.ExceptionCode, GetCurrentThreadId(), + static_cast<unsigned long long>(record.NumberParameters > 0 ? record.ExceptionInformation[0] : 0), + static_cast<unsigned long long>(record.NumberParameters > 1 ? record.ExceptionInformation[1] : 0)); + PrintCodeAddress("at", record.ExceptionAddress); + void* frames[48]; + const USHORT count = RtlCaptureStackBackTrace(0, 48, frames, nullptr); + for (USHORT i = 0; i < count; ++i) { + PrintCodeAddress("frame", frames[i]); + } + std::fflush(stderr); + struct ExceptionInformation { + DWORD thread_id; + EXCEPTION_POINTERS* exception_pointers; + BOOL client_pointers; + }; + using MiniDumpWriteDumpFn = BOOL(WINAPI*)(HANDLE, DWORD, HANDLE, int, const ExceptionInformation*, + const void*, const void*); + const HMODULE dbghelp = LoadLibraryA("dbghelp.dll"); + const auto write = dbghelp ? reinterpret_cast<MiniDumpWriteDumpFn>( + GetProcAddress(dbghelp, "MiniDumpWriteDump")) + : nullptr; + if (write == nullptr) { + return EXCEPTION_CONTINUE_SEARCH; + } + char path[MAX_PATH]; + std::snprintf(path, sizeof(path), "%s\\eden-crash-%lu.dmp", directory, GetCurrentProcessId()); + const HANDLE file = CreateFileA(path, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (file != INVALID_HANDLE_VALUE) { + const ExceptionInformation information{GetCurrentThreadId(), exception, FALSE}; + // MiniDumpWithThreadInfo; the larger MiniDumpWithIndirectlyReferencedMemory failed on + // Eden's address space and left an empty file. + const BOOL ok = write(GetCurrentProcess(), GetCurrentProcessId(), file, 0x1000, + &information, nullptr, nullptr); + const DWORD error = ok ? 0 : GetLastError(); + CloseHandle(file); + std::fprintf(stderr, ok ? "Crash dump written to %s\n" : "Crash dump to %s failed (%#lx)\n", + path, error); + } + return EXCEPTION_CONTINUE_SEARCH; +} #endif static void OnStateChanged(const Network::RoomMember::State& state) { @@ -159,17 +238,30 @@ static void OnStatusMessageReceived(const Network::StatusMessageEntry& msg) { struct SdlState { Core::System system{}; + // Must outlive emu_window, which keeps a pointer to it. + InputCommon::InputSubsystem input_subsystem{}; std::unique_ptr<EmuWindow_SDL3> emu_window; }; extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { +#ifdef _WIN32 + SetUnhandledExceptionFilter(WriteCrashDump); +#endif + Nx2Sampler::StartFromEnvironment(); + // SDL calls SDL_AppIterate back to back by default, which kept a host core at 100% polling + // input; 240 Hz is plenty for input and the key script. + SDL_SetHint(SDL_HINT_MAIN_CALLBACK_RATE, "240"); SdlState* state = new SdlState(); + *appstate = state; #ifdef _WIN32 if (AttachConsole(ATTACH_PARENT_PROCESS)) { freopen("CONOUT$", "wb", stdout); freopen("CONOUT$", "wb", stderr); } + // Raise the CRT's open-file limit like the Qt frontend: loose RomFS folders keep thousands of + // files open through the VFS cache. + _setmaxstdio(8192); #endif Common::Log::Initialize(); @@ -202,7 +294,6 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { state->system.Initialize(); - InputCommon::InputSubsystem input_subsystem{}; // Apply the command line arguments state->system.ApplySettings(); @@ -212,14 +303,14 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { case Settings::RendererBackend::OpenGL_GLSL: case Settings::RendererBackend::OpenGL_GLASM: case Settings::RendererBackend::OpenGL_SPIRV: - state->emu_window = std::make_unique<EmuWindow_SDL3_GL>(&input_subsystem, state->system, lp.fullscreen); + state->emu_window = std::make_unique<EmuWindow_SDL3_GL>(&state->input_subsystem, state->system, lp.fullscreen); break; #endif case Settings::RendererBackend::Vulkan: - state->emu_window = std::make_unique<EmuWindow_SDL3_VK>(&input_subsystem, state->system, lp.fullscreen); + state->emu_window = std::make_unique<EmuWindow_SDL3_VK>(&state->input_subsystem, state->system, lp.fullscreen); break; case Settings::RendererBackend::Null: - state->emu_window = std::make_unique<EmuWindow_SDL3_Null>(&input_subsystem, state->system, lp.fullscreen); + state->emu_window = std::make_unique<EmuWindow_SDL3_Null>(&state->input_subsystem, state->system, lp.fullscreen); break; default: LOG_CRITICAL(Frontend, "Invalid renderer backend"); @@ -319,16 +410,73 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { void(state->system.Run()); if (state->system.DebuggerEnabled()) state->system.InitializeDebugger(); - return SDL_APP_SUCCESS; + return SDL_APP_CONTINUE; } +// EDEN_NX2_KEY_SCRIPT="seconds:code,seconds:-code,..." (or "@<file>"): press (code) or release (-code) keyboard +// keys, by SDL scancode, that many seconds after start. Lets hidden test runs drive menus. +double Nx2KeyClockSeconds(); // emu_window_sdl3.cpp + +static void RunKeyScript(InputCommon::InputSubsystem& input_subsystem) { + struct KeyEvent { + double seconds; // when frame is 0 + u64 frame; // "f<frame>:code": after that many presents + int code; + }; + static const std::vector<KeyEvent> script = [] { + std::vector<KeyEvent> events; + if (const char* text = std::getenv("EDEN_NX2_KEY_SCRIPT")) { + // "@<file>" reads the list from a file: long scripts in the environment slow down + // every getenv in the process. + std::string list{text}; + if (!list.empty() && list[0] == '@') { + std::ifstream file{list.substr(1)}; + list.assign(std::istreambuf_iterator<char>{file}, {}); + } + size_t start = 0; + while (start < list.size()) { + const size_t end = std::min(list.find(',', start), list.size()); + const std::string item = list.substr(start, end - start); + if (const size_t colon = item.find(':'); colon != std::string::npos) { + const std::string when = item.substr(0, colon); + const int code = std::stoi(item.substr(colon + 1)); + if (!when.empty() && when[0] == 'f') { + events.push_back({0.0, std::stoull(when.substr(1)), code}); + } else { + events.push_back({std::stod(when), 0, code}); + } + } + start = end + 1; + } + } + return events; + }(); + static size_t next = 0; + const double elapsed = Nx2KeyClockSeconds(); + const u64 frame = Core::g_nx2_present_count.load(); + while (next < script.size() && (script[next].frame != 0 ? script[next].frame <= frame + : script[next].seconds <= elapsed)) { + const int code = script[next].code; + LOG_INFO(Frontend, "Key script: {} key {} at {:.1f} s (frame {})", + code < 0 ? "release" : "press", code < 0 ? -code : code, elapsed, frame); + if (code < 0) { + input_subsystem.GetKeyboard()->ReleaseKey(static_cast<std::size_t>(-code)); + } else { + input_subsystem.GetKeyboard()->PressKey(static_cast<std::size_t>(code)); + } + ++next; + } +} + extern "C" SDL_AppResult SDL_AppIterate(void *appstate) { SdlState *state = (SdlState *)appstate; - return state->emu_window->IsOpen() ? SDL_APP_CONTINUE : SDL_APP_SUCCESS; + RunKeyScript(state->input_subsystem); + return state->emu_window && state->emu_window->IsOpen() ? SDL_APP_CONTINUE : SDL_APP_SUCCESS; } extern "C" SDL_AppResult SDL_AppEvent(void *appstate, SDL_Event *event) { SdlState *state = (SdlState *)appstate; - state->emu_window->OnEvent(*event); - return SDL_APP_SUCCESS; + if (state->emu_window) + state->emu_window->OnEvent(*event); + return SDL_APP_CONTINUE; } extern "C" void SDL_AppQuit(void *appstate, SDL_AppResult result) { SdlState *state = (SdlState *)appstate; |
