diff options
| author | Djihads <76681596+Djihads80@users.noreply.github.com> | 2026-10-29 02:25:36 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-10-29 02:25:36 +0100 |
| commit | 0bb5055b6541c74f47603a8dff0569a509cbaf18 (patch) | |
| tree | da827ff9c4f2f9a5309ee2804ee689f608c0ac2f /src | |
| parent | 1af7fe72927af9fc64aa4af117bc9b30043c4dc7 (diff) | |
| parent | 535b6769709eb22333af26269f1b16f37adce4db (diff) | |
Merge pull request #1 from Djihads80/yumekiru/ns2-progress-v2
Experimental NS2 support and accumulated GPU fixes
Diffstat (limited to 'src')
156 files changed, 12227 insertions, 179 deletions
diff --git a/src/audio_core/renderer/behavior/info_updater.cpp b/src/audio_core/renderer/behavior/info_updater.cpp index a88eae91ad..b9a3745b7e 100644 --- a/src/audio_core/renderer/behavior/info_updater.cpp +++ b/src/audio_core/renderer/behavior/info_updater.cpp @@ -30,6 +30,9 @@ InfoUpdater::InfoUpdater(std::span<const u8> input_, std::span<u8> output_, expected_input_size{input_.size()}, expected_output_size{output_.size()}, process_handle{process_handle_}, behaviour{behaviour_} { std::construct_at<UpdateDataHeader>(out_header, behaviour.GetProcessRevision()); + if ((in_header->revision & 0x00ffffffU) == 0x00764552U) { + out_header->revision ^= 0x00200000U; + } } Result InfoUpdater::UpdateVoiceChannelResources(VoiceContext& voice_context) { @@ -584,11 +587,19 @@ Result InfoUpdater::UpdatePerformanceBuffer(std::span<u8> performance_output, Result InfoUpdater::UpdateBehaviorInfo(BehaviorInfo& behaviour_) { const auto in_params{reinterpret_cast<const BehaviorInfo::InParameter*>(input)}; + static bool logged_header = false; + if (!logged_header) { + logged_header = true; + LOG_WARNING(Service_Audio, "Audio update header revision={:#x}, behavior revision={:#x}, expected={:#x}, behavior_size={}, total={}", + in_header->revision, in_params->revision, behaviour_.GetUserRevision(), in_header->behaviour_size, in_header->size); + } if (!CheckValidRevision(in_params->revision)) { return Service::Audio::ResultInvalidUpdateInfo; } - if (in_params->revision != behaviour_.GetUserRevision()) { + const u32 revision = (in_params->revision & 0x00ffffffU) == 0x00764552U + ? in_params->revision ^ 0x00200000U : in_params->revision; + if (revision != behaviour_.GetUserRevision()) { return Service::Audio::ResultInvalidUpdateInfo; } diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp index eee8ff6d18..b96340999d 100644 --- a/src/common/sparse_large_vector.cpp +++ b/src/common/sparse_large_vector.cpp @@ -26,7 +26,7 @@ static std::mutex vector_regions_mutex {}; // Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { DWORD code = info->ExceptionRecord->ExceptionCode; - u64 exception_addr = reinterpret_cast<u64>(info->ExceptionRecord->ExceptionInformation[1]); + u64 exception_addr = static_cast<u64>(info->ExceptionRecord->ExceptionInformation[1]); if (code != EXCEPTION_ACCESS_VIOLATION || info->ExceptionRecord->ExceptionInformation[0] == 1) { // Not our problem diff --git a/src/common/zbic_compression.cpp b/src/common/zbic_compression.cpp index 5e0e4d947b..a298c6f81e 100644 --- a/src/common/zbic_compression.cpp +++ b/src/common/zbic_compression.cpp @@ -17,9 +17,16 @@ #endif #include "zstd.h" +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4505) +#endif #define g_ZSTD_threading_useless_symbol g_ZSTD_zbic_threading_useless_symbol #include "zstd.c" #undef g_ZSTD_threading_useless_symbol +#ifdef _MSC_VER +#pragma warning(pop) +#endif namespace Common::Compression { diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 1b578f67ba..168037e5dc 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -460,6 +460,18 @@ add_library(core STATIC hle/service/am/service/application_proxy.h hle/service/am/service/application_proxy_service.cpp hle/service/am/service/application_proxy_service.h + hle/service/am/service/performance_functions.cpp + hle/service/am/service/performance_functions.h + hle/service/am/service/gpu_session.cpp + hle/service/am/service/gpu_session.h + hle/service/am/service/graphics_session.cpp + hle/service/am/service/graphics_session.h + hle/service/am/service/graphics_supervisor.cpp + hle/service/am/service/graphics_supervisor.h + hle/service/am/service/graphics_supervisor_protocol.h + hle/service/am/service/graphics_syncpoint_cache.h + hle/service/am/service/graphics_scaling_parameters_functions.cpp + hle/service/am/service/graphics_scaling_parameters_functions.h hle/service/am/service/audio_controller.cpp hle/service/am/service/audio_controller.h hle/service/am/service/common_state_getter.cpp @@ -855,6 +867,8 @@ add_library(core STATIC hle/service/nvdrv/devices/nvhost_ctrl_gpu.h hle/service/nvdrv/devices/nvhost_gpu.cpp hle/service/nvdrv/devices/nvhost_gpu.h + hle/service/nvdrv/devices/nvhost_tsg_gpu.cpp + hle/service/nvdrv/devices/nvhost_tsg_gpu.h hle/service/nvdrv/devices/nvhost_nvdec.cpp hle/service/nvdrv/devices/nvhost_nvdec.h hle/service/nvdrv/devices/nvhost_nvdec_common.cpp diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index 894857a86d..b5190e40cf 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -194,6 +194,7 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s // Callbacks config.callbacks = std::addressof(*m_cb); + config.enable_pauth_returns = m_cb->m_process && m_cb->m_process->UsesPAuthReturns(); // Memory if (page_table) { @@ -444,6 +445,8 @@ void ArmDynarmic64::ClearInstructionCache() { } void ArmDynarmic64::InvalidateCacheRange(u64 addr, std::size_t size) { + // Refetch the source page too, including instructions restored by the debugger. + m_cb->last_code_addr = u64(-1); m_jit->InvalidateCacheRange(addr, size); } diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 97c2706abb..0087548978 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -21,6 +21,7 @@ #include "common/scope_exit.h" #include "common/settings.h" #include "core/device_memory.h" +#include "core/hle/kernel/board/nintendo/nx/k_system_control.h" #include "core/device_memory_manager.h" #include "core/memory.h" @@ -172,7 +173,7 @@ template <typename Traits> DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memory_) : physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())} , device_inter{nullptr} - , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS)) + , compressed_device_addr(Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize() >> Memory::YUZU_PAGEBITS) , tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique<DeviceMemoryManagerAllocator<Traits>>(); diff --git a/src/core/file_sys/program_metadata.cpp b/src/core/file_sys/program_metadata.cpp index 63ca9d3fe3..b528523e67 100644 --- a/src/core/file_sys/program_metadata.cpp +++ b/src/core/file_sys/program_metadata.cpp @@ -20,6 +20,10 @@ ProgramMetadata::ProgramMetadata() = default; ProgramMetadata::~ProgramMetadata() = default; Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) { + uses_extended_acid = false; + if (!file) { + return Loader::ResultStatus::ErrorBadNPDMHeader; + } const std::size_t total_size = file->GetSize(); if (total_size < sizeof(Header)) { return Loader::ResultStatus::ErrorBadNPDMHeader; @@ -29,17 +33,55 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) { return Loader::ResultStatus::ErrorBadNPDMHeader; } - if (sizeof(AcidHeader) != file->ReadObject(&acid_header, npdm_header.acid_offset)) { + if (npdm_header.magic != std::array{'M', 'E', 'T', 'A'}) { + return Loader::ResultStatus::ErrorBadNPDMHeader; + } + + const auto contains = [](u64 offset, u64 size, u64 limit) { + return offset <= limit && size <= limit - offset; + }; + const u64 acid_offset = npdm_header.acid_offset; + const u64 acid_size = npdm_header.acid_size; + const u64 aci_offset = npdm_header.aci_offset; + const u64 aci_size = npdm_header.aci_size; + if (acid_offset < sizeof(Header) || !contains(acid_offset, acid_size, total_size)) { return Loader::ResultStatus::ErrorBadACIDHeader; } - if (sizeof(AciHeader) != file->ReadObject(&aci_header, npdm_header.aci_offset)) { + const auto read_acid = [&](u64 body_offset) { + return contains(body_offset, sizeof(AcidHeader), acid_size) && + file->ReadObject(&acid_header, acid_offset + body_offset) == sizeof(AcidHeader) && + acid_header.magic == std::array{'A', 'C', 'I', 'D'}; + }; + if (!read_acid(0x200)) { + if (!read_acid(0x300)) { + return Loader::ResultStatus::ErrorBadACIDHeader; + } + uses_extended_acid = true; + } + + if (aci_offset < sizeof(Header) || !contains(aci_offset, aci_size, total_size) || + aci_size < sizeof(AciHeader) || + sizeof(AciHeader) != file->ReadObject(&aci_header, aci_offset) || + aci_header.magic != std::array{'A', 'C', 'I', '0'}) { return Loader::ResultStatus::ErrorBadACIHeader; } + constexpr u64 fac_read_size = 4 + sizeof(FileAccessControl::permissions) + + sizeof(FileAccessControl::unknown); + if (acid_header.fac_size < fac_read_size || + !contains(acid_header.fac_offset, acid_header.fac_size, acid_size)) { + return Loader::ResultStatus::ErrorBadFileAccessControl; + } + constexpr u64 fah_read_size = 4 + sizeof(FileAccessHeader::permissions) + 4 * sizeof(u32); + if (aci_header.fah_size < fah_read_size || + !contains(aci_header.fah_offset, aci_header.fah_size, aci_size)) { + return Loader::ResultStatus::ErrorBadFileAccessHeader; + } + // Load acid_file_access per-component instead of the entire struct, since this struct does not // reflect the layout of the real data. - std::size_t current_offset = acid_header.fac_offset; + std::size_t current_offset = acid_offset + acid_header.fac_offset; if (sizeof(FileAccessControl::version) != file->ReadBytes(&acid_file_access.version, sizeof(FileAccessControl::version), current_offset)) { @@ -57,7 +99,7 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) { } // Load aci_file_access per-component instead of the entire struct, same as acid_file_access - current_offset = aci_header.fah_offset; + current_offset = aci_offset + aci_header.fah_offset; if (sizeof(FileAccessHeader::version) != file->ReadBytes(&aci_file_access.version, sizeof(FileAccessHeader::version), current_offset)) { @@ -89,6 +131,10 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) { return Loader::ResultStatus::ErrorBadFileAccessHeader; } + if (aci_header.kac_size % sizeof(u32) != 0 || + !contains(aci_header.kac_offset, aci_header.kac_size, aci_size)) { + return Loader::ResultStatus::ErrorBadKernelCapabilityDescriptors; + } aci_kernel_capabilities.resize(aci_header.kac_size / sizeof(u32)); const u64 read_size = aci_header.kac_size; const u64 read_offset = npdm_header.aci_offset + aci_header.kac_offset; @@ -128,6 +174,7 @@ void ProgramMetadata::LoadManual(bool is_64_bit, ProgramAddressSpaceType address u32 main_thread_stack_size, u64 title_id, u64 filesystem_permissions, u32 system_resource_size, KernelCapabilityDescriptors capabilities) { + uses_extended_acid = false; npdm_header.has_64_bit_instructions.Assign(is_64_bit); npdm_header.address_space_type.Assign(address_space); npdm_header.main_thread_priority = static_cast<u8>(main_thread_prio); diff --git a/src/core/file_sys/program_metadata.h b/src/core/file_sys/program_metadata.h index 115e6d6cd1..eb4cf11894 100644 --- a/src/core/file_sys/program_metadata.h +++ b/src/core/file_sys/program_metadata.h @@ -77,6 +77,9 @@ public: u8 GetMainThreadCore() const; u32 GetMainThreadStackSize() const; u64 GetTitleID() const; + bool UsesExtendedAcid() const { + return uses_extended_acid; + } u64 GetFilesystemPermissions() const; u32 GetSystemResourceSize() const; PoolPartition GetPoolPartition() const; @@ -116,8 +119,6 @@ private: static_assert(sizeof(Header) == 0x80, "NPDM header structure size is wrong"); struct AcidHeader { - std::array<u8, 0x100> signature; - std::array<u8, 0x100> nca_modulus; std::array<char, 4> magic; u32_le nca_size; std::array<u8, 0x4> reserved; @@ -139,7 +140,7 @@ private: INSERT_PADDING_BYTES(0x8); }; - static_assert(sizeof(AcidHeader) == 0x240, "ACID header structure size is wrong"); + static_assert(sizeof(AcidHeader) == 0x40, "ACID header body size is wrong"); struct AciHeader { std::array<char, 4> magic; @@ -177,6 +178,7 @@ private: }; Header npdm_header{}; + bool uses_extended_acid{}; AciHeader aci_header{}; AcidHeader acid_header{}; diff --git a/src/core/hle/kernel/k_memory_block.h b/src/core/hle/kernel/k_memory_block.h index f6543bdb3a..2abca31d87 100644 --- a/src/core/hle/kernel/k_memory_block.h +++ b/src/core/hle/kernel/k_memory_block.h @@ -195,7 +195,9 @@ enum class KMemoryAttribute : u8 { Uncached = static_cast<u8>(Svc::MemoryAttribute::Uncached), PermissionLocked = static_cast<u8>(Svc::MemoryAttribute::PermissionLocked), - SetMask = Uncached | PermissionLocked, + GpuAccessible = static_cast<u8>(Svc::MemoryAttribute::GpuAccessible), + + SetMask = Uncached | PermissionLocked | GpuAccessible, }; DECLARE_ENUM_FLAG_OPERATORS(KMemoryAttribute); diff --git a/src/core/hle/kernel/k_page_table_base.cpp b/src/core/hle/kernel/k_page_table_base.cpp index 2a0a7d49d5..fd5d2c2dce 100644 --- a/src/core/hle/kernel/k_page_table_base.cpp +++ b/src/core/hle/kernel/k_page_table_base.cpp @@ -2102,7 +2102,8 @@ Result KPageTableBase::SetMemoryAttribute(KProcessAddress addr, size_t size, KMe constexpr KMemoryAttribute AttributeTestMask = ~(KMemoryAttribute::SetMask | KMemoryAttribute::DeviceShared); const KMemoryState state_test_mask = - (True(mask & KMemoryAttribute::Uncached) ? KMemoryState::FlagCanChangeAttribute + (True(mask & (KMemoryAttribute::Uncached | KMemoryAttribute::GpuAccessible)) + ? KMemoryState::FlagCanChangeAttribute : KMemoryState::None) | (True(mask & KMemoryAttribute::PermissionLocked) ? KMemoryState::FlagCanPermissionLock : KMemoryState::None); diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp index 27d1b1bfc4..813d8e9402 100644 --- a/src/core/hle/kernel/k_process.cpp +++ b/src/core/hle/kernel/k_process.cpp @@ -1260,6 +1260,7 @@ Result KProcess::LoadFromMetadata(KernelCore& kernel, const FileSys::ProgramMeta // Assign remaining properties. m_ideal_core_id = metadata.GetMainThreadCore(); + m_uses_pauth_returns = metadata.UsesExtendedAcid(); // Set up emulation context. this->InitializeInterfaces(kernel); diff --git a/src/core/hle/kernel/k_process.h b/src/core/hle/kernel/k_process.h index b5693751c3..ca96591ed0 100644 --- a/src/core/hle/kernel/k_process.h +++ b/src/core/hle/kernel/k_process.h @@ -135,6 +135,7 @@ private: bool m_is_signaled : 1 = false; bool m_is_initialized : 1 = false; bool m_is_application : 1 = false; + bool m_uses_pauth_returns : 1 = false; bool m_is_default_application_system_resource : 1 = false; bool m_is_suspended : 1 = false; bool m_is_immortal : 1 = false; @@ -248,6 +249,10 @@ public: return m_is_application; } + bool UsesPAuthReturns() const { + return m_uses_pauth_returns; + } + bool IsDefaultApplicationSystemResource() const { return m_is_default_application_system_resource; } diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index 4f1d849209..c7beb9c3ce 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -120,13 +120,13 @@ struct KernelCore::Impl { } void TerminateAllProcesses() { - std::scoped_lock lk{process_list_lock}; - for (auto& process : process_list) { - process->Terminate(system.Kernel()); - process->Close(system.Kernel()); - process = nullptr; + auto& kernel = system.Kernel(); + // Applet cleanup can remove processes while termination waits for its threads. + auto processes = kernel.GetProcessList(); + for (auto& process : processes) { + process->Terminate(kernel); + kernel.RemoveProcess(process.GetPointerUnsafe()); } - process_list.clear(); } void Shutdown(KernelCore& kernel) { diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp index 3542d5a3b6..c91ae43dd8 100644 --- a/src/core/hle/kernel/svc.cpp +++ b/src/core/hle/kernel/svc.cpp @@ -2370,6 +2370,10 @@ static void Call32(Core::System& system, u32 imm, std::span<uint64_t, 8> args) { static void Call64(Core::System& system, u32 imm, std::span<uint64_t, 8> args) { switch (SvcId(imm)) { + case SvcId::SetGpuMemoryAttribute: + args[0] = SetGpuMemoryAttribute(system, args[0], args[1], + static_cast<u32>(args[2]), static_cast<u32>(args[3])).raw; + return; case SvcId::SetHeapSize: return SvcWrap_SetHeapSize64(system, args); case SvcId::SetMemoryPermission: return SvcWrap_SetMemoryPermission64(system, args); case SvcId::SetMemoryAttribute: return SvcWrap_SetMemoryAttribute64(system, args); diff --git a/src/core/hle/kernel/svc.h b/src/core/hle/kernel/svc.h index 356c1c109a..422b8b8719 100644 --- a/src/core/hle/kernel/svc.h +++ b/src/core/hle/kernel/svc.h @@ -22,6 +22,7 @@ namespace Kernel::Svc { Result SetHeapSize(Core::System& system, uint64_t* out_address, uint64_t size); Result SetMemoryPermission(Core::System& system, uint64_t address, uint64_t size, MemoryPermission perm); Result SetMemoryAttribute(Core::System& system, uint64_t address, uint64_t size, uint32_t mask, uint32_t attr); +Result SetGpuMemoryAttribute(Core::System& system, uint64_t address, uint64_t size, uint32_t mask, uint32_t attr); Result MapMemory(Core::System& system, uint64_t dst_address, uint64_t src_address, uint64_t size); Result UnmapMemory(Core::System& system, uint64_t dst_address, uint64_t src_address, uint64_t size); Result QueryMemory(Core::System& system, uint64_t out_memory_info, PageInfo* out_page_info, uint64_t address); @@ -504,6 +505,7 @@ enum class SvcId : u32 { CreateResourceLimit = 0x7d, SetResourceLimitLimitValue = 0x7e, CallSecureMonitor = 0x7f, + SetGpuMemoryAttribute = 0x80, MapInsecureMemory = 0x90, UnmapInsecureMemory = 0x91, }; diff --git a/src/core/hle/kernel/svc/svc_info.cpp b/src/core/hle/kernel/svc/svc_info.cpp index 863bb58fa8..aea7407542 100644 --- a/src/core/hle/kernel/svc/svc_info.cpp +++ b/src/core/hle/kernel/svc/svc_info.cpp @@ -14,11 +14,32 @@ #include "core/hle/kernel/svc_version.h" namespace Kernel::Svc { +namespace { +// Identified from the extended SDK's RemoteRegionManager initialization. +constexpr InfoType RemoteRegionAddress = static_cast<InfoType>(29); +constexpr InfoType RemoteRegionSize = static_cast<InfoType>(30); +constexpr InfoType RemoteMemoryUsed = static_cast<InfoType>(31); +constexpr InfoType RemoteMemoryTotal = static_cast<InfoType>(32); +constexpr InfoType MemoryMappingAlignment = static_cast<InfoType>(33); +} // namespace /// Gets system/memory information for the current process Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle handle, u64 info_sub_id) { LOG_TRACE(Kernel_SVC, "called info_id={:#x}, info_sub_id={:#x}, handle={:#08x}", info_id_type, info_sub_id, handle); u32 info_id = u32(info_id_type); + if (info_id_type == RemoteRegionAddress || info_id_type == RemoteRegionSize || + info_id_type == RemoteMemoryUsed || info_id_type == RemoteMemoryTotal || + info_id_type == MemoryMappingAlignment) { + R_UNLESS(info_sub_id == 0, ResultInvalidEnumValue); + const auto& handle_table = GetCurrentProcess(system.Kernel()).GetHandleTable(); + KScopedAutoObject process = handle_table.GetObject<KProcess>(system.Kernel(), handle); + R_UNLESS(process.IsNotNull(), ResultInvalidHandle); + R_UNLESS(process->UsesPAuthReturns(), ResultInvalidEnumValue); + // Remote mappings are absent; ordinary mappings use the kernel page size. + *result = info_id_type == MemoryMappingAlignment ? PageSize : 0; + LOG_DEBUG(Kernel_SVC, "Extended memory info_id={:#x}, value={:#x}", info_id, *result); + R_SUCCEED(); + } switch (info_id_type) { case InfoType::CoreMask: case InfoType::PriorityMask: @@ -124,7 +145,8 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle default: break; } - LOG_ERROR(Kernel_SVC, "Unimplemented svcGetInfo id={:#016x}", info_id); + LOG_ERROR(Kernel_SVC, "Unimplemented svcGetInfo id={:#x}, sub_id={:#x}, handle={:#x}", + info_id, info_sub_id, handle); R_THROW(ResultInvalidEnumValue); } case InfoType::DebuggerAttached: @@ -258,7 +280,8 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle R_SUCCEED(); } default: - LOG_ERROR(Kernel_SVC, "Unimplemented svcGetInfo id={:#016x}", info_id); + LOG_ERROR(Kernel_SVC, "Unimplemented svcGetInfo id={:#x}, sub_id={:#x}, handle={:#x}", + info_id, info_sub_id, handle); R_THROW(ResultInvalidEnumValue); } } diff --git a/src/core/hle/kernel/svc/svc_ipc.cpp b/src/core/hle/kernel/svc/svc_ipc.cpp index 875590d6f7..fadde6c093 100644 --- a/src/core/hle/kernel/svc/svc_ipc.cpp +++ b/src/core/hle/kernel/svc/svc_ipc.cpp @@ -13,6 +13,7 @@ #include "core/hle/kernel/k_scoped_resource_reservation.h" #include "core/hle/kernel/k_server_session.h" #include "core/hle/kernel/k_session.h" +#include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/svc.h" #include "core/hle/kernel/svc_results.h" @@ -142,7 +143,23 @@ Result ReplyAndReceiveImpl(KernelCore& kernel, int32_t* out_index, uintptr_t mes /// Makes a blocking IPC call to a service. Result SendSyncRequest(Core::System& system, Handle session_handle) { - R_RETURN(SendSyncRequestImpl(system.Kernel(), 0, 0, session_handle)); + auto& process = GetCurrentProcess(system.Kernel()); + if (!process.UsesPAuthReturns()) { + R_RETURN(SendSyncRequestImpl(system.Kernel(), 0, 0, session_handle)); + } + const auto tls = GetCurrentThread(system.Kernel()).GetTlsAddress(); + const u64 header = process.GetMemory().Read64(tls); + const bool trace_tipc = (header & 0xffff) >= 16; + if (trace_tipc) { + LOG_DEBUG(Kernel_SVC, "TIPC send handle={:#x}, header={:#x}, special={:#x}", + session_handle, header, process.GetMemory().Read64(tls + 8)); + } + const auto result = SendSyncRequestImpl(system.Kernel(), 0, 0, session_handle); + if (trace_tipc) { + LOG_DEBUG(Kernel_SVC, "TIPC receive result={:#x}, header={:#x}, data={:#x}", + result.raw, process.GetMemory().Read64(tls), process.GetMemory().Read64(tls + 8)); + } + R_RETURN(result); } Result SendSyncRequestWithUserBuffer(Core::System& system, uint64_t message, uint64_t buffer_size, diff --git a/src/core/hle/kernel/svc/svc_memory.cpp b/src/core/hle/kernel/svc/svc_memory.cpp index 970912fe50..53c0d43456 100644 --- a/src/core/hle/kernel/svc/svc_memory.cpp +++ b/src/core/hle/kernel/svc/svc_memory.cpp @@ -126,6 +126,31 @@ Result SetMemoryAttribute(Core::System& system, u64 address, u64 size, u32 mask, R_UNLESS(page_table.Contains(address, size), ResultInvalidCurrentMemory); // Set the memory attribute. + const auto result = page_table.SetMemoryAttribute(address, size, + static_cast<KMemoryAttribute>(mask), + static_cast<KMemoryAttribute>(attr)); + if (GetCurrentProcess(system.Kernel()).UsesPAuthReturns()) { + LOG_DEBUG(Kernel_SVC, "native memory attribute result={:#x}", result.raw); + } + R_RETURN(result); +} + +// Captured native SDK callers set bit 0x20 before registering GPU-accessible +// memory and clear it during destruction. This tracks eligibility on real mapped +// memory; GPU mappings and submission remain separate operations. +Result SetGpuMemoryAttribute(Core::System& system, u64 address, u64 size, u32 mask, u32 attr) { + auto& process = GetCurrentProcess(system.Kernel()); + R_UNLESS(process.UsesPAuthReturns(), ResultNotImplemented); + LOG_DEBUG(Kernel_SVC, "GPU memory address={:#x}, size={:#x}, mask={:#x}, attr={:#x}", + address, size, mask, attr); + R_UNLESS(Common::IsAligned(address, PageSize), ResultInvalidAddress); + R_UNLESS(size > 0 && Common::IsAligned(size, PageSize), ResultInvalidSize); + R_UNLESS(address < address + size, ResultInvalidCurrentMemory); + constexpr u32 SupportedMask = static_cast<u32>(MemoryAttribute::GpuAccessible); + R_UNLESS((attr & ~mask) == 0 && ((mask | attr) & ~SupportedMask) == 0, + ResultInvalidCombination); + auto& page_table = process.GetPageTable(); + R_UNLESS(page_table.Contains(address, size), ResultInvalidCurrentMemory); R_RETURN(page_table.SetMemoryAttribute(address, size, static_cast<KMemoryAttribute>(mask), static_cast<KMemoryAttribute>(attr))); } diff --git a/src/core/hle/kernel/svc/svc_synchronization.cpp b/src/core/hle/kernel/svc/svc_synchronization.cpp index 2b019c52d1..2f72f60215 100644 --- a/src/core/hle/kernel/svc/svc_synchronization.cpp +++ b/src/core/hle/kernel/svc/svc_synchronization.cpp @@ -15,13 +15,24 @@ namespace Kernel::Svc { +namespace { +void LogInvalidNativeSyncHandle(Core::System& system, const char* operation, Handle handle) { + if (!GetCurrentProcess(system.Kernel()).UsesPAuthReturns()) { return; } + const auto& context = GetCurrentThread(system.Kernel()).GetContext(); + LOG_DEBUG(Kernel_SVC, "native {} invalid handle={:#x}, pc={:#x}, lr={:#x}, fp={:#x}", + operation, handle, context.pc, context.lr, context.fp); +} +} + /// Close a handle Result CloseHandle(Core::System& system, Handle handle) { LOG_TRACE(Kernel_SVC, "Closing handle {:#08x}", handle); // Remove the handle. - R_UNLESS(GetCurrentProcess(system.Kernel()).GetHandleTable().Remove(system.Kernel(), handle), - ResultInvalidHandle); + if (!GetCurrentProcess(system.Kernel()).GetHandleTable().Remove(system.Kernel(), handle)) { + LogInvalidNativeSyncHandle(system, "CloseHandle", handle); + R_THROW(ResultInvalidHandle); + } R_SUCCEED(); } @@ -49,6 +60,7 @@ Result ResetSignal(Core::System& system, Handle handle) { } } + LogInvalidNativeSyncHandle(system, "ResetSignal", handle); R_THROW(ResultInvalidHandle); } @@ -72,7 +84,10 @@ Result WaitSynchronization(Core::System& system, int32_t* out_index, u64 user_ha R_UNLESS(GetCurrentMemory(system.Kernel()).ReadBlock(user_handles, handles.data(), sizeof(Handle) * num_handles), ResultInvalidPointer); // Convert the handles to objects. - R_UNLESS(handle_table.GetMultipleObjects<KSynchronizationObject>(system.Kernel(), objs.data(), handles.data(), num_handles), ResultInvalidHandle); + if (!handle_table.GetMultipleObjects<KSynchronizationObject>(system.Kernel(), objs.data(), handles.data(), num_handles)) { + for (s32 i = 0; i < num_handles; ++i) { LogInvalidNativeSyncHandle(system, "WaitSynchronization set", handles[i]); } + R_THROW(ResultInvalidHandle); + } } // Ensure handles are closed when we're done. diff --git a/src/core/hle/kernel/svc/svc_thread.cpp b/src/core/hle/kernel/svc/svc_thread.cpp index dbca13dd26..b4eb0b5719 100644 --- a/src/core/hle/kernel/svc/svc_thread.cpp +++ b/src/core/hle/kernel/svc/svc_thread.cpp @@ -16,8 +16,11 @@ namespace Kernel::Svc { namespace { -constexpr bool IsValidVirtualCoreId(int32_t core_id) { - return (0 <= core_id && core_id < static_cast<int32_t>(Core::Hardware::NUM_CPU_CORES)); +bool IsValidVirtualCoreId(const KProcess& process, int32_t core_id) { + // Extended processes use the kernel's existing virtual-core mapping. + const auto core_count = process.UsesPAuthReturns() ? Core::Hardware::NumVirtualCores + : Core::Hardware::NUM_CPU_CORES; + return 0 <= core_id && core_id < static_cast<int32_t>(core_count); } } // Anonymous namespace @@ -38,7 +41,7 @@ Result CreateThread(Core::System& system, Handle* out_handle, u64 entry_point, u } // Validate arguments. - R_UNLESS(IsValidVirtualCoreId(core_id), ResultInvalidCoreId); + R_UNLESS(IsValidVirtualCoreId(process, core_id), ResultInvalidCoreId); R_UNLESS(((1ull << core_id) & process.GetCoreMask()) != 0, ResultInvalidCoreId); R_UNLESS(HighestThreadPriority <= priority && priority <= LowestThreadPriority, @@ -262,7 +265,7 @@ Result SetThreadCoreMask(Core::System& system, Handle thread_handle, s32 core_id R_UNLESS(affinity_mask != 0, ResultInvalidCombination); // Validate the core id. - if (IsValidVirtualCoreId(core_id)) { + if (IsValidVirtualCoreId(GetCurrentProcess(system.Kernel()), core_id)) { R_UNLESS(((1ULL << core_id) & affinity_mask) != 0, ResultInvalidCombination); } else { R_UNLESS(core_id == IdealCoreNoUpdate || core_id == IdealCoreDontCare, diff --git a/src/core/hle/kernel/svc_types.h b/src/core/hle/kernel/svc_types.h index b0f4796da4..c7b68cc0df 100644 --- a/src/core/hle/kernel/svc_types.h +++ b/src/core/hle/kernel/svc_types.h @@ -50,6 +50,7 @@ enum class MemoryAttribute : u32 { DeviceShared = (1 << 2), Uncached = (1 << 3), PermissionLocked = (1 << 4), + GpuAccessible = (1 << 5), }; DECLARE_ENUM_FLAG_OPERATORS(MemoryAttribute); diff --git a/src/core/hle/service/am/applet.h b/src/core/hle/service/am/applet.h index 571fa65219..69d2876363 100644 --- a/src/core/hle/service/am/applet.h +++ b/src/core/hle/service/am/applet.h @@ -84,6 +84,7 @@ struct Applet { // Application functions bool game_play_recording_supported{}; bool media_playback_state{}; + bool graphics_scale_changed_notification_enabled{}; GamePlayRecordingState game_play_recording_state{GamePlayRecordingState::Disabled}; bool jit_service_launched{}; bool application_crash_report_enabled{}; diff --git a/src/core/hle/service/am/service/application_functions.cpp b/src/core/hle/service/am/service/application_functions.cpp index 90d292c4a2..f86806dd3c 100644 --- a/src/core/hle/service/am/service/application_functions.cpp +++ b/src/core/hle/service/am/service/application_functions.cpp @@ -118,7 +118,8 @@ IApplicationFunctions::IApplicationFunctions(Core::System& system_, std::shared_ {220, nullptr, "Unknown220"}, // [20.0.0+] {300, nullptr, "CreateMovieWriter"}, // [19.0.0+] {310, nullptr, "Unknown310"}, // [20.0.0+] - {320, nullptr, "Unknown320"}, // [20.0.0+] + {320, D<&IApplicationFunctions::GetScreenShotAndContinuousRecordingInitiallyEnabled>, + "GetScreenShotAndContinuousRecordingInitiallyEnabled"}, {330, D<&IApplicationFunctions::Unknown330>, "Unknown330"}, // [20.0.0+] {500, nullptr, "StartContinuousRecordingFlushForDebug"}, {1000, nullptr, "CreateMovieMaker"}, @@ -544,6 +545,17 @@ Result IApplicationFunctions::GetUnknownEvent210( R_SUCCEED(); } +Result IApplicationFunctions::GetScreenShotAndContinuousRecordingInitiallyEnabled( + Out<bool> out_screenshot_enabled, Out<bool> out_recording_enabled) { + std::scoped_lock lk{m_applet->lock}; + *out_screenshot_enabled = m_applet->screenshot_permission != ScreenshotPermission::Disable; + *out_recording_enabled = m_applet->game_play_recording_supported && + m_applet->game_play_recording_state == GamePlayRecordingState::Enabled; + LOG_DEBUG(Service_AM, "called, screenshot={}, recording={}", *out_screenshot_enabled, + *out_recording_enabled); + R_SUCCEED(); +} + Result IApplicationFunctions::Unknown330(Out<u8> out) { LOG_DEBUG(Service_AM, "called"); *out = 0; diff --git a/src/core/hle/service/am/service/application_functions.h b/src/core/hle/service/am/service/application_functions.h index b97ed7e25e..172b712f14 100644 --- a/src/core/hle/service/am/service/application_functions.h +++ b/src/core/hle/service/am/service/application_functions.h @@ -82,6 +82,8 @@ private: Result GetHealthWarningDisappearedSystemEvent(OutCopyHandle<Kernel::KReadableEvent> out_event); Result CreateApplicationAndRequestToStart(u64 application_id); Result GetUnknownEvent210(OutCopyHandle<Kernel::KReadableEvent> out_event); + Result GetScreenShotAndContinuousRecordingInitiallyEnabled( + Out<bool> out_screenshot_enabled, Out<bool> out_recording_enabled); Result Unknown330(Out<u8> out); Result PrepareForJit(); diff --git a/src/core/hle/service/am/service/application_proxy.cpp b/src/core/hle/service/am/service/application_proxy.cpp index df438e1e17..02502ae764 100644 --- a/src/core/hle/service/am/service/application_proxy.cpp +++ b/src/core/hle/service/am/service/application_proxy.cpp @@ -11,7 +11,9 @@ #include "core/hle/service/am/service/common_state_getter.h" #include "core/hle/service/am/service/debug_functions.h" #include "core/hle/service/am/service/display_controller.h" +#include "core/hle/service/am/service/graphics_scaling_parameters_functions.h" #include "core/hle/service/am/service/library_applet_creator.h" +#include "core/hle/service/am/service/performance_functions.h" #include "core/hle/service/am/service/process_winding_controller.h" #include "core/hle/service/am/service/self_controller.h" #include "core/hle/service/am/service/window_controller.h" @@ -33,6 +35,9 @@ IApplicationProxy::IApplicationProxy(Core::System& system_, std::shared_ptr<Appl {10, D<&IApplicationProxy::GetProcessWindingController>, "GetProcessWindingController"}, {11, D<&IApplicationProxy::GetLibraryAppletCreator>, "GetLibraryAppletCreator"}, {20, D<&IApplicationProxy::GetApplicationFunctions>, "GetApplicationFunctions"}, + {40, D<&IApplicationProxy::GetPerformanceFunctions>, "GetPerformanceFunctions"}, + {60, D<&IApplicationProxy::GetGraphicsScalingParametersFunctions>, + "GetGraphicsScalingParametersFunctions"}, {1000, D<&IApplicationProxy::GetDebugFunctions>, "GetDebugFunctions"}, }; // clang-format on @@ -106,4 +111,18 @@ Result IApplicationProxy::GetApplicationFunctions( R_SUCCEED(); } +Result IApplicationProxy::GetPerformanceFunctions( + Out<SharedPointer<IPerformanceFunctions>> out_functions) { + LOG_DEBUG(Service_AM, "called"); + *out_functions = std::make_shared<IPerformanceFunctions>(system); + R_SUCCEED(); +} + +Result IApplicationProxy::GetGraphicsScalingParametersFunctions( + Out<SharedPointer<IGraphicsScalingParametersFunctions>> out_functions) { + LOG_DEBUG(Service_AM, "called"); + *out_functions = std::make_shared<IGraphicsScalingParametersFunctions>(system, m_applet); + R_SUCCEED(); +} + } // namespace Service::AM diff --git a/src/core/hle/service/am/service/application_proxy.h b/src/core/hle/service/am/service/application_proxy.h index b56f597605..aa1ab1938a 100644 --- a/src/core/hle/service/am/service/application_proxy.h +++ b/src/core/hle/service/am/service/application_proxy.h @@ -17,7 +17,9 @@ class IApplicationFunctions; class ICommonStateGetter; class IDebugFunctions; class IDisplayController; +class IGraphicsScalingParametersFunctions; class ILibraryAppletCreator; +class IPerformanceFunctions; class IProcessWindingController; class ISelfController; class IWindowController; @@ -43,6 +45,10 @@ private: Result GetApplicationFunctions( Out<SharedPointer<IApplicationFunctions>> out_application_functions); + Result GetPerformanceFunctions(Out<SharedPointer<IPerformanceFunctions>> out_functions); + Result GetGraphicsScalingParametersFunctions( + Out<SharedPointer<IGraphicsScalingParametersFunctions>> out_functions); + private: WindowSystem& m_window_system; Kernel::KProcess* const m_process; diff --git a/src/core/hle/service/am/service/application_proxy_service.cpp b/src/core/hle/service/am/service/application_proxy_service.cpp index 7727951d8c..da11cce94d 100644 --- a/src/core/hle/service/am/service/application_proxy_service.cpp +++ b/src/core/hle/service/am/service/application_proxy_service.cpp @@ -19,6 +19,8 @@ IApplicationProxyService::IApplicationProxyService(Core::System& system_, : ServiceFramework{system_, "appletOE"}, m_window_system{window_system} { static const FunctionInfo functions[] = { {0, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy"}, + {10, D<&IApplicationProxyService::OpenApplicationProxyWithFlags>, + "OpenApplicationProxyWithFlags"}, }; RegisterHandlers(functions); } @@ -40,6 +42,15 @@ Result IApplicationProxyService::OpenApplicationProxy( } } +Result IApplicationProxyService::OpenApplicationProxyWithFlags( + Out<SharedPointer<IApplicationProxy>> out_application_proxy, u32 flags, ClientProcessId pid, + InCopyHandle<Kernel::KProcess> process_handle) { + LOG_DEBUG(Service_AM, "called, flags={:#x}", flags); + // The extended SDK supplies an additional u32 before the process ID. + R_UNLESS(flags == 0 || flags == 2, ResultUnknown); + R_RETURN(OpenApplicationProxy(out_application_proxy, pid, process_handle)); +} + std::shared_ptr<Applet> IApplicationProxyService::GetAppletFromProcessId(ProcessId process_id) { return m_window_system.GetByAppletResourceUserId(process_id.pid); } diff --git a/src/core/hle/service/am/service/application_proxy_service.h b/src/core/hle/service/am/service/application_proxy_service.h index fce860a7c1..5812f6133a 100644 --- a/src/core/hle/service/am/service/application_proxy_service.h +++ b/src/core/hle/service/am/service/application_proxy_service.h @@ -26,6 +26,10 @@ private: Result OpenApplicationProxy(Out<SharedPointer<IApplicationProxy>> out_application_proxy, ClientProcessId pid, InCopyHandle<Kernel::KProcess> process_handle); + Result OpenApplicationProxyWithFlags( + Out<SharedPointer<IApplicationProxy>> out_application_proxy, u32 flags, ClientProcessId pid, + InCopyHandle<Kernel::KProcess> process_handle); + private: std::shared_ptr<Applet> GetAppletFromProcessId(ProcessId pid); diff --git a/src/core/hle/service/am/service/gpu_memory_registration.h b/src/core/hle/service/am/service/gpu_memory_registration.h new file mode 100644 index 0000000000..37ac7fd377 --- /dev/null +++ b/src/core/hle/service/am/service/gpu_memory_registration.h @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once +#include <optional> +#include <span> +#include "common/common_types.h" + +namespace Service::AM { +struct GpuMemoryRegistration { + u64 begin; + u64 end; + u8 flags; + u8 device; +}; + +constexpr std::optional<GpuMemoryRegistration> DecodeGpuMemoryRegistration(std::span<const u32> packet) { + if (packet.size() < 8 || packet[0] != 0x75 || packet[1] != 6) return std::nullopt; + // The SDK writes STRB at offsets 0x18 and 0x1c, leaving three padding bytes. + return GpuMemoryRegistration{u64{packet[2]} | (u64{packet[3]} << 32), + u64{packet[4]} | (u64{packet[5]} << 32), static_cast<u8>(packet[6]), static_cast<u8>(packet[7])}; +} +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/gpu_session.cpp b/src/core/hle/service/am/service/gpu_session.cpp new file mode 100644 index 0000000000..a83381d7bc --- /dev/null +++ b/src/core/hle/service/am/service/gpu_session.cpp @@ -0,0 +1,633 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <algorithm> +#include <cstring> +#include <mutex> +#include <vector> +#include "common/swap.h" +#include "common/alignment.h" +#include "core/hle/service/nvdrv/core/nvmap.h" +#include "core/core.h" +#include "core/hle/kernel/k_client_session.h" +#include "core/hle/kernel/k_event.h" +#include "core/hle/kernel/k_process.h" +#include "core/hle/kernel/svc_results.h" +#include "core/hle/service/am/service/graphics_session.h" +#include "core/hle/service/am/service/gpu_session.h" +#include "core/hle/service/am/service/gpu_memory_registration.h" +#include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/nvdrv/nvdrv.h" +#include "core/hle/service/nvdrv/devices/nvhost_as_gpu.h" +#include "core/hle/service/nvdrv/devices/native_gpu_kind.h" + +namespace Service::AM { +struct GpuDeviceRegistry { + GpuDeviceRegistry(Core::System& system, std::shared_ptr<Nvidia::Module> module_, Kernel::KProcess* process_) + : context{system, "GpuDevices"}, module{std::move(module_)}, process{process_} { + process->Open(context.kernel); + } + ~GpuDeviceRegistry() { + for (const auto& fd : address_spaces) { if (fd) { module->Close(*fd); } } + if (nv_session) { module->GetContainer().CloseSession(Nvidia::NvCore::SessionId{*nv_session}); } + process->Close(context.kernel); + for (auto* event : events) { context.CloseEvent(event); } + } + KernelHelpers::ServiceContext context; + std::array<Kernel::KEvent*, 64> events{}; + struct Device { u64 name; u64 debug_label; }; + std::mutex mutex; + std::shared_ptr<Nvidia::Module> module; + Kernel::KProcess* process; + std::array<std::optional<s32>, 8> address_spaces{}; + std::array<bool, 8> address_space_registered{}; + std::optional<size_t> nv_session; + std::array<std::optional<Device>, 256> devices{}; +}; +namespace { +class IGpuWorker final : public ServiceFramework<IGpuWorker> { +public: + IGpuWorker(Core::System& system, std::shared_ptr<Nvidia::Module> module, + Kernel::KProcess* process, std::shared_ptr<GpuDeviceRegistry> devices) + : ServiceFramework{system, "IGpuWorker"}, m_devices{std::move(devices)}, + m_module{std::move(module)}, m_process{process} { + m_process->Open(kernel); + static const FunctionInfo functions[] = { + {101, &IGpuWorker::CreateGpuAccessibleMemory, "CreateGpuAccessibleMemory"}, + {102, &IGpuWorker::DestroyGpuAccessibleMemory, "DestroyGpuAccessibleMemory"}, + {211, &IGpuWorker::NormalMapGpuMemory4K, "NormalMapGpuMemory4K"}, + {212, &IGpuWorker::NormalModifyGpuMemory4K, "NormalModifyGpuMemory4K"}, + {213, &IGpuWorker::NormalUnmapGpuMemory4K, "NormalUnmapGpuMemory4K"}, + {214, &IGpuWorker::CreateSparseRegion, "CreateSparseRegion"}, + {215, &IGpuWorker::DestroySparseRegion, "DestroySparseRegion"}, + }; + RegisterHandlersTipc(functions); + } + ~IGpuWorker() override { + for (const auto& mapping : m_mappings) { + if (auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(mapping.as_id)) { + as->UnmapNative4K(mapping.address, mapping.size); + } + } + for (const auto& region : m_sparse_regions) { + if (auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(region.as_id)) { + as->DestroyNativeSparseRegion(region.address, region.size); + } + } + auto& nvmap = m_module->GetContainer().GetNvMapFile(); + for (const auto id : m_memory) { + if (const auto info = nvmap.FreeHandle(id, false); info && info->can_unlock) { + const auto result = m_process->GetPageTable().UnlockForDeviceAddressSpace( + info->address, info->size); + if (result.IsError()) { + LOG_ERROR(Service_AM, "GPU memory cleanup failed: {:#x}", result.raw); + } + } + } + if (m_nv_session) { + m_module->GetContainer().CloseSession(Nvidia::NvCore::SessionId{*m_nv_session}); + } + m_process->Close(kernel); + } + Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override { + if (!ctx.IsTipc() || handlers_tipc.contains(ctx.GetCommand())) { + return ServiceFrameworkBase::HandleSyncRequest(session, ctx); + } + const auto guard = LockService(); + InvokeRequestTipc(ctx); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(Kernel::ResultNotImplemented.raw); + return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess; + } +private: + struct Mapping { u32 as_id; u64 address; u64 size; u32 memory_id; }; + std::vector<Mapping> m_mappings; + std::vector<Mapping> m_sparse_regions; + static u64 ReadPair(const u32* packet, size_t index) { + return u64{packet[index]} | (u64{packet[index + 1]} << 32); + } + bool OwnsAddressSpace(u32 id) const { + for (size_t i = 0; i < m_devices->address_spaces.size(); ++i) { + if (m_devices->address_spaces[i] && + static_cast<u32>(*m_devices->address_spaces[i]) == id && + m_devices->address_space_registered[i]) { return true; } + } + return false; + } + void CreateSparseRegion(HLERequestContext& ctx) { + const std::scoped_lock device_lock{m_devices->mutex}; + const auto* p = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + const u64 begin = ReadPair(p, 3), end = ReadPair(p, 5); + const u32 notification = p[11]; + if (p[0] == 0xe6 && p[1] == 10 && notification < m_devices->events.size() && m_devices->events[notification]) { + result = Kernel::ResultNotFound; + const u32 shift = p[7] & 0xff, device = p[9] & 0xff; + if (OwnsAddressSpace(p[2]) && device < m_devices->devices.size() && m_devices->devices[device]) { + result = Kernel::ResultInvalidArgument; + if ((shift == 12 || shift == 16) && (p[8] & 0xff) <= 1 && (p[10] & 0xff) <= 1 && + end >= begin && end != ~u64{0} && Common::IsAligned(begin, 1ULL << shift) && + Common::IsAligned(end - begin + 1, 1ULL << shift)) { + const auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(p[2]); + result = as && as->CreateNativeSparseRegion(begin, end - begin + 1) == Nvidia::NvResult::Success + ? ResultSuccess : Kernel::ResultInvalidCurrentMemory; + if (result.IsSuccess()) { m_sparse_regions.push_back({p[2], begin, end - begin + 1, 0}); } + } + } + m_devices->events[notification]->Signal(kernel); + } + LOG_DEBUG(Service_AM, "GPU sparse region as={}, VA={:#x}..{:#x}, result={:#x}", p[2], begin, end, result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void DestroySparseRegion(HLERequestContext& ctx) { + const std::scoped_lock device_lock{m_devices->mutex}; + const auto* p = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + const u64 begin = ReadPair(p, 3), end = ReadPair(p, 5); + const u32 notification = p[8]; + if (p[0] == 0xe7 && p[1] == 7 && (p[7] & 0xff) <= 1 && + notification < m_devices->events.size() && m_devices->events[notification]) { + const auto it = std::find_if(m_sparse_regions.begin(), m_sparse_regions.end(), [&](const auto& region) { + return region.as_id == p[2] && region.address == begin && end >= begin && region.size == end - begin + 1; + }); + result = Kernel::ResultNotFound; + if (it != m_sparse_regions.end()) { + const auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(p[2]); + if (as && as->DestroyNativeSparseRegion(begin, it->size) == Nvidia::NvResult::Success) { + std::erase_if(m_mappings, [&](const auto& mapping) { + return mapping.as_id == p[2] && mapping.address >= begin && mapping.address <= end; + }); + m_sparse_regions.erase(it); + result = ResultSuccess; + } + } + m_devices->events[notification]->Signal(kernel); + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + Result MapMemory(const u32* packet, u8& selected_kind) { + R_UNLESS(OwnsAddressSpace(packet[2]), Kernel::ResultNotFound); + R_UNLESS(std::find(m_memory.begin(), m_memory.end(), packet[7]) != m_memory.end(), + Kernel::ResultNotFound); + R_UNLESS(packet[15] < m_devices->devices.size() && m_devices->devices[packet[15]], + Kernel::ResultNotFound); + R_UNLESS(packet[10] == 0 && packet[11] <= 1 && packet[14] == 0 && packet[16] <= 1, + Kernel::ResultNotImplemented); + const auto kind = Nvidia::Devices::SelectNativeMemoryKind( + static_cast<u8>(packet[12]), static_cast<u8>(packet[13])); + R_UNLESS(kind, Kernel::ResultNotImplemented); + const u64 begin = ReadPair(packet, 3), end = ReadPair(packet, 5); + R_UNLESS(end >= begin && end != ~u64{0}, Kernel::ResultInvalidSize); + R_UNLESS(m_mappings.size() < 1024, Kernel::ResultOutOfResource); + const auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(packet[2]); + R_UNLESS(as, Kernel::ResultNotFound); + const auto result = as->MapNative4K(packet[7], begin, ReadPair(packet, 8), + end - begin + 1, kind->backend); + R_UNLESS(result == Nvidia::NvResult::Success, Kernel::ResultInvalidCurrentMemory); + m_mappings.push_back({packet[2], begin, end - begin + 1, packet[7]}); + selected_kind = kind->selected; + R_SUCCEED(); + } + void NormalMapGpuMemory4K(HLERequestContext& ctx) { + std::array<u32, 18> words; + std::copy_n(ctx.CommandBuffer(), words.size(), words.begin()); + // These arguments are single-byte TIPC fields; SDK padding is not zeroed. + for (size_t i = 10; i <= 16; ++i) { words[i] &= 0xff; } + const auto* packet = words.data(); + const std::scoped_lock device_lock{m_devices->mutex}; + Result result = Kernel::ResultInvalidCombination; + u8 selected_kind{}; + if (packet[0] == 0xe3 && packet[1] == 16 && packet[17] < m_devices->events.size() && + m_devices->events[packet[17]]) { + result = MapMemory(packet, selected_kind); + m_devices->events[packet[17]]->Signal(kernel); + LOG_DEBUG(Service_AM, "GPU map4K as={}, memory={}, VA={:#x}..{:#x}, permission={}, cache={}, kinds={:#x}/{:#x}, mode={}, sync={}, notification={}, selected={:#x}, result={:#x}", + packet[2], packet[7], ReadPair(packet, 3), ReadPair(packet, 5), packet[10], + packet[11], packet[12], packet[13], packet[14], packet[16], packet[17], selected_kind, result.raw); + } + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(result.raw); + rb.Push<u32>(selected_kind); + } + void NormalModifyGpuMemory4K(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const std::scoped_lock device_lock{m_devices->mutex}; + Result result = Kernel::ResultInvalidCombination; + u8 selected{}; + const u64 begin = ReadPair(packet, 3), end = ReadPair(packet, 5); + const auto requested = static_cast<u8>(packet[7]); + const auto fallback = static_cast<u8>(packet[8]); + if (packet[0] == 0xe4 && packet[1] == 9 && (packet[9] & 0xff) <= 1 && + packet[10] < m_devices->events.size() && m_devices->events[packet[10]]) { + result = Kernel::ResultNotFound; + const auto kind = Nvidia::Devices::SelectNativeMemoryKind(requested, fallback); + const auto owner = std::find_if(m_mappings.begin(), m_mappings.end(), [&](const auto& mapping) { + return mapping.as_id == packet[2] && begin >= mapping.address && + begin - mapping.address < mapping.size && end >= begin && + end - mapping.address < mapping.size; + }); + if (kind && OwnsAddressSpace(packet[2]) && owner != m_mappings.end()) { + const auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(packet[2]); + if (as && as->ModifyNativeKind(begin, end - begin + 1, requested, fallback) == + Nvidia::NvResult::Success) { + selected = kind->selected; + result = ResultSuccess; + } + } + m_devices->events[packet[10]]->Signal(kernel); + } + LOG_DEBUG(Service_AM, "GPU modify4K as={}, VA={:#x}..{:#x}, kinds={:#x}/{:#x}, sync={}, notification={}, selected={}, result={:#x}", + packet[2], begin, end, requested, fallback, packet[9], packet[10], selected, result.raw); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(result.raw); + rb.Push<u32>(selected); + } + void NormalUnmapGpuMemory4K(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const std::scoped_lock device_lock{m_devices->mutex}; + Result result = Kernel::ResultInvalidCombination; + u64 unmapped{}; + if (packet[0] == 0xe5 && packet[1] == 7 && (packet[7] & 0xff) <= 1 && + packet[8] < m_devices->events.size() && m_devices->events[packet[8]]) { + const u64 begin = ReadPair(packet, 3), end = ReadPair(packet, 5); + const auto it = std::find_if(m_mappings.begin(), m_mappings.end(), [&](const auto& mapping) { + return mapping.as_id == packet[2] && mapping.address == begin && + end >= begin && end - begin + 1 == mapping.size; + }); + result = Kernel::ResultNotFound; + if (it != m_mappings.end()) { + const auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(it->as_id); + result = Kernel::ResultInvalidState; + if (as && as->UnmapNative4K(it->address, it->size) == Nvidia::NvResult::Success) { + unmapped = it->size; + m_mappings.erase(it); + result = ResultSuccess; + } + } + m_devices->events[packet[8]]->Signal(kernel); + LOG_DEBUG(Service_AM, "GPU unmap4K as={}, VA={:#x}..{:#x}, bytes={:#x}, result={:#x}", + packet[2], begin, end, unmapped, result.raw); + } + IPC::ResponseBuilder rb{ctx, 4}; + rb.Push(result.raw); + rb.Push(unmapped); + } + Result RegisterMemory(u64 begin, u64 end, u32 flags, u32 device, u32& id) { + R_UNLESS(flags == 0, Kernel::ResultNotImplemented); + R_UNLESS(device < m_devices->devices.size(), Kernel::ResultNotFound); + const std::scoped_lock device_lock{m_devices->mutex}; + R_UNLESS(m_devices->devices[device], Kernel::ResultNotFound); + R_UNLESS(end >= begin && end != ~u64{0}, Kernel::ResultInvalidSize); + const u64 size = end - begin + 1; + R_UNLESS(Common::IsAligned(begin, Kernel::PageSize), Kernel::ResultInvalidAddress); + R_UNLESS(Common::IsAligned(size, Kernel::PageSize), Kernel::ResultInvalidSize); + R_UNLESS(m_memory.size() < 1024, Kernel::ResultOutOfResource); + auto& page_table = m_process->GetPageTable(); + R_UNLESS(page_table.Contains(begin, size), Kernel::ResultInvalidCurrentMemory); + for (u64 address = begin; address <= end;) { + Kernel::KMemoryInfo info{}; + Kernel::Svc::PageInfo page{}; + R_TRY(page_table.QueryInfo(&info, &page, address)); + R_UNLESS(True(info.m_attribute & Kernel::KMemoryAttribute::GpuAccessible), + Kernel::ResultInvalidCurrentMemory); + const u64 next = info.m_address + info.m_size; + R_UNLESS(next > address, Kernel::ResultInvalidCurrentMemory); + address = next; + } + if (!m_nv_session) { + m_nv_session = m_module->GetContainer().OpenSession(m_process).id; + } + auto& nvmap = m_module->GetContainer().GetNvMapFile(); + std::shared_ptr<Nvidia::NvCore::NvMap::Handle> handle; + R_UNLESS(nvmap.CreateHandle(size, handle) == Nvidia::NvResult::Success, + Kernel::ResultOutOfResource); + bool is_io{}; + const auto lock_result = page_table.LockForMapDeviceAddressSpace( + &is_io, begin, size, Kernel::KMemoryPermission::None, true, true); + if (lock_result.IsError()) { + nvmap.FreeHandle(handle->id, false); + R_RETURN(lock_result); + } + const auto alloc_result = handle->Alloc({}, Kernel::PageSize, 0, begin, + Nvidia::NvCore::SessionId{*m_nv_session}); + if (alloc_result != Nvidia::NvResult::Success) { + page_table.UnlockForDeviceAddressSpace(begin, size); + nvmap.FreeHandle(handle->id, false); + R_RETURN(Kernel::ResultOutOfResource); + } + id = handle->id; + m_memory.push_back(id); + R_SUCCEED(); + } + Result ReleaseMemory(u32 id, u32 notification) { + const std::scoped_lock device_lock{m_devices->mutex}; + R_UNLESS(notification < m_devices->events.size() && m_devices->events[notification], + Kernel::ResultInvalidCombination); + const auto it = std::find(m_memory.begin(), m_memory.end(), id); + R_UNLESS(it != m_memory.end(), Kernel::ResultNotFound); + auto& nvmap = m_module->GetContainer().GetNvMapFile(); + auto handle = nvmap.GetHandle(id); + R_UNLESS(handle, Kernel::ResultNotFound); + { + const std::scoped_lock handle_lock{handle->mutex}; + // No asynchronous GPU work is implemented yet. Never report release + // completion for memory still retained or pinned by another user. + R_UNLESS(handle->pins == 0 && handle->internal_dupes == 0 && handle->dupes == 1, + Kernel::ResultBusy); + } + // FreeHandle uses the final shared reference to decide when CPU memory + // may be unshared. Drop this temporary inspection reference first. + handle.reset(); + const auto info = nvmap.FreeHandle(id, false); + R_UNLESS(info && info->can_unlock, Kernel::ResultInvalidState); + m_memory.erase(it); + R_TRY(m_process->GetPageTable().UnlockForDeviceAddressSpace(info->address, info->size)); + m_devices->events[notification]->Signal(kernel); + R_SUCCEED(); + } + void DestroyGpuAccessibleMemory(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const auto result = packet[0] == 0x76 && packet[1] == 2 + ? ReleaseMemory(packet[2], packet[3]) : Kernel::ResultInvalidCombination; + LOG_DEBUG(Service_AM, "GPU memory release id={}, notification={}, result={:#x}", + packet[2], packet[3], result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void CreateGpuAccessibleMemory(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + u32 id{}; + Result result = Kernel::ResultInvalidCombination; + if (const auto request = DecodeGpuMemoryRegistration(std::span<const u32>{packet, 8})) { + result = RegisterMemory(request->begin, request->end, request->flags, request->device, id); + LOG_DEBUG(Service_AM, "GPU memory {:#x}..{:#x}, device={}, raw_device={:#x}, flags={}, id={}, result={:#x}", + request->begin, request->end, request->device, packet[7], request->flags, id, result.raw); + } + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(result.raw); + rb.Push(id); + } + std::shared_ptr<GpuDeviceRegistry> m_devices; + std::optional<size_t> m_nv_session; + std::vector<u32> m_memory; + std::shared_ptr<Nvidia::Module> m_module; + Kernel::KProcess* const m_process; +}; +} // namespace + +IGpuSession::IGpuSession(Core::System& system, std::shared_ptr<Nvidia::Module> module, + Kernel::KProcess* process, bool legacy_protocol) + : ServiceFramework{system, "IGpuSession"}, + m_process{process}, m_module{std::move(module)}, m_legacy_protocol{legacy_protocol} { + m_process->Open(kernel); + m_devices = std::make_shared<GpuDeviceRegistry>(system, m_module, m_process); + static const FunctionInfo functions[] = { + {11, &IGpuSession::GetNotificationRange, "GetNotificationRange"}, + {12, &IGpuSession::GetNotificationEvent, "GetNotificationEvent"}, + {21, &IGpuSession::RequestWorker, "RequestWorker"}, + {22, &IGpuSession::AcquireWorker, "AcquireWorker"}, + {33, &IGpuSession::CreateAddressSpace, "CreateAddressSpace"}, + {41, &IGpuSession::CreateGraphicsDevice, "CreateGraphicsDevice"}, + {42, &IGpuSession::DestroyGraphicsDevice, "DestroyGraphicsDevice"}, + {201, &IGpuSession::GetAddressSpacePartition, "GetLegacyGsvPartition"}, + {202, &IGpuSession::GetAddressSpacePartition, "GetGsvPartition"}, + {203, &IGpuSession::GetAddressSpacePartition, "GetUserPartition"}, + }; + RegisterHandlersTipc(functions); +} + +IGpuSession::~IGpuSession() { + if (m_worker) { m_worker->Close(kernel); } + m_process->Close(kernel); +} + +Result IGpuSession::HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) { + if (!ctx.IsTipc() || handlers_tipc.contains(ctx.GetCommand())) { + return ServiceFrameworkBase::HandleSyncRequest(session, ctx); + } + const auto guard = LockService(); + InvokeRequestTipc(ctx); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(Kernel::ResultNotImplemented.raw); + return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess; +} + +Result IGpuSession::RegisterAddressSpace(u32 id) { + const auto guard = LockService(); + const std::scoped_lock device_lock{m_devices->mutex}; + for (size_t i = 0; i < m_devices->address_spaces.size(); ++i) { + if (m_devices->address_spaces[i] && static_cast<u32>(*m_devices->address_spaces[i]) == id) { + m_devices->address_space_registered[i] = true; + R_SUCCEED(); + } + } + R_THROW(Kernel::ResultNotFound); +} + +void IGpuSession::GetAddressSpacePartition(HLERequestContext& ctx) { + const std::scoped_lock device_lock{m_devices->mutex}; + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + std::array<u64, 9> values{}; + const bool legacy_user = m_legacy_protocol && (ctx.GetCommand() == 201 || ctx.GetCommand() == 202); + const bool user = legacy_user || ctx.GetCommand() == 203; + if (!user) { + values[0] = ~u64{0}; // No supervisor-owned partition is reserved by this adapter. + } + if (packet[0] == 16 + ctx.GetCommand() && packet[1] == 1) { + const u32 id = packet[2]; + const bool exists = std::any_of(m_devices->address_spaces.begin(), m_devices->address_spaces.end(), + [id](const auto& fd) { return fd && static_cast<u32>(*fd) == id; }); + result = exists ? ResultSuccess : Kernel::ResultNotFound; + if (exists && user) { + std::array<u8, 64> input{}; + std::array<u8, 64> output{}; + const u32_le buffer_size{48}; + std::memcpy(input.data() + 8, &buffer_size, sizeof(buffer_size)); + Nvidia::Ioctl command{}; + command.raw = 0xc0404108; + if (m_module->Ioctl1(static_cast<s32>(id), command, input, output) == + Nvidia::NvResult::Success) { + std::array<Nvidia::Devices::nvhost_as_gpu::VaRegion, 2> regions; + std::memcpy(regions.data(), output.data() + 16, sizeof(regions)); + const u64 end = regions[1].offset + regions[1].pages * regions[1].page_size - 1; + if (legacy_user) { + const auto& region = regions[ctx.GetCommand() == 201 ? 0 : 1]; + values[0] = region.offset; + values[1] = region.offset + region.pages * region.page_size - 1; + } else { + values[0] = 1; + values[1] = regions[0].offset; + values[2] = end; + } + } else { + result = Kernel::ResultInvalidState; + } + } + LOG_DEBUG(Service_AM, "GPU partition command={}, id={}, begin={:#x}, end={:#x}, result={:#x}", + ctx.GetCommand(), id, user ? values[1] : values[0], + user ? values[2] : values[1], result.raw); + } + IPC::ResponseBuilder rb{ctx, user && !legacy_user ? 20U : 6U}; + rb.Push(result.raw); + for (size_t i = 0; i < (user && !legacy_user ? values.size() : size_t{2}); ++i) { + rb.Push(values[i]); + } +} + +void IGpuSession::CreateAddressSpace(HLERequestContext& ctx) { + const std::scoped_lock device_lock{m_devices->mutex}; + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + u32 id{}; + if (packet[0] == 0x31 && packet[1] == 1 && packet[2] == 0) { + result = Kernel::ResultOutOfResource; + for (auto& slot : m_devices->address_spaces) { + if (slot) { + continue; + } + if (!m_devices->nv_session) { + m_devices->nv_session = m_module->GetContainer().OpenSession(m_process).id; + } + const auto fd = m_module->Open("/dev/nvhost-as-gpu", + Nvidia::NvCore::SessionId{*m_devices->nv_session}); + if (fd == Nvidia::INVALID_NVDRV_FD) { + break; + } + std::array<u8, 40> input{}; + std::array<u8, 40> output{}; + const u32_le big_page_size{0x20000}; + std::memcpy(input.data() + 8, &big_page_size, sizeof(big_page_size)); + Nvidia::Ioctl command{}; + command.raw = 0x40284109; + const auto nv_result = m_module->Ioctl1(fd, command, input, output); + if (nv_result == Nvidia::NvResult::Success) { + slot = fd; + id = fd; + result = ResultSuccess; + } else { + m_module->Close(fd); + result = Kernel::ResultInvalidState; + } + break; + } + } + LOG_DEBUG(Service_AM, "GPU address space id={}, result={:#x}", id, result.raw); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(result.raw); + rb.Push(id); +} + +void IGpuSession::CreateGraphicsDevice(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + u32 id{}; + if (packet[0] == 0x39 && packet[1] == 6) { + const u64 name = u64{packet[2]} | (u64{packet[3]} << 32); + const u64 flags = u64{packet[4]} | (u64{packet[5]} << 32); + const u64 debug_label = u64{packet[6]} | (u64{packet[7]} << 32); + result = flags == 0 ? Kernel::ResultOutOfResource : Kernel::ResultNotImplemented; + if (flags == 0) { + const std::scoped_lock device_lock{m_devices->mutex}; + for (u32 candidate = 1; candidate < m_devices->devices.size(); ++candidate) { + if (!m_devices->devices[candidate]) { + id = candidate; + m_devices->devices[id] = GpuDeviceRegistry::Device{name, debug_label}; + result = ResultSuccess; + break; + } + } + } + LOG_DEBUG(Service_AM, "graphics device name={:#x}, flags={:#x}, id={}, result={:#x}", + name, flags, id, result.raw); + } + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(result.raw); + rb.Push(id); +} + +void IGpuSession::DestroyGraphicsDevice(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x3a && packet[1] == 1) { + const u8 id = static_cast<u8>(packet[2]); + const std::scoped_lock device_lock{m_devices->mutex}; + result = m_devices->devices[id] ? ResultSuccess : Kernel::ResultNotFound; + m_devices->devices[id].reset(); + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGpuSession::GetNotificationRange(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const Result result = packet[0] == 0x1b && packet[1] == 0 + ? ResultSuccess : Kernel::ResultInvalidCombination; + IPC::ResponseBuilder rb{ctx, 4}; + rb.Push(result.raw); + rb.Push<u32>(0); + rb.Push<u32>(static_cast<u32>(m_devices->events.size() - 1)); + LOG_DEBUG(Service_AM, "GPU notification range 0..{}", m_devices->events.size() - 1); +} + +void IGpuSession::GetNotificationEvent(HLERequestContext& ctx) { + const std::scoped_lock device_lock{m_devices->mutex}; + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + Kernel::KReadableEvent* readable{}; + if (packet[0] == 0x1c && packet[1] == 1 && packet[2] < m_devices->events.size()) { + const u32 index = packet[2]; + if (!m_devices->events[index]) { m_devices->events[index] = m_devices->context.CreateEvent("GpuNotification"); } + if (m_devices->events[index]) { + readable = &m_devices->events[index]->GetReadableEvent(); + readable->Open(kernel); + result = ResultSuccess; + } else { result = Kernel::ResultOutOfResource; } + LOG_DEBUG(Service_AM, "GPU notification event index={}, result={:#x}", index, result.raw); + } + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(result.raw); + ctx.AddMoveObject(readable); +} + +void IGpuSession::RequestWorker(HLERequestContext& ctx) { + const std::scoped_lock device_lock{m_devices->mutex}; + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x25 && packet[1] == 1 && packet[2] < m_devices->events.size()) { + const u32 index = packet[2]; + if (!m_devices->events[index]) { + result = Kernel::ResultInvalidState; + } else { + result = ResultSuccess; + if (!m_worker) { + result = CreateGraphicsSession(system, ctx, + std::make_shared<IGpuWorker>(system, m_module, m_process, m_devices), &m_worker); + } + if (result.IsSuccess()) { m_devices->events[index]->Signal(kernel); } + } + LOG_DEBUG(Service_AM, "GPU worker request index={}, result={:#x}", index, result.raw); + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGpuSession::AcquireWorker(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + Kernel::KClientSession* client{}; + if (packet[0] == 0x26 && packet[1] == 0) { + if (m_worker) { + client = m_worker; + client->Open(kernel); + result = ResultSuccess; + } else { result = Kernel::ResultInvalidState; } + } + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(result.raw); + ctx.AddMoveObject(client); + LOG_DEBUG(Service_AM, "GPU worker acquisition result={:#x}", result.raw); +} +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/gpu_session.h b/src/core/hle/service/am/service/gpu_session.h new file mode 100644 index 0000000000..e8f580bef1 --- /dev/null +++ b/src/core/hle/service/am/service/gpu_session.h @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <array> +#include <optional> +#include "core/hle/service/service.h" +#include "core/hle/service/kernel_helpers.h" + +namespace Kernel { +class KClientSession; +class KProcess; +class KEvent; +} +namespace Service::Nvidia { class Module; } +namespace Service::AM { + +struct GpuDeviceRegistry; + +class IGpuSession final : public ServiceFramework<IGpuSession> { +public: + IGpuSession(Core::System& system, std::shared_ptr<Nvidia::Module> module, + Kernel::KProcess* process, bool legacy_protocol = false); + ~IGpuSession() override; + Result RegisterAddressSpace(u32 id); + Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override; + +private: + void GetAddressSpacePartition(HLERequestContext& ctx); + void CreateAddressSpace(HLERequestContext& ctx); + void CreateGraphicsDevice(HLERequestContext& ctx); + void DestroyGraphicsDevice(HLERequestContext& ctx); + void GetNotificationRange(HLERequestContext& ctx); + void GetNotificationEvent(HLERequestContext& ctx); + void RequestWorker(HLERequestContext& ctx); + void AcquireWorker(HLERequestContext& ctx); + + std::shared_ptr<GpuDeviceRegistry> m_devices; + Kernel::KClientSession* m_worker{}; + Kernel::KProcess* const m_process; + std::shared_ptr<Nvidia::Module> m_module; + const bool m_legacy_protocol; +}; +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/graphics_scaling_parameters_functions.cpp b/src/core/hle/service/am/service/graphics_scaling_parameters_functions.cpp new file mode 100644 index 0000000000..931c81850d --- /dev/null +++ b/src/core/hle/service/am/service/graphics_scaling_parameters_functions.cpp @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "core/hle/service/am/applet.h" +#include "core/hle/service/am/service/graphics_scaling_parameters_functions.h" +#include "core/hle/service/cmif_serialization.h" + +namespace Service::AM { + +IGraphicsScalingParametersFunctions::IGraphicsScalingParametersFunctions( + Core::System& system, std::shared_ptr<Applet> applet) + : ServiceFramework{system, "IGraphicsScalingParametersFunctions"}, m_applet{std::move(applet)} { + static const FunctionInfo functions[] = { + {10, D<&IGraphicsScalingParametersFunctions::GetGraphicsScalingParameters>, + "GetGraphicsScalingParameters"}, + {1000, nullptr, "SetSupportingGraphicsScalingParametersForDebug"}, + {1010, nullptr, "SetGpuQuotaForDebug"}, + {1020, nullptr, "ClearGpuQuotaForDebug"}, + {1030, nullptr, "SetScreenRatioForDebug"}, + {1040, nullptr, "ClearScreenRatioForDebug"}, + {1050, D<&IGraphicsScalingParametersFunctions::SetGraphicsScaleChangedNotificationEnabled>, + "SetGraphicsScaleChangedNotificationEnabled"}, + }; + RegisterHandlers(functions); +} + +IGraphicsScalingParametersFunctions::~IGraphicsScalingParametersFunctions() = default; + +Result IGraphicsScalingParametersFunctions::GetGraphicsScalingParameters( + Out<GraphicsScalingParameters> out_parameters) { + // Prototype policy: lowest SDK scale (10 maps to public scale 0), unit screen ratio. + // The ratio field interpretation is inferred from the adjacent debug API. + *out_parameters = {10, 1, 1}; + LOG_DEBUG(Service_AM, "called, prototype scale=10, screen ratio=1/1"); + R_SUCCEED(); +} + +Result IGraphicsScalingParametersFunctions::SetGraphicsScaleChangedNotificationEnabled(bool enabled) { + LOG_DEBUG(Service_AM, "called, enabled={}", enabled); + std::scoped_lock lk{m_applet->lock}; + m_applet->graphics_scale_changed_notification_enabled = enabled; + R_SUCCEED(); +} + +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/graphics_scaling_parameters_functions.h b/src/core/hle/service/am/service/graphics_scaling_parameters_functions.h new file mode 100644 index 0000000000..7c92cb53f4 --- /dev/null +++ b/src/core/hle/service/am/service/graphics_scaling_parameters_functions.h @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "core/hle/service/cmif_types.h" +#include "core/hle/service/service.h" + +namespace Service::AM { + +struct Applet; + +struct GraphicsScalingParameters { + u32 graphics_scale; + u32 screen_ratio_numerator; + u32 screen_ratio_denominator; +}; +static_assert(sizeof(GraphicsScalingParameters) == 12); + +class IGraphicsScalingParametersFunctions final + : public ServiceFramework<IGraphicsScalingParametersFunctions> { +public: + explicit IGraphicsScalingParametersFunctions(Core::System& system, + std::shared_ptr<Applet> applet); + ~IGraphicsScalingParametersFunctions() override; + +private: + Result GetGraphicsScalingParameters(Out<GraphicsScalingParameters> out_parameters); + Result SetGraphicsScaleChangedNotificationEnabled(bool enabled); + + const std::shared_ptr<Applet> m_applet; +}; + +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/graphics_session.cpp b/src/core/hle/service/am/service/graphics_session.cpp new file mode 100644 index 0000000000..902e7f9706 --- /dev/null +++ b/src/core/hle/service/am/service/graphics_session.cpp @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "core/core.h" +#include "core/hle/kernel/k_client_session.h" +#include "core/hle/kernel/k_scoped_resource_reservation.h" +#include "core/hle/kernel/k_session.h" +#include "core/hle/kernel/svc_results.h" +#include "core/hle/service/am/service/graphics_session.h" +#include "core/hle/service/server_manager.h" + +namespace Service::AM { +Result CreateGraphicsSession(Core::System& system, HLERequestContext& ctx, + std::shared_ptr<SessionRequestHandler> handler, + Kernel::KClientSession** out_client) { + auto& kernel = system.Kernel(); + Kernel::KScopedResourceReservation reservation( + kernel, Kernel::GetCurrentProcessPointer(kernel), Kernel::LimitableResource::SessionCountMax); + R_UNLESS(reservation.Succeeded(), Kernel::ResultLimitReached); + auto* session = Kernel::KSession::Create(kernel); + R_UNLESS(session != nullptr, Kernel::ResultOutOfResource); + session->Initialize(kernel, nullptr, 0); + reservation.Commit(); + Kernel::KSession::Register(kernel, session); + auto& server = ctx.GetManager()->GetServerManager(); + auto manager = std::make_shared<SessionRequestManager>(kernel, server); + manager->SetSessionHandler(std::move(handler)); + R_TRY(server.RegisterSession(&session->GetServerSession(), std::move(manager))); + *out_client = &session->GetClientSession(); + R_SUCCEED(); +} + +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/graphics_session.h b/src/core/hle/service/am/service/graphics_session.h new file mode 100644 index 0000000000..698b5ee8ee --- /dev/null +++ b/src/core/hle/service/am/service/graphics_session.h @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <memory> +#include "core/hle/result.h" + +namespace Core { class System; } +namespace Kernel { class KClientSession; } +namespace Service { +class HLERequestContext; +class SessionRequestHandler; +} +namespace Service::AM { +Result CreateGraphicsSession(Core::System& system, HLERequestContext& ctx, + std::shared_ptr<SessionRequestHandler> handler, + Kernel::KClientSession** out_client); +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/graphics_supervisor.cpp b/src/core/hle/service/am/service/graphics_supervisor.cpp new file mode 100644 index 0000000000..7759f575e7 --- /dev/null +++ b/src/core/hle/service/am/service/graphics_supervisor.cpp @@ -0,0 +1,1612 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <algorithm> +#include <atomic> +#include <bit> +#include <array> +#include "common/alignment.h" +#include "core/hle/service/nvdrv/core/nvmap.h" +#include "core/core.h" +#include "core/core_timing.h" +#include "core/hle/service/nvnflinger/buffer_queue_producer.h" +#include "core/hle/service/nvnflinger/ui/graphic_buffer.h" +#include "core/hle/kernel/k_client_session.h" +#include "core/hle/kernel/k_scoped_resource_reservation.h" +#include "core/hle/kernel/k_session.h" +#include "core/hle/kernel/k_readable_event.h" +#include "core/hle/kernel/k_transfer_memory.h" +#include "core/hle/kernel/k_event.h" +#include "core/hle/kernel/k_process.h" +#include "core/hle/kernel/svc_results.h" +#include "core/hle/service/am/service/graphics_supervisor.h" +#include "core/hle/service/am/service/graphics_session.h" +#include "core/hle/service/am/service/gpu_session.h" +#include "core/hle/service/am/service/graphics_supervisor_protocol.h" +#include "core/hle/service/am/service/graphics_syncpoint_cache.h" +#include "core/hle/service/am/service/native_image_layout.h" +#include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/nvdrv/nvdrv_interface.h" +#include "core/hle/service/nvdrv/core/container.h" +#include "core/hle/service/nvdrv/core/syncpoint_manager.h" +#include "core/hle/service/server_manager.h" +#include "core/hle/service/sm/sm.h" +#include "core/hle/service/vi/application_display_service.h" +#include "core/hle/service/vi/container.h" +#include "core/hle/service/vi/manager_root_service.h" +#include "core/hle/service/vi/vi_types.h" + +namespace Service::AM { +struct NativeImageRegistry { + struct Image { + u32 id; + u32 memory_id; + std::array<u32, 16> preference; + u64 size; + bool window_pool{}; + }; + NativeImageRegistry(Kernel::KernelCore& kernel_, Kernel::KProcess* process_) + : kernel{kernel_}, process{process_} { + process->Open(kernel); + } + ~NativeImageRegistry() { + if (module) { + for (const auto& image : images) { + ReleaseMemory(image.memory_id); + } + } + if (memory_session) { + module->GetContainer().CloseSession(*memory_session); + } + process->Close(kernel); + } + void ReleaseMemory(u32 id) { + const auto info = module->GetContainer().GetNvMapFile().FreeHandle(id, true); + if (info && info->can_unlock && info->address != 0) { + process->GetPageTable().UnlockForDeviceAddressSpace(info->address, info->size); + } + } + Kernel::KernelCore& kernel; + Kernel::KProcess* process; + std::mutex mutex; + std::shared_ptr<Nvidia::Module> module; + std::optional<Nvidia::NvCore::SessionId> memory_session; + std::vector<Image> images; +}; +using NativeImage = NativeImageRegistry::Image; + + +namespace { + +// TIPC payloads follow a 32-bit result, so these handlers push result.raw. + +class ISyncpointSupervisor final : public ServiceFramework<ISyncpointSupervisor> { +public: + ISyncpointSupervisor(Core::System& system, std::shared_ptr<Nvidia::Module> module) + : ServiceFramework{system, "ISyncpointSupervisor"}, m_module{std::move(module)}, + m_context{system, "NativeSyncpoint"} { + static const FunctionInfo functions[] = { + {0, &ISyncpointSupervisor::ExchangeClientToken, "ExchangeClientToken"}, + {1, &ISyncpointSupervisor::GetMaximumEventCount, "GetMaximumEventCount"}, + {21, &ISyncpointSupervisor::SetEventCount, "SetEventCount"}, + {22, &ISyncpointSupervisor::GetEvent, "GetEvent"}, + {201, &ISyncpointSupervisor::ArmEvent, "ArmEvent"}, + {202, &ISyncpointSupervisor::CancelEvent, "CancelEvent"}, + {103, &ISyncpointSupervisor::QueryCounter, "QueryCounter"}, + }; + RegisterHandlersTipc(functions); + } + + ~ISyncpointSupervisor() override { + if (m_poll) system.CoreTiming().UnscheduleEvent(m_poll); + for (auto& event : m_events) m_context.CloseEvent(event.event); + } + + Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override { + if (!ctx.IsTipc() || handlers_tipc.contains(ctx.GetCommand())) { + return ServiceFrameworkBase::HandleSyncRequest(session, ctx); + } + const auto guard = LockService(); + LOG_DEBUG(Service_AM, "syncpoint unsupported command={}, words={:08x}", ctx.GetCommand(), + fmt::join(std::span<const u32>(ctx.CommandBuffer() + 2, std::min<u32>(ctx.CommandBuffer()[1], 32)), ",")); + InvokeRequestTipc(ctx); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(Kernel::ResultNotImplemented.raw); + return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess; + } + +private: + static constexpr u32 MaximumEventCount = 63; + + void ExchangeClientToken(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const Result result = packet[0] == 0x10 && packet[1] == 2 + ? ResultSuccess : Kernel::ResultInvalidCombination; + const u64 token = u64{packet[2]} | (u64{packet[3]} << 32); + IPC::ResponseBuilder rb{ctx, 4}; + rb.Push(result.raw); + rb.Push(token); + LOG_DEBUG(Service_AM, "syncpoint token={:#x}, result={:#x}", token, result.raw); + } + void GetMaximumEventCount(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const Result result = packet[0] == 0x11 && packet[1] == 0 + ? ResultSuccess : Kernel::ResultInvalidCombination; + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(result.raw); + rb.Push(MaximumEventCount); + LOG_DEBUG(Service_AM, "syncpoint maximum events={}", MaximumEventCount); + } + + void SetEventCount(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x25 && packet[1] == 1 && packet[2] <= MaximumEventCount) { + m_event_count = packet[2]; + result = ResultSuccess; + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + LOG_DEBUG(Service_AM, "syncpoint event count={}, result={:#x}", m_event_count, result.raw); + } + + void GetEvent(HLERequestContext& ctx) { + const auto* p = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + Kernel::KReadableEvent* readable{}; + if (p[0] == 0x26 && p[1] == 1 && p[2] < m_event_count) { + auto& event = m_events[p[2]]; + if (!event.event) event.event = m_context.CreateEvent("SyncpointCompletion"); + result = event.event ? ResultSuccess : Kernel::ResultOutOfResource; + if (event.event) { + readable = &event.event->GetReadableEvent(); + readable->Open(kernel); + } + } + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(result.raw); + ctx.AddMoveObject(readable); + } + void RefreshEvents() { + auto& manager = m_module->GetContainer().GetSyncpointManager(); + for (auto& event : m_events) { + if (!event.pending) continue; + const auto current = manager.ReadAllocatedSyncpoint(event.id); + if (current && event.last_observed != *current) { + event.last_observed = *current; + LOG_INFO(Service_AM, "Syncpoint wait progress: id={}, current={}, target={}", event.id, *current, event.threshold); + } + if (current && static_cast<s32>(*current - event.threshold) >= 0) { + event.pending = false; + event.event->Signal(kernel); + } + } + } + void ArmEvent(HLERequestContext& ctx) { + const auto* p = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + const u32 id = ~p[3]; + if (p[0] == 0xd9 && p[1] == 5 && p[2] < m_event_count && + p[5] == 0 && p[6] == 0 && + m_module->GetContainer().GetSyncpointManager().ReadAllocatedSyncpoint(id)) { + auto& event = m_events[p[2]]; + if (!event.event) event.event = m_context.CreateEvent("SyncpointCompletion"); + result = event.event ? ResultSuccess : Kernel::ResultOutOfResource; + if (event.event) { + event.event->Clear(kernel); + event.id = id; + event.threshold = p[4]; + event.last_observed = ~u32{0}; + LOG_INFO(Service_AM, "Arm syncpoint wait: slot={}, id={}, target={}", p[2], id, event.threshold); + event.pending = true; + RefreshEvents(); + if (!m_poll) { + m_poll = Core::Timing::CreateEvent("NativeSyncpointCompletion", + [this](s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> { + const auto guard = LockService(); + RefreshEvents(); + return std::nullopt; + }); + system.CoreTiming().ScheduleLoopingEvent(std::chrono::milliseconds{1}, + std::chrono::milliseconds{1}, m_poll); + } + } + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + LOG_DEBUG(Service_AM, "syncpoint arm event={}, id={}, threshold={}, result={:#x}", + p[2], id, p[4], result.raw); + } + void CancelEvent(HLERequestContext& ctx) { + const auto* p = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + if (p[0] == 0xda && p[1] == 1 && p[2] < m_event_count) { + auto& event = m_events[p[2]]; + event.pending = false; + // The native SDK waits for this event after cancelling a timed wait + // before it releases the event slot. Cancellation acknowledges that + // the wait is retired; it does not advance the syncpoint counter. + if (event.event) event.event->Signal(kernel); + result = ResultSuccess; + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + + void QueryCounter(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + u64 value{}; + if (packet[0] == 0x77 && packet[1] == 1) { + const u32 id = ~packet[2]; + auto current = m_module->GetContainer().GetSyncpointManager().ReadAllocatedSyncpoint(id); + result = current ? ResultSuccess : Kernel::ResultNotFound; + value = current.value_or(0); + LOG_DEBUG(Service_AM, "query syncpoint id={}, value={}, result={:#x}", id, value, result.raw); + } + IPC::ResponseBuilder rb{ctx, 5}; + rb.Push(result.raw); + rb.Push(value); + rb.Push<u32>(0); // Existing NVDRV counters use 32-bit modular comparisons. + } + + std::shared_ptr<Nvidia::Module> m_module; + u32 m_event_count{}; + struct CompletionEvent { + Kernel::KEvent* event{}; + u32 id{}; + u32 threshold{}; + u32 last_observed = ~u32{0}; + bool pending{}; + }; + KernelHelpers::ServiceContext m_context; + std::array<CompletionEvent, MaximumEventCount> m_events{}; + std::shared_ptr<Core::Timing::EventType> m_poll; +}; + +class IVi2Session final : public ServiceFramework<IVi2Session> { + struct ClientState { + u64 token{}; + u32 slots{}; + std::optional<u64> display; + std::optional<u64> layer; + u64 available{0x7fff}; + u32 retire_notification{}; + u64 output_encoding{0x14}; + u32 available_notification{}; + std::shared_ptr<android::BufferQueueProducer> producer; + std::array<std::optional<s32>, 15> queued{}; + std::optional<size_t> player_display; + }; + +public: + explicit IVi2Session(Core::System& system, u64 aruid, std::shared_ptr<NativeImageRegistry> images) : ServiceFramework{system, "IVi2Session"}, m_images{std::move(images)}, m_aruid{aruid}, m_context{system, "Vi2Session"} { + static const FunctionInfo functions[] = { + {1, &IVi2Session::GetSessionLimit, "GetNotificationLimit"}, + {11, &IVi2Session::GetNotificationEvent, "GetNotificationEvent"}, + {13, &IVi2Session::ReleaseNotificationEvent, "ReleaseNotificationEvent"}, + {21, &IVi2Session::GetNotifications, "GetNotifications"}, + {31, &IVi2Session::DuplicateVsyncEvent, "GetDisplayEvent"}, + {32, &IVi2Session::DuplicateDisplayEvent, "DuplicateDisplayEvent"}, + {3, &IVi2Session::GetSessionLimit, "GetSessionLimit"}, + {5, &IVi2Session::GetSessionLimit, "GetRequestGroupLimit"}, + {51, &IVi2Session::ConfigureRequestGroups, "ConfigureRequestGroups"}, + {101, &IVi2Session::OpenDisplay, "OpenDisplay"}, + {102, &IVi2Session::CloseDisplay, "CloseDisplay"}, + {111, &IVi2Session::OpenEndpoint, "OpenEndpoint"}, + {112, &IVi2Session::CloseEndpoint, "CloseEndpoint"}, + {201, &IVi2Session::RegisterClientState, "RegisterClientState"}, + {202, &IVi2Session::UnregisterClientState, "UnregisterClientState"}, + {212, &IVi2Session::GetClientAvailability, "GetClientAvailability"}, + {229, &IVi2Session::PresentClientBuffer, "PresentClientBuffer"}, + {228, &IVi2Session::CancelClientBuffer, "CancelClientBuffer"}, + {351, &IVi2Session::SetClientOutputEncoding, "SetClientOutputEncoding"}, + {261, &IVi2Session::SetDisplayCapability, "SetDisplayCapability261"}, + {265, &IVi2Session::SetDisplayCapability, "SetDisplayCapability265"}, + {267, &IVi2Session::SetDisplayCapability, "SetDisplayCapability267"}, + {271, &IVi2Session::SetDisplayCapability, "SetDisplayCapability271"}, + {251, &IVi2Session::BindClientDisplay, "BindClientDisplay"}, + {252, &IVi2Session::UnbindClientDisplay, "UnbindClientDisplay"}, + }; + RegisterHandlersTipc(functions); + } + + ~IVi2Session() override { + if (m_release_poll) system.CoreTiming().UnscheduleEvent(m_release_poll); + if (m_display_service) { + for (auto& state : m_client_states) { + if (state.layer) m_display_service->GetContainer()->DestroyManagedLayer(*state.layer); + } + } + if (m_default_display) { m_display_service->CloseDisplay(*m_default_display); } + for (auto* event : m_display_change_events) m_context.CloseEvent(event); + m_context.CloseEvent(m_event); + } + + Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override { + if (!ctx.IsTipc() || handlers_tipc.contains(ctx.GetCommand())) { + return ServiceFrameworkBase::HandleSyncRequest(session, ctx); + } + const auto guard = LockService(); + LOG_DEBUG(Service_AM, "VI2 unsupported command={}, words={:08x}", ctx.GetCommand(), + fmt::join(std::span<const u32>(ctx.CommandBuffer() + 2, std::min<u32>(ctx.CommandBuffer()[1], 32)), ",")); + InvokeRequestTipc(ctx); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(Kernel::ResultNotImplemented.raw); + return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess; + } +private: + void SetDisplayCapability(HLERequestContext& ctx) { + const auto* p = ctx.CommandBuffer(); + const u64 name = u64{p[2]} | (u64{p[3]} << 32); + Result result = Kernel::ResultInvalidCombination; + if (p[0] == ctx.GetCommand() + 16 && p[1] == 3 && p[4] <= 1) { + result = name == 0x00746c7561666564ULL && m_default_display + ? ResultSuccess : Kernel::ResultNotFound; + if (result.IsSuccess()) { + // Per-client display mode capabilities do not change the host video mode. + m_display_capabilities[(ctx.GetCommand() - 261) / 2] = p[4] != 0; + } + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void DuplicateVsyncEvent(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + Kernel::KReadableEvent* readable{}; + if (packet[0] == 0x2f && packet[1] == 1 && packet[2] < 5) { + result = m_display_service && m_default_display + ? ResultSuccess : Kernel::ResultInvalidState; + if (result.IsSuccess() && !m_vsync_event) { + result = m_display_service->GetDisplayVsyncEvent(&m_vsync_event, *m_default_display); + } + if (result.IsSuccess()) { + if (packet[2] == 0) { + readable = m_vsync_event; + } else { + // Fixed host display properties have no change to report until changed. + auto*& event = m_display_change_events[packet[2] - 1]; + if (!event) event = m_context.CreateEvent("DisplayPropertyChange"); + if (event) readable = &event->GetReadableEvent(); + else result = Kernel::ResultOutOfResource; + } + if (readable) readable->Open(kernel); + } + } + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(result.raw); + ctx.AddMoveObject(readable); + LOG_DEBUG(Service_AM, "VI2 vsync event result={:#x}", result.raw); + } + void DuplicateDisplayEvent(HLERequestContext& ctx) { + const auto* p = ctx.CommandBuffer(); + auto source = ctx.GetObjectFromHandle<Kernel::KReadableEvent>(ctx.GetCopyHandle(0)); + const Result result = p[0] == 0x30 && p[1] == 0x80000001 && p[4] <= 1 && source.IsNotNull() + ? ResultSuccess : Kernel::ResultInvalidCombination; + Kernel::KReadableEvent* readable{}; + if (result.IsSuccess()) { + readable = source.GetPointerUnsafe(); + readable->Open(kernel); + } + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(result.raw); + ctx.AddMoveObject(readable); + } + static std::optional<size_t> GetPlayerDisplayIndex(u64 name) { + if ((name & 0xffffffffffffULL) != 0x726579616c70ULL || (name >> 56) != 0) { + return std::nullopt; + } + const u32 player = (name >> 48) & 0xff; + if (player < '1' || player > '3') return std::nullopt; + return player - '1'; + } + Result OpenDefaultDisplay(u64 name, u32 flags) { + if (const auto player = GetPlayerDisplayIndex(name)) { + R_UNLESS(flags < m_notifications.size() * 32, Kernel::ResultInvalidCombination); + R_UNLESS(m_player_display_references[*player] < 1024, Kernel::ResultOutOfResource); + // GameShare's detached player displays retain their own lifetime; + // they do not target the host's default display compositor. + m_player_display_notifications[*player] = flags; + ++m_player_display_references[*player]; + R_SUCCEED(); + } + // The native display token is an eight-byte name; command 101 returns + // only a result. Retain the real VI display ID for later operations. + R_UNLESS(name == 0x00746c7561666564ULL, Kernel::ResultNotFound); + R_UNLESS(flags == 1, Kernel::ResultNotImplemented); + R_UNLESS(m_display_references < 1024, Kernel::ResultOutOfResource); + if (!m_display_service) { + const auto root = system.ServiceManager().GetService<VI::IManagerRootService>("vi:m", true); + R_UNLESS(root, Kernel::ResultInvalidState); + R_TRY(root->GetDisplayService(&m_display_service, VI::Policy::Compositor)); + } + if (!m_default_display) { + u64 id{}; + R_TRY(m_display_service->OpenDefaultDisplay(&id)); + m_default_display = id; + } + m_display_notification = flags; + ++m_display_references; + R_SUCCEED(); + } + void OpenDisplay(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 name = u64{packet[2]} | (u64{packet[3]} << 32); + const auto result = packet[0] == 0x75 && packet[1] == 3 + ? OpenDefaultDisplay(name, packet[4]) : Kernel::ResultInvalidCombination; + LOG_DEBUG(Service_AM, "VI2 open display name={:#x}, flags={:#x}, references={}, result={:#x}", + name, packet[4], m_display_references, result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void CloseDisplay(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + const u64 name = u64{packet[2]} | (u64{packet[3]} << 32); + if (packet[0] == 0x76 && packet[1] == 2 && GetPlayerDisplayIndex(name)) { + const auto player = *GetPlayerDisplayIndex(name); + result = Kernel::ResultNotFound; + if (m_player_display_references[player]) { + if (--m_player_display_references[player] == 0) { + const auto notification = m_player_display_notifications[player]; + m_notifications[notification / 32] |= 1U << (notification % 32); + if (m_event) m_event->Signal(kernel); + } + result = ResultSuccess; + } + } else if (packet[0] == 0x76 && packet[1] == 2 && name == 0x00746c7561666564ULL) { + result = Kernel::ResultNotFound; + if (m_default_display && m_display_references) { + result = ResultSuccess; + if (m_display_references == 1) { + result = m_display_service->CloseDisplay(*m_default_display); + if (result.IsSuccess()) { m_default_display.reset(); } + } + if (result.IsSuccess()) { + --m_display_references; + if (!m_display_references) { + m_notifications[m_display_notification / 32] |= 1U << (m_display_notification % 32); + if (m_event) m_event->Signal(kernel); + LOG_DEBUG(Service_AM, "VI2 display retired notification={}", m_display_notification); + } + } + } + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void RegisterClientState(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 token = u64{packet[2]} | (u64{packet[3]} << 32); + const u32 first = packet[4]; + const u32 second = packet[5]; + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0xd9 && packet[1] == 4 && token != 0 && + first < 32 && second < 32 && first != second) { + const u32 slots = (1U << first) | (1U << second); + const auto existing = std::ranges::find(m_client_states, token, &ClientState::token); + const auto free = std::ranges::find(m_client_states, u64{0}, &ClientState::token); + result = Kernel::ResultInvalidState; + if (existing == m_client_states.end() && !(m_client_slots & slots)) { + result = Kernel::ResultOutOfResource; + if (free != m_client_states.end()) { + // The SDK sends its object address as an opaque identity, not a + // service-accessible buffer. Keep the associated slots session-local. + *free = {token, slots}; + free->retire_notification = first; + free->available_notification = second; + m_client_slots |= slots; + result = ResultSuccess; + } + } + } + LOG_DEBUG(Service_AM, "VI2 register client state token={:#x}, slots={}/{}, result={:#x}", + token, first, second, result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void UnregisterClientState(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 token = u64{packet[2]} | (u64{packet[3]} << 32); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0xda && packet[1] == 2 && token != 0) { + const auto state = std::ranges::find(m_client_states, token, &ClientState::token); + result = Kernel::ResultNotFound; + if (state != m_client_states.end()) { + if (m_default_endpoint == token) { m_default_endpoint.reset(); } + if (state->layer) m_display_service->GetContainer()->DestroyManagedLayer(*state->layer); + m_notifications[state->retire_notification / 32] |= 1U << (state->retire_notification % 32); + if (m_event) m_event->Signal(kernel); + LOG_DEBUG(Service_AM, "VI2 client retired token={:#x}, notification={}", token, state->retire_notification); + m_client_slots &= ~state->slots; + *state = {}; + result = ResultSuccess; + } + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void GetClientAvailability(HLERequestContext& ctx) { + RefreshReleasedBuffers(); + const auto* packet = ctx.CommandBuffer(); + const u64 token = u64{packet[2]} | (u64{packet[3]} << 32); + Result result = Kernel::ResultInvalidCombination; + u64 id{}; + if (packet[0] == 0xe4 && packet[1] == 2 && token) { + auto state = std::ranges::find(m_client_states, token, &ClientState::token); + result = Kernel::ResultNotFound; + if (state != m_client_states.end() && m_default_display) { + result = ResultSuccess; + if (state->player_display) { + id = 0; + } else if (!state->layer) { + result = m_display_service->GetContainer()->CreateManagedLayer(&id, *m_default_display, m_aruid); + if (result.IsSuccess()) state->layer = id; + } else { + id = *state->layer; + } + } + } + const auto state = std::ranges::find(m_client_states, token, &ClientState::token); + const u64 available = result.IsSuccess() ? state->available : 0; + LOG_DEBUG(Service_AM, "VI2 client availability token={:#x}, layer={}, mask={:#x}, result={:#x}", token, id, available, result.raw); + IPC::ResponseBuilder rb{ctx, 4}; + rb.Push(result.raw); + rb.Push(available); + } + void CancelClientBuffer(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 token = u64{packet[2]} | (u64{packet[3]} << 32); + const u32 slot = GraphicsSupervisorProtocol::DecodeBufferSlot(packet[4]); + Result result = Kernel::ResultInvalidCombination; + u64 available{}; + if (packet[0] == 0xf4 && packet[1] == 19 && slot < 15) { + auto state = std::ranges::find(m_client_states, token, &ClientState::token); + result = Kernel::ResultNotFound; + if (token && state != m_client_states.end()) { + // Unlike present (229), cancel has no image or input GPU fence. + state->available |= u64{1} << slot; + available = state->available; + result = ResultSuccess; + } + } + LOG_DEBUG(Service_AM, "VI2 cancel buffer token={:#x}, slot={}, mask={:#x}, result={:#x}", token, slot, available, result.raw); + IPC::ResponseBuilder rb{ctx, 8}; + rb.Push(result.raw); + rb.Push(available); + for (u32 i = 0; i < 4; ++i) rb.Push<u32>(0); + } + void RefreshReleasedBuffers() { + for (auto& state : m_client_states) { + if (!state.producer || !m_images->module) continue; + bool changed{}; + for (size_t slot = 0; slot < state.queued.size(); ++slot) { + if (!state.queued[slot]) continue; + const auto fence = state.producer->GetReleasedBufferFence(*state.queued[slot]); + if (!fence) continue; + bool complete = true; + for (u32 i = 0; i < fence->num_fences; ++i) { + const auto& point = fence->fences[i]; + if (point.id != ~u32{0}) { + const auto counter = m_images->module->GetContainer().GetSyncpointManager().ReadAllocatedSyncpoint(point.id); + if (!counter || static_cast<s32>(*counter - point.value) < 0) { + complete = false; + } + } + } + if (!complete) continue; + state.queued[slot].reset(); + state.available |= u64{1} << slot; + LOG_DEBUG(Service_AM, "Native display buffer retired: slot={}, mask={:#x}", slot, state.available); + changed = true; + } + if (changed) { + m_notifications[state.available_notification / 32] |= 1U << (state.available_notification % 32); + if (m_event) m_event->Signal(kernel); + } + } + } + Result QueueNativeImage(ClientState& state, const u32* packet) { + const u32 slot = GraphicsSupervisorProtocol::DecodeBufferSlot(packet[4]); + R_UNLESS(!state.player_display, Kernel::ResultNotImplemented); + R_UNLESS(state.layer && state.display && slot < 15 && !state.queued[slot], Kernel::ResultInvalidState); + const std::scoped_lock images_lock{m_images->mutex}; + const auto image = std::ranges::find(m_images->images, packet[5], &NativeImage::id); + R_UNLESS(image != m_images->images.end() && image->window_pool && m_images->module, Kernel::ResultNotFound); + // The observed presentation is a four-byte color image with the + // backend's 16-GOB block height and no byte offset within its pool. + R_UNLESS(packet[6] == 0 && packet[7] == image->preference[2] && packet[8] == 3 && + image->preference[4] == 7 && + (state.output_encoding == 0x14 || state.output_encoding == 0x5), + Kernel::ResultNotImplemented); + const u32 fence_id = ~packet[22]; + R_UNLESS(packet[23] == 0 && packet[25] == 0 && + (fence_id == ~u32{0} || m_images->module->GetContainer().GetSyncpointManager().IsSyncpointAllocated(fence_id)), + Kernel::ResultInvalidCombination); + if (!state.producer) { + R_TRY(m_display_service->GetContainer()->GetLayerProducerHandle(&state.producer, *state.layer)); + android::QueueBufferOutput output{}; + R_UNLESS(state.producer->Connect({}, android::NativeWindowApi::Egl, false, &output) == android::Status::NoError, + Kernel::ResultInvalidState); + R_UNLESS(state.producer->SetBufferCount(16) == android::Status::NoError, Kernel::ResultInvalidState); + s32 binder{}; + R_TRY(m_display_service->GetContainer()->OpenLayer(&binder, *state.layer, m_aruid)); + R_TRY(m_display_service->GetContainer()->SetLayerVisibility(*state.layer, true)); + } + auto native = std::make_shared<android::NvGraphicBuffer>(); + native->width = static_cast<s32>((image->preference[2] & 0xffffU) + 1); + native->height = static_cast<s32>((image->preference[2] >> 16) + 1); + native->stride = static_cast<s32>(image->preference[5] * 16); + native->format = android::PixelFormat::Rgba8888; + native->external_format = native->format; + native->buffer_id = image->memory_id; + auto buffer = std::make_shared<android::GraphicBuffer>(m_images->module->GetContainer().GetNvMapFile(), native); + s32 queue_slot{}; + R_UNLESS(state.producer->AttachBuffer(&queue_slot, buffer) == android::Status::NoError, Kernel::ResultOutOfResource); + android::QueueBufferInput input{}; + android::QueueBufferOutput output{}; + input.crop = {0, 0, native->width, native->height}; + input.swap_interval = 1; + input.fence = android::Fence::NoFence(); + if (fence_id != ~u32{0}) { + input.fence.num_fences = 1; + input.fence.fences[0] = {static_cast<s32>(fence_id), packet[24]}; + } + if (state.producer->QueueBuffer(queue_slot, input, &output) != android::Status::NoError) { + state.producer->CancelBuffer(queue_slot, input.fence); + return Kernel::ResultInvalidState; + } + state.queued[slot] = queue_slot; + state.available &= ~(u64{1} << slot); + if (!m_release_poll) { + m_release_poll = Core::Timing::CreateEvent("NativeViBufferRelease", [this](s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> { + const auto guard = LockService(); + RefreshReleasedBuffers(); + return std::nullopt; + }); + system.CoreTiming().ScheduleLoopingEvent(std::chrono::milliseconds{1}, std::chrono::milliseconds{1}, m_release_poll); + } + LOG_INFO(Service_AM, "Native image queued for display: image={}, slot={}, fence={}/{}, dimensions={}x{}", + image->id, slot, fence_id, packet[24], native->width, native->height); + R_SUCCEED(); + } + void PresentClientBuffer(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 token = u64{packet[2]} | (u64{packet[3]} << 32); + RefreshReleasedBuffers(); + Result result = Kernel::ResultInvalidCombination; + u64 available{}; + if (packet[0] == 0xf5 && packet[1] == 24) { + const auto state = std::ranges::find(m_client_states, token, &ClientState::token); + result = Kernel::ResultNotFound; + if (token && state != m_client_states.end()) { + result = QueueNativeImage(*state, packet); + available = state->available; + } + } + LOG_DEBUG(Service_AM, "VI2 present token={:#x}, image={}, slot={}, mask={:#x}, result={:#x}", token, packet[5], packet[4], available, result.raw); + IPC::ResponseBuilder rb{ctx, 8}; + rb.Push(result.raw); + rb.Push(available); + for (u32 i = 0; i < 4; ++i) rb.Push<u32>(0); + } + void SetClientOutputEncoding(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 token = u64{packet[2]} | (u64{packet[3]} << 32); + const u64 encoding = u64{packet[4]} | (u64{packet[5]} << 32); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x16f && packet[1] == 4) { + const auto state = std::ranges::find(m_client_states, token, &ClientState::token); + result = Kernel::ResultNotFound; + if (token && state != m_client_states.end()) { + result = Kernel::ResultNotImplemented; + // Only the SDK's observed default output encoding is supported. + if (encoding == 0x14 || encoding == 0x5) { + state->output_encoding = encoding; + result = ResultSuccess; + } + } + } + LOG_DEBUG(Service_AM, "VI2 output encoding token={:#x}, encoding={:#x}, result={:#x}", token, encoding, result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void BindClientDisplay(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 token = u64{packet[2]} | (u64{packet[3]} << 32); + const u64 name = u64{packet[4]} | (u64{packet[5]} << 32); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x10b && packet[1] == 4) { + const auto state = std::ranges::find(m_client_states, token, &ClientState::token); + result = Kernel::ResultNotFound; + if (token && state != m_client_states.end() && GetPlayerDisplayIndex(name)) { + const auto player = *GetPlayerDisplayIndex(name); + result = Kernel::ResultInvalidState; + if (m_player_display_references[player] && !state->display) { + result = ResultSuccess; + if (state->layer) { + result = m_display_service->GetContainer()->DestroyManagedLayer(*state->layer); + } + if (result.IsSuccess()) { + state->layer.reset(); + state->display = name; + state->player_display = player; + } + } + } else if (token && state != m_client_states.end() && name == 0x00746c7561666564ULL && + m_default_display && m_display_references) { + result = Kernel::ResultInvalidState; + if (!state->display) { + state->display = m_default_display; + result = ResultSuccess; + } + } + } + LOG_DEBUG(Service_AM, "VI2 bind client token={:#x}, display={:#x}, result={:#x}", + token, name, result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void UnbindClientDisplay(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 token = u64{packet[2]} | (u64{packet[3]} << 32); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x10c && packet[1] == 2) { + const auto state = std::ranges::find(m_client_states, token, &ClientState::token); + result = Kernel::ResultNotFound; + if (token && state != m_client_states.end() && state->display) { + state->display.reset(); + state->player_display.reset(); + result = ResultSuccess; + } + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void OpenEndpoint(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 name = u64{packet[2]} | (u64{packet[3]} << 32); + const u64 display_name = u64{packet[4]} | (u64{packet[5]} << 32); + const u64 token = u64{packet[6]} | (u64{packet[7]} << 32); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x7f && packet[1] == 6) { + const auto state = std::ranges::find(m_client_states, token, &ClientState::token); + result = Kernel::ResultNotFound; + if (token && state != m_client_states.end() && state->display && + m_default_display == state->display && name == 0x00746c7561666564ULL && + display_name == name) { + result = Kernel::ResultInvalidState; + if (!m_default_endpoint) { + m_default_endpoint = token; + result = ResultSuccess; + } + } + } + LOG_DEBUG(Service_AM, "VI2 open endpoint name={:#x}, display={:#x}, client={:#x}, result={:#x}", + name, display_name, token, result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void CloseEndpoint(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 name = u64{packet[2]} | (u64{packet[3]} << 32); + const u64 display_name = u64{packet[4]} | (u64{packet[5]} << 32); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x80 && packet[1] == 4 && name == 0x00746c7561666564ULL && + display_name == name) { + result = Kernel::ResultNotFound; + if (m_default_endpoint) { + m_default_endpoint.reset(); + result = ResultSuccess; + } + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void ConfigureRequestGroups(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x43 && packet[1] == 1 && packet[2] < 8) { + m_request_group_limit = packet[2]; + m_request_groups_initialized = true; + result = ResultSuccess; + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void GetSessionLimit(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const Result result = packet[0] == 16 + ctx.GetCommand() && packet[1] == 0 + ? ResultSuccess : Kernel::ResultInvalidCombination; + // The SDK converts this inclusive upper group index into 32 client slots. + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(result.raw); + rb.Push<u32>(ctx.GetCommand() == 5 ? 7 : 0); + } + + void GetNotifications(HLERequestContext& ctx) { + Result result = Kernel::ResultInvalidSize; + if (ctx.GetWriteBufferSize() >= sizeof(m_notifications)) { + ctx.WriteBuffer(m_notifications); + LOG_DEBUG(Service_AM, "VI2 notifications={:08x}", fmt::join(m_notifications, ",")); + m_notifications.fill(0); + if (m_event) m_event->Clear(kernel); + result = ResultSuccess; + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void ReleaseNotificationEvent(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const auto result = packet[0] == 0x1d && packet[1] == 1 && packet[2] == 0 + ? ResultSuccess : Kernel::ResultInvalidCombination; + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + } + void GetNotificationEvent(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + Kernel::KReadableEvent* readable{}; + if (packet[0] == 0x1b && packet[1] == 1 && packet[2] == 0) { + if (!m_event) { + m_event = m_context.CreateEvent("Vi2Notification"); + } + result = m_event ? ResultSuccess : Kernel::ResultOutOfResource; + if (m_event) { + if (std::ranges::any_of(m_notifications, [](u32 value) { return value != 0; })) m_event->Signal(kernel); + readable = &m_event->GetReadableEvent(); + readable->Open(kernel); + } + } + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(result.raw); + ctx.AddMoveObject(readable); + } + + std::shared_ptr<NativeImageRegistry> m_images; + std::shared_ptr<Core::Timing::EventType> m_release_poll; + std::array<ClientState, 16> m_client_states{}; + u32 m_client_slots{}; + std::shared_ptr<VI::IApplicationDisplayService> m_display_service; + Kernel::KReadableEvent* m_vsync_event{}; + std::array<Kernel::KEvent*, 4> m_display_change_events{}; + std::optional<u64> m_default_display; + std::array<bool, 6> m_display_capabilities{}; + u32 m_display_references{}; + u32 m_display_notification{}; + std::array<u32, 3> m_player_display_references{}; + std::array<u32, 3> m_player_display_notifications{}; + bool m_request_groups_initialized{}; + u32 m_request_group_limit{}; + std::optional<u64> m_default_endpoint; + std::array<u32, 32> m_notifications{}; + const u64 m_aruid; + KernelHelpers::ServiceContext m_context; + Kernel::KEvent* m_event{}; +}; + +} // namespace + + +IGraphicsSupervisor::IGraphicsSupervisor(Core::System& system, Kernel::KProcess* process, u64 aruid) + : ServiceFramework{system, "IGraphicsSupervisor"}, m_context{system, "GraphicsSupervisor"}, + m_syncpoint_cache_event{m_context}, m_process{process}, m_aruid{aruid} { + m_process->Open(kernel); + m_image_registry = std::make_shared<NativeImageRegistry>(kernel, m_process); + static const FunctionInfo functions[] = { + {4, &IGraphicsSupervisor::Initialize, "Initialize"}, + {11, &IGraphicsSupervisor::BindProcess, "BindProcess"}, + {101, &IGraphicsSupervisor::CompleteBootstrap, "CompleteBootstrap"}, + {115, &IGraphicsSupervisor::ConfigureNativeMemory, "ConfigureNativeMemory"}, + {111, &IGraphicsSupervisor::CreateNativeMemory, "CreateNativeMemory"}, + {112, &IGraphicsSupervisor::DestroyNativeMemory, "DestroyNativeMemory"}, + {114, &IGraphicsSupervisor::BindNativeMemory, "BindNativeMemory"}, + {201, &IGraphicsSupervisor::InitializeSyncpointContext, "InitializeSyncpointContext"}, + {202, &IGraphicsSupervisor::GetSyncpointSession, "GetSyncpointSession"}, + {204, &IGraphicsSupervisor::BindSyncpointMemory, "BindSyncpointMemory"}, + {301, &IGraphicsSupervisor::InitializeDisplayContext, "InitializeDisplayContext"}, + {305, &IGraphicsSupervisor::OpenDisplaySession, "OpenDisplaySession"}, + {314, &IGraphicsSupervisor::ConfigureNativeWindowImagePool, "DestroyNativeWindowAliasImagePool"}, + {315, &IGraphicsSupervisor::ConfigureNativeWindowImagePool, "CreateNativeWindowAliasImagePool"}, + {401, &IGraphicsSupervisor::InitializeGpuContext, "InitializeGpuContext"}, + {402, &IGraphicsSupervisor::InitializeGpuContext, "ActivateGpuContext"}, + {423, &IGraphicsSupervisor::BindNativeGpuMemory, "BindNativeGpuMemory"}, + {429, &IGraphicsSupervisor::CreateNativeImage, "CreateNativeImage"}, + {430, &IGraphicsSupervisor::DestroyNativeImage, "DestroyNativeImage"}, + {434, &IGraphicsSupervisor::RegisterGpuAddressSpace, "RegisterGpuAddressSpace"}, + {497, &IGraphicsSupervisor::OpenGpuSession, "OpenGpuSession"}, + {1101, &IGraphicsSupervisor::InitializeFrameContext, "InitializeFrameContext"}, + {1102, &IGraphicsSupervisor::InitializeFrameContext, "ActivateFrameContext"}, + {1402, &IGraphicsSupervisor::GetNvidiaDriver, "GetNvidiaDriver"}, + {1501, &IGraphicsSupervisor::InitializeImageContext, "InitializeImageContext1501"}, + {435, &IGraphicsSupervisor::CalculateImageRequirements, "CalculateNativeImageSize"}, + {1510, &IGraphicsSupervisor::CalculateImageRequirements, "CalculateImageRequirements"}, + {1502, &IGraphicsSupervisor::InitializeImageContext, "InitializeImageContext1502"}, + }; + RegisterHandlersTipc(functions); +} + +IGraphicsSupervisor::~IGraphicsSupervisor() { + if (m_cache_poll) { + system.CoreTiming().UnscheduleEvent(m_cache_poll); + } + if (m_syncpoint_rw) { + m_syncpoint_rw->Close(kernel); + m_syncpoint_ro->Close(kernel); + } + if (m_nv_module) { + auto& nvmap = m_nv_module->GetContainer().GetNvMapFile(); + for (const auto id : m_native_memories) { + if (const auto info = nvmap.FreeHandle(id, false); info && info->can_unlock && info->address != 0) { + m_process->GetPageTable().UnlockForDeviceAddressSpace(info->address, info->size); + } + } + if (m_native_memory_session) { + m_nv_module->GetContainer().CloseSession(Nvidia::NvCore::SessionId{*m_native_memory_session}); + } + } + m_context.CloseEvent(m_syncpoint_request_event); + m_process->Close(kernel); +} + +Result IGraphicsSupervisor::HandleSyncRequest(Kernel::KServerSession& session, + HLERequestContext& ctx) { + if (!ctx.IsTipc() || handlers_tipc.contains(ctx.GetCommand())) { + return ServiceFrameworkBase::HandleSyncRequest(session, ctx); + } + const auto guard = LockService(); + InvokeRequestTipc(ctx); + // Unsupported TIPC must return an error instead of leaving request data as a reply. + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(Kernel::ResultNotImplemented.raw); + return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess; +} + +void IGraphicsSupervisor::Initialize(HLERequestContext& ctx) { + const auto request = GraphicsSupervisorProtocol::DecodeBootstrap( + std::span<const u32>{ctx.CommandBuffer(), IPC::COMMAND_BUFFER_LENGTH}); + Result result = Kernel::ResultInvalidCombination; + if (request) { + auto process = ctx.GetObjectFromHandle<Kernel::KProcess>(request->process_handle); + const bool empty_remote_memory = std::ranges::all_of( + request->parameters, [](u64 value) { return value == 0; }); + if (process.IsNull() || process.GetPointerUnsafe() != m_process) { + result = Kernel::ResultInvalidHandle; + } else if (m_phase != Phase::Created) { + result = Kernel::ResultInvalidState; + } else if (request->applet_resource_user_id != m_aruid) { + result = Kernel::ResultInvalidArgument; + } else if ((request->flags != 0x8000000000000001ULL && request->flags != 0x8000000000000005ULL && request->flags != 0x8000000000000006ULL && request->flags != 0x8000000000000007ULL) || !empty_remote_memory) { + result = Kernel::ResultNotImplemented; + } else { + m_phase = Phase::Initialized; + result = ResultSuccess; + } + LOG_DEBUG(Service_AM, "graphics bootstrap aruid={:#x}, flags={:#x}, result={:#x}", + request->applet_resource_user_id, request->flags, result.raw); + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::BindProcess(HLERequestContext& ctx) { + Result result = Kernel::ResultInvalidCombination; + const auto* packet = ctx.CommandBuffer(); + if (packet[0] == 0x1b && packet[1] == 0x80000000 && packet[2] == 2) { + auto process = ctx.GetObjectFromHandle<Kernel::KProcess>(ctx.GetCopyHandle(0)); + if (process.IsNull() || process.GetPointerUnsafe() != m_process) { + result = Kernel::ResultInvalidHandle; + } else if (m_phase != Phase::Initialized) { + result = Kernel::ResultInvalidState; + } else { + m_phase = Phase::ProcessBound; + result = ResultSuccess; + } + } + LOG_DEBUG(Service_AM, "graphics process binding result={:#x}", result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::CompleteBootstrap(HLERequestContext& ctx) { + Result result = Kernel::ResultInvalidCombination; + const auto* packet = ctx.CommandBuffer(); + if (packet[0] == 0x75 && packet[1] == 0) { + if (m_phase != Phase::ProcessBound) { + result = Kernel::ResultInvalidState; + } else { + m_phase = Phase::Ready; + result = ResultSuccess; + } + } + LOG_DEBUG(Service_AM, "graphics bootstrap completion result={:#x}", result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::InitializeImageContext(HLERequestContext& ctx) { + Result result = Kernel::ResultInvalidCombination; + const auto* packet = ctx.CommandBuffer(); + if (packet[0] == 16 + ctx.GetCommand() && packet[1] == 0) { + if (m_phase != Phase::Ready) { + result = Kernel::ResultInvalidState; + } else if (ctx.GetCommand() == 1501) { + m_image_context_initialized = true; + result = ResultSuccess; + } else if (m_image_context_initialized) { + m_image_formats_initialized = true; + result = ResultSuccess; + } else { + result = Kernel::ResultInvalidState; + } + } + LOG_DEBUG(Service_AM, "empty image context command={}, result={:#x}", + ctx.GetCommand(), result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::OpenGpuSession(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + Kernel::KClientSession* client{}; + const u64 flags = u64{packet[2]} | (u64{packet[3]} << 32); + if (packet[0] == 0x201 && packet[1] == 2 && (flags == 3 || flags == 4)) { + if (m_phase != Phase::Ready || !m_nv_module) { + result = Kernel::ResultInvalidState; + } else { + auto handler = std::make_shared<IGpuSession>(system, m_nv_module, m_process, flags == 3); + result = CreateGraphicsSession(system, ctx, handler, &client); + if (result.IsSuccess()) { + if (flags == 3) { m_image_formats_initialized = true; } + std::erase_if(m_gpu_sessions, [](const auto& item) { return item.expired(); }); + m_gpu_sessions.emplace_back(handler); + } + } + } + LOG_DEBUG(Service_AM, "GPU control session flags={:#x}, result={:#x}", flags, result.raw); + // This SDK reads the move-handle slot before the result, including on failure. + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(result.raw); + ctx.AddMoveObject(client); +} + +void IGraphicsSupervisor::RegisterGpuAddressSpace(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x1c2 && packet[1] == 1) { + result = m_gpu_context_ready ? Kernel::ResultNotFound : Kernel::ResultInvalidState; + if (m_gpu_context_ready) { + for (const auto& item : m_gpu_sessions) { + if (auto handler = item.lock()) { + result = handler->RegisterAddressSpace(packet[2]); + if (result.IsSuccess()) { + break; + } + } + } + } + } + LOG_DEBUG(Service_AM, "register GPU address space id={}, result={:#x}", packet[2], result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::InitializeDisplayContext(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x13d && packet[1] == 0) { + result = m_phase == Phase::Ready ? ResultSuccess : Kernel::ResultInvalidState; + if (result.IsSuccess()) { + m_display_context_initialized = true; + } + } + LOG_DEBUG(Service_AM, "display context initialization result={:#x}", result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::OpenDisplaySession(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Kernel::KClientSession* client{}; + const u64 flags = u64{packet[2]} | (u64{packet[3]} << 32); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x141 && packet[1] == 2 && (flags == 0x300010000ULL || flags == 0x300020000ULL)) { + result = m_display_context_initialized ? ResultSuccess : Kernel::ResultInvalidState; + if (result.IsSuccess()) { + result = CreateGraphicsSession(system, ctx, std::make_shared<IVi2Session>(system, m_aruid, m_image_registry), + &client); + } + } + LOG_DEBUG(Service_AM, "VI2 session flags={:#x}, result={:#x}", flags, result.raw); + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(result.raw); + ctx.AddMoveObject(client); +} + +void IGraphicsSupervisor::InitializeGpuContext(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 16 + ctx.GetCommand() && packet[1] == 0) { + if (m_phase != Phase::Ready) { + result = Kernel::ResultInvalidState; + } else if (ctx.GetCommand() == 401) { + m_gpu_context_initialized = true; + result = ResultSuccess; + } else if (m_gpu_context_initialized) { + m_gpu_context_ready = true; + result = ResultSuccess; + } else { + result = Kernel::ResultInvalidState; + } + } + LOG_DEBUG(Service_AM, "GPU context command={}, result={:#x}", ctx.GetCommand(), result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::InitializeFrameContext(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 16 + ctx.GetCommand() && packet[1] == 0) { + if (m_phase != Phase::Ready) { + result = Kernel::ResultInvalidState; + } else if (ctx.GetCommand() == 1101) { + m_frame_context_initialized = true; + result = ResultSuccess; + } else if (m_frame_context_initialized) { + m_frame_context_ready = true; + result = ResultSuccess; + } else { + result = Kernel::ResultInvalidState; + } + } + LOG_DEBUG(Service_AM, "empty frame context command={}, result={:#x}", ctx.GetCommand(), result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::ConfigureNativeMemory(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x83 && packet[1] == 2 && packet[3] == 0 && + (packet[2] == 0 || packet[2] == 2)) { + result = m_phase == Phase::Ready ? ResultSuccess : Kernel::ResultInvalidState; + } + // Clear the optional Gpuk-memory mode: Eden currently provides the NVDRV path. + IPC::ResponseBuilder rb{ctx, 4}; + rb.Push(result.raw); + rb.Push<u64>(0); + LOG_DEBUG(Service_AM, "native memory uses NVDRV-compatible mode, result={:#x}", result.raw); +} + +void IGraphicsSupervisor::InitializeSyncpointContext(HLERequestContext& ctx) { + Result result = Kernel::ResultInvalidCombination; + const auto* packet = ctx.CommandBuffer(); + if (packet[0] == 0xd9 && packet[1] == 0) { + if (m_phase != Phase::Ready) { + result = Kernel::ResultInvalidState; + } else { + m_syncpoint_context_initialized = true; + result = ResultSuccess; + } + } + LOG_DEBUG(Service_AM, "syncpoint context initialization result={:#x}", result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::GetSyncpointSession(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidState; + Kernel::KClientSession* client{}; + if (packet[0] == 0xda && packet[1] == 0 && m_syncpoint_context_initialized && m_nv_module) { + result = CreateGraphicsSession(system, ctx, + std::make_shared<ISyncpointSupervisor>(system, m_nv_module), &client); + } + if (result.IsError()) { + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + return; + } + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess.raw); + rb.PushMoveObjects(ctx, client); + LOG_DEBUG(Service_AM, "opened syncpoint-supervisor session"); +} + +void IGraphicsSupervisor::GetNvidiaDriver(HLERequestContext& ctx) { + const auto Fail = [&](Result result) { + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); + }; + const auto* packet = ctx.CommandBuffer(); + if (packet[0] != 0x58a || packet[1] != 0) { + Fail(Kernel::ResultInvalidCombination); + return; + } + if (m_phase != Phase::Ready) { + Fail(Kernel::ResultInvalidState); + return; + } + auto source = system.ServiceManager().GetService<Nvidia::NVDRV>("nvdrv:s", true); + if (!source) { + Fail(Kernel::ResultNotFound); + return; + } + m_nv_module = source->GetModule(); + Kernel::KClientSession* client{}; + const auto result = CreateGraphicsSession(system, ctx, + std::make_shared<Nvidia::NVDRV>(system, m_nv_module, "nvdrv:gsv", m_process, m_aruid), + &client); + if (result.IsError()) { + Fail(result); + return; + } + LOG_DEBUG(Service_AM, "opened process-bound NVDRV session for aruid={:#x}", m_aruid); + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess.raw); + rb.PushMoveObjects(ctx, client); +} + +void IGraphicsSupervisor::BindSyncpointMemory(HLERequestContext& ctx) { + using namespace GraphicsSyncpointCache; + const auto Fail = [&](Result result) { + IPC::ResponseBuilder rb{ctx, 2, 0, 2}; + rb.Push(result.raw); + ctx.AddMoveObject(nullptr); + ctx.AddMoveObject(nullptr); + }; + const auto* packet = ctx.CommandBuffer(); + if (packet[0] != 0xdc || packet[1] != 0x80000006 || packet[2] != 4 || + packet[5] != 1 || packet[6] != 0x1ff || packet[7] != 0x1000 || packet[8] != 0 || + packet[9] != 0x2000 || packet[10] != 0) { + Fail(Kernel::ResultInvalidCombination); + return; + } + if (!m_syncpoint_context_initialized || !m_nv_module || m_syncpoint_rw) { + Fail(Kernel::ResultInvalidState); + return; + } + auto rw = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(ctx.GetCopyHandle(0)); + auto ro = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(ctx.GetCopyHandle(1)); + if (rw.IsNull() || ro.IsNull() || rw->GetOwner() != m_process || ro->GetOwner() != m_process || + rw.GetPointerUnsafe() == ro.GetPointerUnsafe()) { + Fail(Kernel::ResultInvalidHandle); + return; + } + if (rw->GetSize() != 0x1000 || ro->GetSize() != 0x2000) { + Fail(Kernel::ResultInvalidSize); + return; + } + auto* request_event = m_context.CreateEvent("SyncpointCacheRequest"); + if (request_event == nullptr) { + Fail(Kernel::ResultOutOfResource); + return; + } + m_syncpoint_request_event = request_event; + m_syncpoint_rw = rw.GetPointerUnsafe(); + m_syncpoint_ro = ro.GetPointerUnsafe(); + m_syncpoint_rw->Open(kernel); + m_syncpoint_ro->Open(kernel); + auto& memory = m_process->GetMemory(); + const auto rw_base = rw->GetSourceAddress(); + const auto ro_base = ro->GetSourceAddress(); + const std::array<u8, 0x2000> zeros{}; + memory.WriteBlock(rw_base, zeros.data(), 0x1000); + memory.WriteBlock(ro_base, zeros.data(), 0x2000); + memory.WriteBlock(ro_base, Header.data(), sizeof(Header)); + for (u32 id = 0; id < CounterCount; ++id) { + memory.Write64(ro_base + ValuesOffset + id * sizeof(u64), Unallocated); + } + m_cache_poll = Core::Timing::CreateEvent("GraphicsSyncpointCache", + [this](s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> { + const auto guard = LockService(); + this->RefreshSyncpointCache(); + return std::nullopt; + }); + system.CoreTiming().ScheduleLoopingEvent(std::chrono::milliseconds{1}, + std::chrono::milliseconds{1}, m_cache_poll); + request_event->Open(kernel); + auto* completed = m_syncpoint_cache_event.GetHandle(); + completed->Open(kernel); + IPC::ResponseBuilder rb{ctx, 2, 0, 2}; + rb.Push(ResultSuccess.raw); + ctx.AddMoveObject(request_event); + ctx.AddMoveObject(completed); + LOG_DEBUG(Service_AM, "bound syncpoint cache: {} real counters, rw={:#x}, ro={:#x}", + CounterCount, GetInteger(rw_base), GetInteger(ro_base)); +} + +void IGraphicsSupervisor::RefreshSyncpointCache() { + using namespace GraphicsSyncpointCache; + if (!m_syncpoint_rw || !system.IsPoweredOn()) { + return; + } + auto& memory = m_process->GetMemory(); + const auto rw = m_syncpoint_rw->GetSourceAddress(); + const auto ro = m_syncpoint_ro->GetSourceAddress(); + const u64 requests = memory.Read64(rw + GroupRequestOffset); + const u64 acknowledgements = memory.Read64(ro + GroupAckOffset); + u64 groups = (requests ^ acknowledgements) & ((1ULL << GroupCount) - 1); + if (groups == 0) { + return; + } + auto& syncpoints = m_nv_module->GetContainer().GetSyncpointManager(); + while (groups) { + const u32 group = static_cast<u32>(std::countr_zero(groups)); + groups &= groups - 1; + const u64 requested_slots = memory.Read64(rw + SlotRequestOffset + group * sizeof(u64)); + u64 slots = requested_slots ^ memory.Read64(ro + SlotAckOffset + group * sizeof(u64)); + while (slots) { + const u32 bit = static_cast<u32>(std::countr_zero(slots)); + slots &= slots - 1; + const u32 id = group * 64 + bit; + const auto address = ro + ValuesOffset + id * sizeof(u64); + const auto value = syncpoints.ReadAllocatedSyncpoint(id); + const u64 updated = m_counter_values[id].Observe(value); + memory.Write64(address, updated); + } + std::atomic_thread_fence(std::memory_order_release); + memory.Write64(ro + SlotAckOffset + group * sizeof(u64), requested_slots); + } + std::atomic_thread_fence(std::memory_order_release); + memory.Write64(ro + GroupAckOffset, requests); + m_syncpoint_request_event->Clear(kernel); + m_syncpoint_cache_event.Signal(kernel); +} + +void IGraphicsSupervisor::CalculateImageRequirements(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + std::optional<NativeImageRequirements> requirements; + const bool gpu_image = ctx.GetCommand() == 435; + if (packet[0] == (gpu_image ? 0x1c3U : 0x5f6U) && packet[1] == 17 && + packet[3] == (gpu_image ? 1U : 2U)) { + result = Kernel::ResultInvalidState; + if (m_phase == Phase::Ready && m_image_formats_initialized) { + requirements = CalculateNativeImageRequirements(std::span{packet + 2, 16}, packet[18]); + result = requirements ? ResultSuccess : Kernel::ResultNotImplemented; + } + } + LOG_DEBUG(Service_AM, "native image requirements dimensions={:#x}, block={}, pitch={}, size={:#x}, alignment={:#x}, result={:#x}", + packet[4], packet[6], packet[7], requirements ? requirements->size : 0, + requirements ? requirements->alignment : 0, result.raw); + IPC::ResponseBuilder rb{ctx, gpu_image ? 4U : 6U}; + rb.Push(result.raw); + rb.Push<u64>(requirements ? requirements->size : 0); + if (!gpu_image) { rb.Push<u64>(requirements ? requirements->alignment : 0); } +} +void IGraphicsSupervisor::CreateNativeMemory(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 size = u64{packet[3]} | (u64{packet[4]} << 32); + Result result = Kernel::ResultInvalidCombination; + u32 id{}; + if (packet[0] == 0x7f && packet[1] == 3 && (packet[2] & 0xff) == 3) { + result = Kernel::ResultInvalidState; + if (m_phase == Phase::Ready && m_image_formats_initialized && m_nv_module) { + result = Kernel::ResultInvalidSize; + if (size && size <= (1ULL << 32) && Common::IsAligned(size, Kernel::PageSize)) { + result = Kernel::ResultOutOfResource; + if (m_native_memories.size() < 1024) { + auto& nvmap = m_nv_module->GetContainer().GetNvMapFile(); + std::shared_ptr<Nvidia::NvCore::NvMap::Handle> handle; + if (nvmap.CreateHandle(size, handle) == Nvidia::NvResult::Success) { + id = handle->id; + m_native_memories.push_back(id); + result = ResultSuccess; + } + } + } + } + } + LOG_DEBUG(Service_AM, "native memory create size={:#x}, id={}, result={:#x}", size, id, result.raw); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(result.raw); + rb.Push(id); +} + +Result IGraphicsSupervisor::BindNativeMemoryBacking(u32 id, u64 address, u64 size, u64 flags, u64 offset) { + R_UNLESS(m_nv_module && std::ranges::find(m_native_memories, id) != m_native_memories.end(), + Kernel::ResultNotFound); + R_UNLESS((flags == 0 || flags == 2) && offset == 0, Kernel::ResultNotImplemented); + auto handle = m_nv_module->GetContainer().GetNvMapFile().GetHandle(id); + R_UNLESS(handle && !handle->allocated, Kernel::ResultInvalidState); + R_UNLESS(size && size == handle->size && address != 0 && address + size > address, + Kernel::ResultInvalidSize); + R_UNLESS(Common::IsAligned(address, Kernel::PageSize), Kernel::ResultInvalidAddress); + auto& page_table = m_process->GetPageTable(); + R_UNLESS(page_table.Contains(address, size), Kernel::ResultInvalidCurrentMemory); + if (flags == 2) { + for (u64 current = address; current < address + size;) { + Kernel::KMemoryInfo info{}; + Kernel::Svc::PageInfo page{}; + R_TRY(page_table.QueryInfo(&info, &page, current)); + R_UNLESS(True(info.m_attribute & Kernel::KMemoryAttribute::GpuAccessible), + Kernel::ResultInvalidCurrentMemory); + const u64 next = info.m_address + info.m_size; + R_UNLESS(next > current, Kernel::ResultInvalidCurrentMemory); + current = next; + } + } + bool is_io{}; + R_TRY(page_table.LockForMapDeviceAddressSpace(&is_io, address, size, + Kernel::KMemoryPermission::None, true, true)); + if (is_io) { + page_table.UnlockForDeviceAddressSpace(address, size); + R_RETURN(Kernel::ResultNotImplemented); + } + if (!m_native_memory_session) { + m_native_memory_session = m_nv_module->GetContainer().OpenSession(m_process).id; + } + const auto result = handle->Alloc({}, Kernel::PageSize, 0, address, + Nvidia::NvCore::SessionId{*m_native_memory_session}); + if (result != Nvidia::NvResult::Success) { + page_table.UnlockForDeviceAddressSpace(address, size); + R_RETURN(Kernel::ResultOutOfResource); + } + R_SUCCEED(); +} + +void IGraphicsSupervisor::BindNativeMemory(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const auto pair = [&](size_t i) { return u64{packet[i]} | (u64{packet[i + 1]} << 32); }; + const auto result = packet[0] == 0x82 && packet[1] == 9 + ? BindNativeMemoryBacking(packet[2], pair(3), pair(5), pair(7), pair(9)) + : Kernel::ResultInvalidCombination; + LOG_DEBUG(Service_AM, "native memory bind id={}, address={:#x}, size={:#x}, flags={:#x}, result={:#x}", + packet[2], pair(3), pair(5), pair(7), result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +Result IGraphicsSupervisor::ReleaseNativeMemory(u32 id) { + const auto it = std::ranges::find(m_native_memories, id); + R_UNLESS(m_nv_module && it != m_native_memories.end(), Kernel::ResultNotFound); + auto& nvmap = m_nv_module->GetContainer().GetNvMapFile(); + auto handle = nvmap.GetHandle(id); + R_UNLESS(handle, Kernel::ResultNotFound); + { + const std::scoped_lock lock{handle->mutex}; + R_UNLESS(handle->pins == 0 && handle->internal_dupes == 0 && handle->dupes == 1, + Kernel::ResultBusy); + } + const bool allocated = handle->allocated; + handle.reset(); + const auto info = nvmap.FreeHandle(id, false); + m_native_memories.erase(it); + if (allocated) { + R_UNLESS(info && info->can_unlock, Kernel::ResultInvalidState); + R_TRY(m_process->GetPageTable().UnlockForDeviceAddressSpace(info->address, info->size)); + } + R_SUCCEED(); +} + +void IGraphicsSupervisor::DestroyNativeMemory(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const auto result = packet[0] == 0x80 && packet[1] == 1 + ? ReleaseNativeMemory(packet[2]) : Kernel::ResultInvalidCombination; + LOG_DEBUG(Service_AM, "native memory destroy id={}, result={:#x}", packet[2], result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::BindNativeGpuMemory(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const auto pair = [&](size_t i) { return u64{packet[i]} | (u64{packet[i + 1]} << 32); }; + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x1b7 && packet[1] == 12 && pair(10) == 0 && pair(12) == 0) { + result = Kernel::ResultNotFound; + if (m_nv_module) { + auto& container = m_nv_module->GetContainer(); + const auto parent = container.GetNvMapFile().GetHandle(packet[7]); + if (parent && parent->allocated) { + const auto* session = container.GetSession(parent->session_id); + const u64 offset = pair(8); + const u64 address = pair(3); + const u64 size = pair(5); + if (session && session->process == m_process && offset <= parent->size && + size <= parent->size - offset && parent->address + offset >= parent->address && + address == parent->address + offset) { + result = BindNativeMemoryBacking(packet[2], address, size, 2, 0); + } + } + } + } + LOG_DEBUG(Service_AM, "native GPU memory bind id={}, parent={}, address={:#x}, size={:#x}, offset={:#x}, result={:#x}", + packet[2], packet[7], pair(3), pair(5), pair(8), result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::CreateNativeImage(HLERequestContext& ctx) { + const std::scoped_lock images_lock{m_image_registry->mutex}; + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + u32 id{}; + if (packet[0] == 0x1bd && packet[1] == 18 && packet[3] == 1) { + result = Kernel::ResultInvalidState; + if (m_phase == Phase::Ready && m_gpu_context_ready && m_image_formats_initialized && m_nv_module) { + const auto requirements = CalculateNativeImageRequirements(std::span{packet + 2, 16}, packet[18]); + result = Kernel::ResultNotImplemented; + if (requirements) { + result = Kernel::ResultNotFound; + auto& nvmap = m_nv_module->GetContainer().GetNvMapFile(); + const auto memory = nvmap.GetHandle(packet[19]); + if (std::ranges::find(m_native_memories, packet[19]) != m_native_memories.end() && + memory && memory->allocated) { + result = Kernel::ResultInvalidSize; + if (memory->size >= requirements->size) { + result = Kernel::ResultOutOfResource; + if (m_image_registry->images.size() < 1024 && m_next_native_image_id != 0 && + memory->Duplicate(true) == Nvidia::NvResult::Success) { + NativeImage image{m_next_native_image_id++, packet[19], {}, requirements->size}; + std::copy_n(packet + 2, image.preference.size(), image.preference.begin()); + id = image.id; + if (!m_image_registry->module) { + m_image_registry->module = m_nv_module; + m_image_registry->memory_session = m_nv_module->GetContainer().OpenSession(m_process); + } + m_image_registry->images.push_back(image); + result = ResultSuccess; + } + } + } + } + } + } + LOG_DEBUG(Service_AM, "native image create memory={}, count={}, id={}, result={:#x}", + packet[19], packet[18], id, result.raw); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(result.raw); + rb.Push(id); +} + +void IGraphicsSupervisor::DestroyNativeImage(HLERequestContext& ctx) { + const std::scoped_lock images_lock{m_image_registry->mutex}; + const auto* packet = ctx.CommandBuffer(); + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == 0x1be && packet[1] == 1) { + const auto image = std::ranges::find(m_image_registry->images, packet[2], &NativeImage::id); + result = Kernel::ResultNotFound; + if (m_nv_module && image != m_image_registry->images.end()) { + result = Kernel::ResultBusy; + if (!image->window_pool) { + m_image_registry->ReleaseMemory(image->memory_id); + m_image_registry->images.erase(image); + result = ResultSuccess; + } + } + } + LOG_DEBUG(Service_AM, "native image destroy id={}, result={:#x}", packet[2], result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +void IGraphicsSupervisor::ConfigureNativeWindowImagePool(HLERequestContext& ctx) { + const std::scoped_lock images_lock{m_image_registry->mutex}; + const auto* packet = ctx.CommandBuffer(); + const bool create = ctx.GetCommand() == 315; + Result result = Kernel::ResultInvalidCombination; + if (packet[0] == (create ? 0x14bU : 0x14aU) && packet[1] == (create ? 2U : 1U)) { + const auto image = std::ranges::find(m_image_registry->images, packet[2], &NativeImage::id); + result = Kernel::ResultNotFound; + if (image != m_image_registry->images.end()) { + result = Kernel::ResultNotImplemented; + if (!create || packet[3] == 1) { + result = Kernel::ResultInvalidState; + if (image->window_pool != create) { + image->window_pool = create; + result = ResultSuccess; + } + } + } + } + LOG_DEBUG(Service_AM, "native window image pool id={}, create={}, result={:#x}", + packet[2], create, result.raw); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result.raw); +} + +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/graphics_supervisor.h b/src/core/hle/service/am/service/graphics_supervisor.h new file mode 100644 index 0000000000..4163c2c150 --- /dev/null +++ b/src/core/hle/service/am/service/graphics_supervisor.h @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <vector> + +#include "core/hle/service/service.h" +#include "core/hle/service/am/service/graphics_syncpoint_cache.h" +#include "core/hle/service/kernel_helpers.h" +#include "core/hle/service/os/event.h" + +namespace Kernel { +class KProcess; +class KTransferMemory; +class KEvent; +} + +namespace Core::Timing { +struct EventType; +} +namespace Service::Nvidia { +class Module; +} + +namespace Service::AM { +struct NativeImageRegistry; + +class IGpuSession; + +class IGraphicsSupervisor final : public ServiceFramework<IGraphicsSupervisor> { +public: + explicit IGraphicsSupervisor(Core::System& system, Kernel::KProcess* process, u64 aruid); + ~IGraphicsSupervisor() override; + Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override; + +private: + enum class Phase { Created, Initialized, ProcessBound, Ready }; + void Initialize(HLERequestContext& ctx); + void BindProcess(HLERequestContext& ctx); + void CompleteBootstrap(HLERequestContext& ctx); + void InitializeImageContext(HLERequestContext& ctx); + void GetNvidiaDriver(HLERequestContext& ctx); + void CalculateImageRequirements(HLERequestContext& ctx); + void CreateNativeMemory(HLERequestContext& ctx); + void CreateNativeImage(HLERequestContext& ctx); + void DestroyNativeImage(HLERequestContext& ctx); + void ConfigureNativeWindowImagePool(HLERequestContext& ctx); + void BindNativeMemory(HLERequestContext& ctx); + void BindNativeGpuMemory(HLERequestContext& ctx); + void DestroyNativeMemory(HLERequestContext& ctx); + Result BindNativeMemoryBacking(u32 id, u64 address, u64 size, u64 flags, u64 offset); + Result ReleaseNativeMemory(u32 id); + void ConfigureNativeMemory(HLERequestContext& ctx); + void InitializeFrameContext(HLERequestContext& ctx); + void RegisterGpuAddressSpace(HLERequestContext& ctx); + void OpenGpuSession(HLERequestContext& ctx); + void OpenDisplaySession(HLERequestContext& ctx); + void InitializeDisplayContext(HLERequestContext& ctx); + void InitializeGpuContext(HLERequestContext& ctx); + void InitializeSyncpointContext(HLERequestContext& ctx); + void GetSyncpointSession(HLERequestContext& ctx); + void BindSyncpointMemory(HLERequestContext& ctx); + void RefreshSyncpointCache(); + + std::vector<std::weak_ptr<IGpuSession>> m_gpu_sessions; + std::shared_ptr<Nvidia::Module> m_nv_module; + std::optional<size_t> m_native_memory_session; + std::vector<u32> m_native_memories; + std::shared_ptr<NativeImageRegistry> m_image_registry; + u32 m_next_native_image_id{1}; + std::shared_ptr<Core::Timing::EventType> m_cache_poll; + std::array<GraphicsSyncpointCache::CounterValue, GraphicsSyncpointCache::CounterCount> + m_counter_values{}; + Kernel::KTransferMemory* m_syncpoint_rw{}; + Kernel::KTransferMemory* m_syncpoint_ro{}; + Kernel::KEvent* m_syncpoint_request_event{}; + KernelHelpers::ServiceContext m_context; + Event m_syncpoint_cache_event; + bool m_syncpoint_context_initialized{}; + Kernel::KProcess* const m_process; + const u64 m_aruid; + Phase m_phase{Phase::Created}; + bool m_display_context_initialized{}; + bool m_gpu_context_initialized{}; + bool m_gpu_context_ready{}; + bool m_frame_context_initialized{}; + bool m_frame_context_ready{}; + bool m_image_context_initialized{}; + bool m_image_formats_initialized{}; +}; + +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/graphics_supervisor_protocol.h b/src/core/hle/service/am/service/graphics_supervisor_protocol.h new file mode 100644 index 0000000000..7762ced531 --- /dev/null +++ b/src/core/hle/service/am/service/graphics_supervisor_protocol.h @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <array> +#include <optional> +#include <span> +#include "common/common_types.h" + +namespace Service::AM::GraphicsSupervisorProtocol { + +constexpr u8 DecodeBufferSlot(u32 packed) { + return static_cast<u8>(packed); +} + +struct BootstrapRequest { + u32 process_handle; + u64 applet_resource_user_id; + u64 flags; + std::array<u64, 4> parameters; +}; + +inline std::optional<BootstrapRequest> DecodeBootstrap(std::span<const u32> packet) { + // Captured remote-memory bootstrap: TIPC 4, one copied handle, 48 data bytes. + if (packet.size() < 16 || packet[0] != 0x14 || packet[1] != 0x8000000c || + packet[2] != 2) { + return std::nullopt; + } + const auto Read64 = [&](size_t index) { + return u64{packet[index]} | (u64{packet[index + 1]} << 32); + }; + return BootstrapRequest{packet[3], Read64(4), Read64(6), + {Read64(8), Read64(10), Read64(12), Read64(14)}}; +} + +} // namespace Service::AM::GraphicsSupervisorProtocol diff --git a/src/core/hle/service/am/service/graphics_syncpoint_cache.h b/src/core/hle/service/am/service/graphics_syncpoint_cache.h new file mode 100644 index 0000000000..23361b006c --- /dev/null +++ b/src/core/hle/service/am/service/graphics_syncpoint_cache.h @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <array> +#include <limits> +#include <optional> +#include "common/common_types.h" + +namespace Service::AM::GraphicsSyncpointCache { +constexpr u32 ValuesOffset = 0x40; +constexpr u32 GroupAckOffset = 0x640; +constexpr u32 SlotAckOffset = 0x648; +constexpr u32 GroupRequestOffset = 0; +constexpr u32 SlotRequestOffset = 8; +constexpr u32 CounterCount = 192; +constexpr u32 GroupCount = (CounterCount + 63) / 64; +constexpr u64 Unallocated = (std::numeric_limits<u64>::max)(); + +constexpr std::array<u32, 8> Header{ + 1, 0, CounterCount - 1, ValuesOffset, GroupRequestOffset, + GroupAckOffset, SlotRequestOffset, SlotAckOffset}; +static_assert(ValuesOffset + CounterCount * sizeof(u64) <= GroupAckOffset); +static_assert(SlotAckOffset + GroupCount * sizeof(u64) <= 0x2000); +static_assert(SlotRequestOffset + GroupCount * sizeof(u64) <= 0x1000); + +constexpr u64 ExtendCounter(u64 previous, u32 current) { + if (previous == Unallocated) { + return current; + } + u64 high = previous & 0xffffffff00000000ULL; + if (current < static_cast<u32>(previous)) { + high += 1ULL << 32; + } + return high | current; +} +struct CounterValue { + u64 previous{Unallocated}; + + u64 Observe(std::optional<u32> current) { + if (!current) { + return Unallocated; + } + previous = ExtendCounter(previous, *current); + return previous; + } +}; +} // namespace Service::AM::GraphicsSyncpointCache diff --git a/src/core/hle/service/am/service/native_image_layout.h b/src/core/hle/service/am/service/native_image_layout.h new file mode 100644 index 0000000000..eb45fc470d --- /dev/null +++ b/src/core/hle/service/am/service/native_image_layout.h @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <optional> +#include <span> +#include "common/common_types.h" + +namespace Service::AM { + +struct NativeImageRequirements { + u64 size; + u64 alignment; +}; + +// Experimental inter-module block image layout, restricted to the observed +// four-byte color format. Row pitch is in 64-byte GOBs; block height is in pixels. +inline std::optional<NativeImageRequirements> CalculateNativeImageRequirements( + std::span<const u32> preference, u32 count) { + if (preference.size() != 16 || count != 1 || preference[0] != 0 || + (preference[1] != 1 && preference[1] != 2) || preference[3] != 0 || preference[6] != 1 || + preference[7] != 1 || preference[4] < 3 || preference[4] > 8) { + return std::nullopt; + } + for (size_t i = 8; i < preference.size(); ++i) { + if (preference[i] != 0) { return std::nullopt; } + } + const u64 width = (preference[2] & 0xffffU) + 1; + const u64 height = (preference[2] >> 16) + 1; + const u64 pitch_gobs = preference[5]; + if (pitch_gobs < (width * 4 + 63) / 64 || pitch_gobs > 4096) { + return std::nullopt; + } + const u64 block_height = 1ULL << preference[4]; + const u64 aligned_height = (height + block_height - 1) & ~(block_height - 1); + return NativeImageRequirements{pitch_gobs * 64 * aligned_height, 64 * block_height}; +} + +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/performance_functions.cpp b/src/core/hle/service/am/service/performance_functions.cpp new file mode 100644 index 0000000000..71a46cc5b5 --- /dev/null +++ b/src/core/hle/service/am/service/performance_functions.cpp @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "core/hle/service/am/service/performance_functions.h" + +namespace Service::AM { + +IPerformanceFunctions::IPerformanceFunctions(Core::System& system) + : ServiceFramework{system, "IPerformanceFunctions"} { + static const FunctionInfo functions[] = { + {0, nullptr, "IsCpuOverclockEnabled"}, + {1, nullptr, "SetCpuOverclockEnabled"}, + }; + RegisterHandlers(functions); +} + +IPerformanceFunctions::~IPerformanceFunctions() = default; + +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/performance_functions.h b/src/core/hle/service/am/service/performance_functions.h new file mode 100644 index 0000000000..12195d1217 --- /dev/null +++ b/src/core/hle/service/am/service/performance_functions.h @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "core/hle/service/service.h" + +namespace Service::AM { + +class IPerformanceFunctions final : public ServiceFramework<IPerformanceFunctions> { +public: + explicit IPerformanceFunctions(Core::System& system); + ~IPerformanceFunctions() override; +}; + +} // namespace Service::AM diff --git a/src/core/hle/service/am/service/self_controller.cpp b/src/core/hle/service/am/service/self_controller.cpp index 3a654b9daa..45b9120093 100644 --- a/src/core/hle/service/am/service/self_controller.cpp +++ b/src/core/hle/service/am/service/self_controller.cpp @@ -5,10 +5,17 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include "common/logging.h" +#include "core/hle/kernel/k_scoped_resource_reservation.h" +#include "core/hle/kernel/k_session.h" +#include "core/hle/kernel/k_client_session.h" +#include "core/hle/kernel/k_thread.h" #include "core/hle/result.h" #include "core/hle/service/am/am_results.h" #include "core/hle/service/am/frontend/applets.h" #include "core/hle/service/am/service/self_controller.h" +#include "core/hle/service/am/service/graphics_supervisor.h" +#include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/server_manager.h" #include "core/hle/service/caps/caps_su.h" #include "core/hle/service/cmif_serialization.h" #include "core/hle/service/nvnflinger/nvnflinger.h" @@ -73,6 +80,7 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet> {230, D<&ISelfController::Unknown230>, "Unknown230"}, {240, D<&ISelfController::Unknown240>, "Unknown240"}, {1000, nullptr, "GetDebugStorageChannel"}, + {2000, &ISelfController::GetGraphicsSupervisor, "GetGraphicsSupervisor"}, }; // clang-format on @@ -84,10 +92,47 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet> } ISelfController::~ISelfController() { + if (m_graphics_supervisor_session != nullptr) { + m_graphics_supervisor_session->Close(kernel); + } std::scoped_lock lk{m_applet->lock}; m_applet->display_layer_manager.Finalize(); } +void ISelfController::GetGraphicsSupervisor(HLERequestContext& ctx) { + if (m_graphics_supervisor_session != nullptr) { + IPC::ResponseBuilder rb{ctx, 2, 1}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, m_graphics_supervisor_session); + return; + } + Kernel::KScopedResourceReservation reservation( + kernel, Kernel::GetCurrentProcessPointer(kernel), Kernel::LimitableResource::SessionCountMax); + if (!reservation.Succeeded()) { + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(Kernel::ResultLimitReached); + return; + } + auto* session = Kernel::KSession::Create(kernel); + if (session == nullptr) { + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(Kernel::ResultOutOfResource); + return; + } + session->Initialize(kernel, nullptr, 0); + reservation.Commit(); + Kernel::KSession::Register(kernel, session); + auto& server = ctx.GetManager()->GetServerManager(); + auto manager = std::make_shared<SessionRequestManager>(kernel, server); + manager->SetSessionHandler(std::make_shared<IGraphicsSupervisor>(system, m_process, m_applet->aruid.pid)); + server.RegisterSession(&session->GetServerSession(), std::move(manager)); + LOG_DEBUG(Service_AM, "created graphics-supervisor session"); + m_graphics_supervisor_session = &session->GetClientSession(); + IPC::ResponseBuilder rb{ctx, 2, 1}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, m_graphics_supervisor_session); +} + Result ISelfController::Exit() { LOG_DEBUG(Service_AM, "called"); diff --git a/src/core/hle/service/am/service/self_controller.h b/src/core/hle/service/am/service/self_controller.h index c9f480f575..52970ed4a8 100644 --- a/src/core/hle/service/am/service/self_controller.h +++ b/src/core/hle/service/am/service/self_controller.h @@ -12,6 +12,7 @@ namespace Kernel { class KReadableEvent; +class KClientSession; } namespace Service::Capture { @@ -30,6 +31,7 @@ public: ~ISelfController() override; private: + void GetGraphicsSupervisor(HLERequestContext& ctx); Result Exit(); Result LockExit(); Result UnlockExit(); @@ -70,6 +72,7 @@ private: Result Unknown230(u32 in_val, Out<u16> out_val); Result Unknown240(u32 in_val); + Kernel::KClientSession* m_graphics_supervisor_session{}; Kernel::KProcess* const m_process; const std::shared_ptr<Applet> m_applet; }; diff --git a/src/core/hle/service/audio/audio.cpp b/src/core/hle/service/audio/audio.cpp index fde766a275..63d578462f 100644 --- a/src/core/hle/service/audio/audio.cpp +++ b/src/core/hle/service/audio/audio.cpp @@ -5,6 +5,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include "core/core.h" +#include "core/hle/kernel/svc_results.h" +#include "core/hle/service/ipc_helpers.h" #include "core/hle/service/audio/audio.h" #include "core/hle/service/audio/audio_controller.h" #include "core/hle/service/audio/audio_in_manager.h" @@ -18,6 +20,57 @@ namespace Service::Audio { +class INativeAudioManager final : public ServiceFramework<INativeAudioManager> { +public: + INativeAudioManager(Core::System& system_, std::shared_ptr<IAudioInManager> input, + std::shared_ptr<IAudioOutManager> output, + std::shared_ptr<IAudioRendererManager> renderer) + : ServiceFramework{system_, "aud:u"}, m_input{std::move(input)}, + m_output{std::move(output)}, m_renderer{std::move(renderer)} { + static const FunctionInfo functions[] = { + {0, &INativeAudioManager::GetAudioInManager, "GetAudioInManager"}, + {1, &INativeAudioManager::GetAudioOutManager, "GetAudioOutManager"}, + {2, &INativeAudioManager::GetAudioRendererManager, "GetAudioRendererManager"}, + }; + RegisterHandlers(functions); + } + + Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override { + if (!ctx.IsTipc()) { + return ServiceFrameworkBase::HandleSyncRequest(session, ctx); + } + const auto guard = LockService(); + LOG_WARNING(Service_Audio, "Native audio unsupported command={}, words={:08x}", ctx.GetCommand(), + fmt::join(std::span<const u32>(ctx.CommandBuffer() + 2, + std::min<u32>(ctx.CommandBuffer()[1], 32)), ",")); + InvokeRequestTipc(ctx); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(Kernel::ResultNotImplemented.raw); + return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess; + } + +private: + void GetAudioInManager(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface(ctx, m_input); + } + void GetAudioOutManager(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface(ctx, m_output); + } + void GetAudioRendererManager(HLERequestContext& ctx) { + LOG_DEBUG(Service_Audio, "Native user client opens existing audio renderer manager"); + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface(ctx, m_renderer); + } + std::shared_ptr<IAudioInManager> m_input; + std::shared_ptr<IAudioOutManager> m_output; + std::shared_ptr<IAudioRendererManager> m_renderer; +}; + class IAudioOutManagerForApplet final : public ServiceFramework<IAudioOutManagerForApplet> { public: explicit IAudioOutManagerForApplet(Core::System& system_) @@ -193,18 +246,23 @@ void LoopProcess(Core::System& system) { server_manager->RegisterNamedService("audrec:d", std::make_shared<IFinalOutputRecorderManagerForDebugger>(system), 30); server_manager->RegisterNamedService("audren:d", std::make_shared<IAudioRendererManagerForDebugger>(system), 30); - server_manager->RegisterNamedService("audin:u", std::make_shared<IAudioInManager>(system), 30); + auto audio_input = std::make_shared<IAudioInManager>(system); + server_manager->RegisterNamedService("audin:u", audio_input, 30); server_manager->RegisterNamedService("audin:a", std::make_shared<IAudioInManagerForApplet>(system), 30); - server_manager->RegisterNamedService("audout:u", std::make_shared<IAudioOutManager>(system), 30); + auto audio_output = std::make_shared<IAudioOutManager>(system); + server_manager->RegisterNamedService("audout:u", audio_output, 30); server_manager->RegisterNamedService("audout:a", std::make_shared<IAudioOutManagerForApplet>(system), 30); server_manager->RegisterNamedService("auddev", std::make_shared<IAudioSnoopManager>(system), 30); // Depends on audout:u and audin:u on ctor! server_manager->RegisterNamedService("audctl", std::make_shared<IAudioController>(system), 30); server_manager->RegisterNamedService("audrec:a", std::make_shared<IFinalOutputRecorderManagerForApplet>(system), 30); server_manager->RegisterNamedService("audrec:u", std::make_shared<IFinalOutputRecorderManager>(system), 30); - server_manager->RegisterNamedService("audren:u", std::make_shared<IAudioRendererManager>(system), 30); + auto audio_renderer = std::make_shared<IAudioRendererManager>(system); + server_manager->RegisterNamedService("audren:u", audio_renderer, 30); server_manager->RegisterNamedService("audren:a", std::make_shared<IAudioRendererManagerForApplet>(system), 30); server_manager->RegisterNamedService("hwopus", std::make_shared<IHardwareOpusDecoderManager>(system), 25); + server_manager->RegisterNamedService("aud:u", + std::make_shared<INativeAudioManager>(system, audio_input, audio_output, audio_renderer), 30); ServerManager::RunServer(std::move(server_manager)); } diff --git a/src/core/hle/service/hid/hid_server.cpp b/src/core/hle/service/hid/hid_server.cpp index 9689bce8dd..7ddb944abe 100644 --- a/src/core/hle/service/hid/hid_server.cpp +++ b/src/core/hle/service/hid/hid_server.cpp @@ -132,6 +132,7 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {210, C<&IHidServer::EndPermitVibrationSession>, "EndPermitVibrationSession"}, {211, C<&IHidServer::IsVibrationDeviceMounted>, "IsVibrationDeviceMounted"}, {212, C<&IHidServer::SendVibrationValueInBool>, "SendVibrationValueInBool"}, + {214, C<&IHidServer::SendNativeVibrationValues>, "SendNativeVibrationValues"}, {300, C<&IHidServer::ActivateConsoleSixAxisSensor>, "ActivateConsoleSixAxisSensor"}, {301, C<&IHidServer::StartConsoleSixAxisSensor>, "StartConsoleSixAxisSensor"}, {302, C<&IHidServer::StopConsoleSixAxisSensor>, "StopConsoleSixAxisSensor"}, @@ -237,6 +238,11 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {3013, nullptr, "SetDebugPadGenericPadMap"}, //21.0.0+ {3014, nullptr, "GetDebugPadKeyboardMap"}, //21.0.0+ {3015, nullptr, "SetDebugPadKeyboardMap"}, //21.0.0+ + {3110, C<&IHidServer::RegisterJcmsMemory>, "RegisterJcmsMemory"}, + {3136, C<&IHidServer::SetNmouseButtonConfigJoyLeftJcms>, "SetNmouseButtonConfigJoyLeftJcms"}, + {3137, C<&IHidServer::SetNmouseButtonConfigJoyRightJcms>, "SetNmouseButtonConfigJoyRightJcms"}, + {3138, C<&IHidServer::SetMergedNmouseSourceTypes>, "SetMergedNmouseSourceTypes"}, + {3139, C<&IHidServer::RegisterNativeInputMemory>, "RegisterNativeInputMemory"}, {3150, C<&IHidServer::SetMouseLibraryVersion>, "SetMouseLibraryVersion"}, //21.0.0+ // What? -- {12010, nullptr, "SetButtonConfigLeft"}, }; @@ -245,7 +251,11 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r RegisterHandlers(functions); } -IHidServer::~IHidServer() = default; +IHidServer::~IHidServer() { + for (const auto& registration : native_input_memories) { + registration.memory->Close(kernel); + } +} Result IHidServer::CreateAppletResource(OutInterface<IAppletResource> out_applet_resource, ClientAppletResourceUserId aruid) { @@ -984,6 +994,17 @@ Result IHidServer::SendVibrationValues( R_SUCCEED(); } +Result IHidServer::SendNativeVibrationValues( + u16 flags, AppletResourceUserId aruid, + InArray<Core::HID::VibrationDeviceHandle, BufferAttr_HipcPointer> vibration_handles, + InArray<Core::HID::VibrationValue, BufferAttr_HipcPointer> vibration_values) { + static u32 diagnostic_batches{}; + if (diagnostic_batches++ < 8) LOG_INFO(Service_HID, "Native vibration batch flags={:#x}, aruid={}, handles={}, values={}", + flags, aruid.pid, vibration_handles.size(), vibration_values.size()); + R_UNLESS(flags == 0 || flags == 2, IPC::ResultNotSupported); + R_RETURN(SendVibrationValues(ClientAppletResourceUserId{aruid.pid}, vibration_handles, vibration_values)); +} + Result IHidServer::SendVibrationGcErmCommand( Core::HID::VibrationDeviceHandle vibration_device_handle, ClientAppletResourceUserId aruid, Core::HID::VibrationGcErmCommand gc_erm_command) { @@ -1471,6 +1492,63 @@ Result IHidServer::SetTouchScreenResolution(u32 width, u32 height, R_SUCCEED(); } +Result IHidServer::RegisterNativeInputMemory( + u8 mode, u32 parameter0, u32 parameter1, u32 parameter2, + ClientAppletResourceUserId aruid, u64 size, + InCopyHandle<Kernel::KTransferMemory> memory) { + LOG_INFO(Service_HID, "Native input memory mode={}, parameters={:#x}/{:#x}/{:#x}, aruid={}, size={:#x}", + mode, parameter0, parameter1, parameter2, aruid.pid, size); + R_UNLESS(memory.Get() && memory->GetOwner()->UsesPAuthReturns() && + memory->GetOwner()->GetProcessId() == aruid.pid, Kernel::ResultInvalidHandle); + R_UNLESS(mode <= 1 && size && size == memory->GetSize() && size <= 0x1000000, + Kernel::ResultInvalidSize); + R_UNLESS(native_input_memories.size() < 64, Kernel::ResultOutOfResource); + const auto existing = std::ranges::find(native_input_memories, memory.Get(), &NativeInputMemory::memory); + R_UNLESS(existing == native_input_memories.end(), Kernel::ResultInvalidState); + memory->Open(kernel); + native_input_memories.push_back({memory.Get(), aruid.pid, size, {mode, parameter0, parameter1, parameter2}}); + // Retain the real guest-owned input buffer. No Joy-Con mouse samples are + // produced while those devices are disconnected. + R_SUCCEED(); +} + +Result IHidServer::RegisterJcmsMemory(ClientAppletResourceUserId aruid, u64 size, + InCopyHandle<Kernel::KTransferMemory> memory) { + LOG_INFO(Service_HID, "Native Joy-Con mouse memory: aruid={}, size={:#x}", aruid.pid, size); + return RegisterNativeInputMemory(1, 0, 0, 0, aruid, size, memory); +} + +Result IHidServer::SetMergedNmouseSourceTypes(u32 source_types, ClientAppletResourceUserId aruid) { + LOG_INFO(Service_HID, "Native merged mouse sources: types={:#x}, aruid={}", source_types, aruid.pid); + R_UNLESS((source_types & ~6U) == 0, Kernel::ResultInvalidCombination); + const auto memory = std::ranges::find_if(native_input_memories, [&](const auto& registration) { + return registration.aruid == aruid.pid && registration.parameters[0] == 0; + }); + R_UNLESS(memory != native_input_memories.end(), Kernel::ResultNotFound); + memory->merged_source_types = source_types; + R_SUCCEED(); +} + +Result IHidServer::SetNmouseButtonConfiguration(ClientAppletResourceUserId aruid, u64 configuration, + size_t side) { + LOG_INFO(Service_HID, "Native Joy-Con mouse buttons: side={}, config={:#x}, aruid={}", + side, configuration, aruid.pid); + const auto memory = std::ranges::find_if(native_input_memories, [&](const auto& registration) { + return registration.aruid == aruid.pid && registration.parameters[0] == 0; + }); + R_UNLESS(memory != native_input_memories.end(), Kernel::ResultNotFound); + memory->jcms_button_configurations[side] = configuration; + R_SUCCEED(); +} + +Result IHidServer::SetNmouseButtonConfigJoyLeftJcms(ClientAppletResourceUserId aruid, u64 configuration) { + return SetNmouseButtonConfiguration(aruid, configuration, 0); +} + +Result IHidServer::SetNmouseButtonConfigJoyRightJcms(ClientAppletResourceUserId aruid, u64 configuration) { + return SetNmouseButtonConfiguration(aruid, configuration, 1); +} + Result IHidServer::SetMouseLibraryVersion(ClientAppletResourceUserId aruid) { LOG_INFO(Service_HID, "(STUBBED) called, applet_resource_user_id={}", aruid.pid); diff --git a/src/core/hle/service/hid/hid_server.h b/src/core/hle/service/hid/hid_server.h index 57a1a04b9c..45097f361d 100644 --- a/src/core/hle/service/hid/hid_server.h +++ b/src/core/hle/service/hid/hid_server.h @@ -6,6 +6,8 @@ #pragma once +#include <array> +#include <vector> #include "core/hle/service/cmif_types.h" #include "core/hle/service/service.h" #include "hid_core/hid_types.h" @@ -175,6 +177,10 @@ private: ClientAppletResourceUserId aruid, InArray<Core::HID::VibrationDeviceHandle, BufferAttr_HipcPointer> vibration_handles, InArray<Core::HID::VibrationValue, BufferAttr_HipcPointer> vibration_values); + Result SendNativeVibrationValues( + u16 flags, AppletResourceUserId aruid, + InArray<Core::HID::VibrationDeviceHandle, BufferAttr_HipcPointer> vibration_handles, + InArray<Core::HID::VibrationValue, BufferAttr_HipcPointer> vibration_values); Result SendVibrationGcErmCommand(Core::HID::VibrationDeviceHandle vibration_device_handle, ClientAppletResourceUserId aruid, Core::HID::VibrationGcErmCommand gc_erm_command); @@ -265,6 +271,24 @@ private: Result SetTouchScreenResolution(u32 width, u32 height, ClientAppletResourceUserId aruid); Result SetMouseLibraryVersion(ClientAppletResourceUserId aruid); + Result RegisterNativeInputMemory(u8 mode, u32 parameter0, u32 parameter1, u32 parameter2, + ClientAppletResourceUserId aruid, u64 size, + InCopyHandle<Kernel::KTransferMemory> memory); + Result RegisterJcmsMemory(ClientAppletResourceUserId aruid, u64 size, + InCopyHandle<Kernel::KTransferMemory> memory); + Result SetMergedNmouseSourceTypes(u32 source_types, ClientAppletResourceUserId aruid); + Result SetNmouseButtonConfigJoyLeftJcms(ClientAppletResourceUserId aruid, u64 configuration); + Result SetNmouseButtonConfigJoyRightJcms(ClientAppletResourceUserId aruid, u64 configuration); + Result SetNmouseButtonConfiguration(ClientAppletResourceUserId aruid, u64 configuration, size_t side); + struct NativeInputMemory { + Kernel::KTransferMemory* memory; + u64 aruid; + u64 size; + std::array<u32, 4> parameters; + u32 merged_source_types{}; + std::array<u64, 2> jcms_button_configurations{}; + }; + std::vector<NativeInputMemory> native_input_memories; std::shared_ptr<ResourceManager> resource_manager; std::shared_ptr<HidFirmwareSettings> firmware_settings; }; diff --git a/src/core/hle/service/ngc/ngc.cpp b/src/core/hle/service/ngc/ngc.cpp index e79108a1eb..2c79336d93 100644 --- a/src/core/hle/service/ngc/ngc.cpp +++ b/src/core/hle/service/ngc/ngc.cpp @@ -7,6 +7,7 @@ #include "common/string_util.h" #include "core/core.h" #include "core/hle/kernel/k_client_session.h" +#include "core/hle/kernel/k_shared_memory.h" #include "core/hle/result.h" #include "core/hle/service/acc/profile_manager.h" #include "core/hle/service/cmif_types.h" @@ -182,6 +183,8 @@ public: explicit IUserShimScopedObject(Core::System& system_) : ServiceFramework(system_, "IUserShimScopedObject") { // clang-format off static const FunctionInfo functions[] = { + {1, D<&IUserShimScopedObject::GetRequiredMemorySize>, "GetRequiredMemorySize"}, + {300, D<&IUserShimScopedObject::GetGuestHidSharedMemory>, "GetGuestHidSharedMemory"}, {450, nullptr, "InitializeForSaveData"}, {451, nullptr, "FinalizeForSaveData"}, {452, D<&IUserShimScopedObject::OpenSaveData>, "OpenSaveData"}, @@ -195,6 +198,50 @@ public: RegisterHandlers(functions); } + ~IUserShimScopedObject() override { + if (m_guest_hid_memory) { + m_guest_hid_memory->Close(kernel); + } + } + + struct GuestHidMemoryConfig { + u64 size; + u64 offset; + }; + + Result GetRequiredMemorySize(Out<u32> out_size, u8 type, u8 mode, s32 guest_count) { + LOG_INFO(Service_NGC, "GameShare work memory query type={}, mode={}, guests={}", type, mode, guest_count); + R_UNLESS(type <= 1 && mode == 1 && guest_count >= 0 && guest_count <= 3, + IPC::ResultNotSupported); + // The disconnected implementation reserves one page for its session + // control state. Streaming/encoding work is not implemented. + *out_size = 0x1000; + R_SUCCEED(); + } + + Result GetGuestHidSharedMemory(Out<GuestHidMemoryConfig> out_config, + OutCopyHandle<Kernel::KSharedMemory> out_memory) { + // Three guest pad blocks, touch rings, and JCMS rings. Empty rings and + // zero style sets describe no connected GameShare guests. + constexpr u64 GuestHidMemorySize = 0x84000; + if (!m_guest_hid_memory) { + auto* memory = Kernel::KSharedMemory::Create(kernel); + const auto result = memory->Initialize(kernel, system.DeviceMemory(), nullptr, + Kernel::Svc::MemoryPermission::ReadWrite, Kernel::Svc::MemoryPermission::Read, + GuestHidMemorySize); + if (R_FAILED(result)) { + memory->Close(kernel); + R_RETURN(result); + } + Kernel::KSharedMemory::Register(kernel, memory); + m_guest_hid_memory = memory; + } + *out_config = GuestHidMemoryConfig{GuestHidMemorySize, 0}; + *out_memory = m_guest_hid_memory; + LOG_DEBUG(Service_NGC, "GameShare guest HID memory size={:#x}, no guests connected", GuestHidMemorySize); + R_SUCCEED(); + } + Result OpenSaveData(Account::Uid unk0, Out<SaveDataHandle> unk1) { LOG_WARNING(Service_NGC, "stubbed"); R_THROW(IPC::ResultNotSupported); @@ -210,6 +257,9 @@ public: // to implement R_SUCCEED(); } + +private: + Kernel::KSharedMemory* m_guest_hid_memory{}; }; class IUserService final : public ServiceFramework<IUserService> { diff --git a/src/core/hle/service/nvdrv/core/syncpoint_manager.cpp b/src/core/hle/service/nvdrv/core/syncpoint_manager.cpp index c4c4c25932..8b929a7ef3 100644 --- a/src/core/hle/service/nvdrv/core/syncpoint_manager.cpp +++ b/src/core/hle/service/nvdrv/core/syncpoint_manager.cpp @@ -63,6 +63,14 @@ void SyncpointManager::FreeSyncpoint(u32 id) { syncpoint.reserved = false; } +std::optional<u32> SyncpointManager::ReadAllocatedSyncpoint(u32 id) { + std::lock_guard lock(reservation_lock); + if (id >= SyncpointCount || !syncpoints[id].reserved) { + return std::nullopt; + } + return UpdateMin(id); +} + bool SyncpointManager::IsSyncpointAllocated(u32 id) const { return (id < SyncpointCount) && syncpoints[id].reserved; } diff --git a/src/core/hle/service/nvdrv/core/syncpoint_manager.h b/src/core/hle/service/nvdrv/core/syncpoint_manager.h index 7728ff5967..42e1d547b8 100644 --- a/src/core/hle/service/nvdrv/core/syncpoint_manager.h +++ b/src/core/hle/service/nvdrv/core/syncpoint_manager.h @@ -7,6 +7,7 @@ #include <array> #include <atomic> #include <mutex> +#include <optional> #include "common/common_types.h" #include "core/hle/service/nvdrv/nvdata.h" @@ -38,6 +39,8 @@ enum class ChannelType : u32 { */ class SyncpointManager final { public: + std::optional<u32> ReadAllocatedSyncpoint(u32 id); + explicit SyncpointManager(Tegra::Host1x::Host1x& host1x); ~SyncpointManager(); diff --git a/src/core/hle/service/nvdrv/devices/gpu_address_space_layout.h b/src/core/hle/service/nvdrv/devices/gpu_address_space_layout.h new file mode 100644 index 0000000000..a54ebb18d7 --- /dev/null +++ b/src/core/hle/service/nvdrv/devices/gpu_address_space_layout.h @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "common/common_types.h" + +namespace Service::Nvidia::Devices { + +struct GpuAddressSpaceLayout { + u64 start; + u64 split; + u64 end; +}; + +constexpr GpuAddressSpaceLayout GetGpuAddressSpaceLayout(bool extended) { + // Leave room for other 4 KiB allocations beyond the native bootstrap + // reservation of 0x7ff8000000 bytes. + // Both layouts fit within the GPU's reported 40-bit virtual address width. + return {0x8000000, extended ? 3ULL << 38 : 1ULL << 34, + extended ? 1ULL << 40 : 1ULL << 37}; +} + +constexpr u32 GetGpuAddressSpaceBigPageSize(bool extended, u32 requested) { + // The native SDK passes its preferred allocation size here. A preference for + // ordinary 4 KiB mappings still needs a separate large-page pool in NVDRV. + return extended && requested == 0x1000 ? 0x10000 : requested; +} + +constexpr u64 GpuPageAddress(u32 page, u32 page_bits) { + return static_cast<u64>(page) << page_bits; +} + +} // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/native_gpu_characteristics.h b/src/core/hle/service/nvdrv/devices/native_gpu_characteristics.h new file mode 100644 index 0000000000..44f45cab75 --- /dev/null +++ b/src/core/hle/service/nvdrv/devices/native_gpu_characteristics.h @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <array> +#include <cstring> +#include <span> +#include "common/common_types.h" +#include "common/swap.h" + +namespace Service::Nvidia::Devices { + +inline constexpr size_t NativeGpuCharacteristicsSize = 0xd0; + +inline std::array<u8, NativeGpuCharacteristicsSize> MakeNativeGpuCharacteristics( + std::span<const u8, 0xa0> legacy) { + std::array<u8, NativeGpuCharacteristicsSize> result{}; + // The GPU description and engine classes share the legacy prefix. The native + // SDK consumes signed ioctl interface levels where the legacy ABI has topology. + std::memcpy(result.data(), legacy.data(), 0x70); + // Select the SDK's native control ABI, whose normal mappings support + // 4 KiB CPU/GPU aliases. Engine execution remains a separate implementation. + const u32_le native_arch{0x170}; + const u32_le native_impl{0xe}; + std::memcpy(result.data(), &native_arch, sizeof(native_arch)); + std::memcpy(result.data() + 4, &native_impl, sizeof(native_impl)); + // Native NVN's built-in shader table accepts Ampere SM 8.x. Carrying the + // inherited Maxwell SM 5.3 value here rejects its initialization shaders. + const u32_le native_sm_version{0x807}; + std::memcpy(result.data() + 0x3c, &native_sm_version, sizeof(native_sm_version)); + std::memcpy(result.data() + 0x40, &native_sm_version, sizeof(native_sm_version)); + // Native NVN maps CPU buffers into its shared VA reservation. Those buffers + // are 4 KiB aligned, so large pages must remain optional for allocation. + const u32_le preferred_page_size{0x1000}; + std::memcpy(result.data() + 0x28, &preferred_page_size, sizeof(preferred_page_size)); + u32_le available_page_sizes; + std::memcpy(&available_page_sizes, legacy.data() + 0x34, sizeof(available_page_sizes)); + available_page_sizes = (static_cast<u32>(available_page_sizes) & 0x10000) | 0x1000; + std::memcpy(result.data() + 0x34, &available_page_sizes, sizeof(available_page_sizes)); + u64_le flags; + std::memcpy(&flags, legacy.data() + 0x50, sizeof(flags)); + // Native SDK class enumeration filters 2D/3D/compute/inline families + // with bits 40..43 before selecting context identities from its profile. + // The native SDK gates ModifyKind (AS ioctl 12) on capability bit 34. + flags = static_cast<u64>(flags) | (1ULL << 8) | (1ULL << 22) | (1ULL << 26) | (1ULL << 34) | (0xfULL << 40); + std::memcpy(result.data() + 0x50, &flags, sizeof(flags)); + constexpr std::array<s16, 9> levels{ + 0x1c, // GPU control: GetGpuTime + 11, // TSG: bind/unbind channel with native subcontext IDs + -1, // GPU debugger device is unavailable + 0x1d, // Channel: SetTimeslice + 0x14, // Address space: Remap + 0, 0, 0, 0, // Additional interfaces have not been implemented. + }; + for (size_t i = 0; i < levels.size(); ++i) { + const u16_le value{static_cast<u16>(levels[i])}; + std::memcpy(result.data() + 0x70 + i * sizeof(value), &value, sizeof(value)); + } + // The native SDK allocates shared queue subcontexts locally, then binds + // each channel with TSG ioctl 11. Each emulated channel retains its own state. + const u32_le max_subcontexts{64}; + std::memcpy(result.data() + 0xb4, &max_subcontexts, sizeof(max_subcontexts)); + return result; +} + +} // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/native_gpu_context.h b/src/core/hle/service/nvdrv/devices/native_gpu_context.h new file mode 100644 index 0000000000..7c7131f211 --- /dev/null +++ b/src/core/hle/service/nvdrv/devices/native_gpu_context.h @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <optional> +#include "common/common_types.h" + +namespace Service::Nvidia::Devices { + +// Native profile context identities. This classifies allocation metadata; +// engine method execution is handled separately by the GPU frontend. +inline std::optional<u32> GetNativeGpuContextSlot(u32 identity) { + switch (identity) { + case 0xc797: return 1; + case 0xc7c0: return 2; + case 0xc7b5: return 4; + case 0xc56f: return 5; + default: return std::nullopt; + } +} + +} // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/native_gpu_kind.h b/src/core/hle/service/nvdrv/devices/native_gpu_kind.h new file mode 100644 index 0000000000..ecf3781162 --- /dev/null +++ b/src/core/hle/service/nvdrv/devices/native_gpu_kind.h @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <optional> +#include "common/common_types.h" +#include "video_core/pte_kind.h" + +namespace Service::Nvidia::Devices { + +inline std::optional<u8> TranslateNativeUncompressedKind(u8 kind, u8 fallback) { + // NVIDIA's Turing MMU uses 6 for GENERIC_MEMORY and 7 for INVALID: + // open-gpu-doc/manuals/turing/tu104/dev_mmu.ref.txt. Eden's older kind + // numbering uses 0xfe for generic memory. Equal kinds need no compression. + if (kind != fallback) { return std::nullopt; } + switch (kind) { + case 0: return static_cast<u8>(Tegra::PTEKind::PITCH); + case 1: return static_cast<u8>(Tegra::PTEKind::Z16); + case 3: return static_cast<u8>(Tegra::PTEKind::S8Z24); + case 6: return static_cast<u8>(Tegra::PTEKind::GENERIC_16BX2); + default: return std::nullopt; + } +} + +struct NativeMemoryKind { + u8 backend; + u8 selected; +}; + +inline std::optional<NativeMemoryKind> SelectNativeMemoryKind(u8 requested, u8 fallback) { + if (const auto kind = TranslateNativeUncompressedKind(requested, fallback)) { + return NativeMemoryKind{*kind, requested}; + } + // The native caller supplies an uncompressed alternative explicitly. Return + // that selected native kind so the SDK invalidates its compressed preference. + if (requested == 8 && fallback == 6) { + return NativeMemoryKind{static_cast<u8>(Tegra::PTEKind::GENERIC_16BX2), 6}; + } + // TU104 kind 0x0c is the compressible S8Z24 variant; kind 3 is + // the caller's uncompressed S8Z24 alternative (dev_mmu.ref.txt). + if (requested == 0xb && fallback == 1) { + return NativeMemoryKind{static_cast<u8>(Tegra::PTEKind::Z16), 1}; + } + if (requested == 0xc && fallback == 3) { + return NativeMemoryKind{static_cast<u8>(Tegra::PTEKind::S8Z24), 3}; + } + return std::nullopt; +} + +} // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/nvdevice.h b/src/core/hle/service/nvdrv/devices/nvdevice.h index d23ec63bae..cbcee2a938 100644 --- a/src/core/hle/service/nvdrv/devices/nvdevice.h +++ b/src/core/hle/service/nvdrv/devices/nvdevice.h @@ -27,7 +27,9 @@ namespace Service::Nvidia::Devices { /// implement the ioctl interface. class nvdevice { public: - explicit nvdevice(Core::System& system_) : system{system_} {} + enum class Type { Other, GpuChannel, GpuTsg, GpuAddressSpace }; + explicit nvdevice(Core::System& system_, Type type_ = Type::Other) : system{system_}, type{type_} {} + Type GetType() const { return type; } virtual ~nvdevice() = default; /** @@ -80,6 +82,8 @@ public: protected: Core::System& system; +private: + const Type type; }; } // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/nvdisp_disp0.cpp b/src/core/hle/service/nvdrv/devices/nvdisp_disp0.cpp index b8a8979315..0db83215ca 100644 --- a/src/core/hle/service/nvdrv/devices/nvdisp_disp0.cpp +++ b/src/core/hle/service/nvdrv/devices/nvdisp_disp0.cpp @@ -5,17 +5,24 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <boost/container/small_vector.hpp> +#include <cstdlib> +#include <fstream> +#include <filesystem> +#include "core/frontend/framebuffer_layout.h" +#include "video_core/renderer_base.h" #include "common/adpf.h" #include "common/assert.h" #include "common/logging.h" #include "core/core.h" +#include "core/hle/kernel/k_process.h" #include "core/core_timing.h" #include "core/hle/service/nvdrv/core/container.h" #include "core/hle/service/nvdrv/core/nvmap.h" #include "core/hle/service/nvdrv/devices/nvdisp_disp0.h" #include "core/perf_stats.h" #include "video_core/gpu.h" +#include "video_core/host1x/host1x.h" namespace Service::Nvidia::Devices { @@ -67,6 +74,9 @@ void nvdisp_disp0::Composite(std::span<const Nvnflinger::HwcLayer> sorted_layers output_fences.reserve(sorted_layers.size()); for (auto& layer : sorted_layers) { + const auto handle = nvmap.GetHandle(layer.buffer_handle); + const auto* session = handle ? container.GetSession(handle->session_id) : nullptr; + const bool native_tiling = session && session->process->UsesPAuthReturns(); output_layers.emplace_back(Tegra::FramebufferConfig{ .address = nvmap.GetHandleAddress(layer.buffer_handle), .offset = layer.offset, @@ -77,6 +87,7 @@ void nvdisp_disp0::Composite(std::span<const Nvnflinger::HwcLayer> sorted_layers .transform_flags = layer.transform, .crop_rect = layer.crop_rect, .blending = ConvertBlending(layer.blending), + .native_tiling = native_tiling, .layer_stack_mask = layer.layer_stack_mask, }); @@ -87,6 +98,56 @@ void nvdisp_disp0::Composite(std::span<const Nvnflinger::HwcLayer> sorted_layers } } + const char* capture = std::getenv("EDEN_NATIVE_FRAME_CAPTURE"); + const bool sequence = std::getenv("EDEN_NATIVE_FRAME_SEQUENCE") != nullptr; + const bool on_demand = std::getenv("EDEN_NATIVE_FRAME_ON_DEMAND") != nullptr; + const bool requested = on_demand && !output_layers.empty() && + !diagnostic_frame_pending->load() && std::filesystem::exists("native-frame.request"); + if (requested) std::filesystem::remove("native-frame.request"); + const bool sample_frame = requested || (capture && std::string_view{capture} == "1" && !output_layers.empty() && + diagnostic_frame_count < (sequence ? 36000U : 3840U) && ++diagnostic_frame_count != 0 && + (sequence ? (diagnostic_frame_count == 1 || diagnostic_frame_count % 120 == 0) + : !diagnostic_visible_frame->load())); + if (sample_frame && !diagnostic_frame_pending->exchange(true)) { + for (const auto& fence : output_fences) { + LOG_INFO(Service_NVDRV, "Native composite waits for syncpoint={}/{}, current={}", fence.id, fence.value, + system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(fence.id)); + } + LOG_INFO(Service_NVDRV, "Native composite submitted: layers={}, address={:#x}", output_layers.size(), output_layers.front().address); + const auto layout = Layout::DefaultFrameLayout(1920, 1080); + auto pixels = std::make_shared<std::vector<u8>>(layout.width * layout.height * 4); + const bool first = diagnostic_frame_count == 1; + auto pending = diagnostic_frame_pending; + auto visible = diagnostic_visible_frame; + system.GPU().Renderer().RequestScreenshot(pixels->data(), [pixels, layout, first, pending, visible, sequence, on_demand](bool invert_y) { + std::array<u8, 18> header{}; + header[2] = 2; + header[12] = static_cast<u8>(layout.width & 255); + header[13] = static_cast<u8>(layout.width >> 8); + header[14] = static_cast<u8>(layout.height & 255); + header[15] = static_cast<u8>(layout.height >> 8); + header[16] = 32; + header[17] = invert_y ? 8 : 0x28; + bool nonblack{}; + for (size_t i = 0; i < pixels->size(); i += 4) { + if ((*pixels)[i] || (*pixels)[i + 1] || (*pixels)[i + 2]) { + nonblack = true; + break; + } + } + const auto save = [&](const char* path) { + std::ofstream out(path, std::ios::binary); + out.write(reinterpret_cast<const char*>(header.data()), header.size()); + out.write(reinterpret_cast<const char*>(pixels->data()), pixels->size()); + return out.good(); + }; + if (first) save("native-first-frame.tga"); + const bool saved = (first || nonblack || sequence || on_demand) ? save("native-latest-frame.tga") : true; + if (nonblack && !visible->load() && save("native-visible-frame.tga")) visible->store(true); + LOG_INFO(Service_NVDRV, "Captured composited frame: {}x{}, saved={}, nonblack={}", layout.width, layout.height, saved, nonblack); + pending->store(false); + }, layout); + } system.GPU().RequestComposite(std::move(output_layers), std::move(output_fences)); Common::ADPF::ReportFrameInterval(); system.SpeedLimiter().DoSpeedLimiting(system.CoreTiming().GetGlobalTimeUs()); diff --git a/src/core/hle/service/nvdrv/devices/nvdisp_disp0.h b/src/core/hle/service/nvdrv/devices/nvdisp_disp0.h index ff0bfb2ec9..f795144ebe 100644 --- a/src/core/hle/service/nvdrv/devices/nvdisp_disp0.h +++ b/src/core/hle/service/nvdrv/devices/nvdisp_disp0.h @@ -6,6 +6,7 @@ #pragma once +#include <atomic> #include <memory> #include <vector> #include "common/common_types.h" @@ -45,6 +46,9 @@ public: Kernel::KEvent* QueryEvent(u32 event_id) override; private: + u32 diagnostic_frame_count{}; + std::shared_ptr<std::atomic_bool> diagnostic_frame_pending{std::make_shared<std::atomic_bool>(false)}; + std::shared_ptr<std::atomic_bool> diagnostic_visible_frame{std::make_shared<std::atomic_bool>(false)}; NvCore::Container& container; NvCore::NvMap& nvmap; }; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index 3c9e42f602..947ea6088f 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -6,6 +6,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include <cstring> +#include <limits> #include <utility> #include "common/alignment.h" @@ -13,10 +14,13 @@ #include "common/common_types.h" #include "common/logging.h" #include "core/core.h" +#include "core/hle/kernel/k_process.h" +#include "core/hle/service/nvdrv/devices/gpu_address_space_layout.h" #include "core/hle/service/nvdrv/core/container.h" #include "core/hle/service/nvdrv/core/nvmap.h" #include "core/hle/service/nvdrv/devices/ioctl_serialization.h" #include "core/hle/service/nvdrv/devices/nvhost_as_gpu.h" +#include "core/hle/service/nvdrv/devices/native_gpu_kind.h" #include "core/hle/service/nvdrv/devices/nvhost_gpu.h" #include "core/hle/service/nvdrv/nvdrv.h" #include "video_core/control/channel_state.h" @@ -27,10 +31,12 @@ namespace Service::Nvidia::Devices { nvhost_as_gpu::nvhost_as_gpu(Core::System& system_, Module& module_, NvCore::Container& core) - : nvdevice{system_}, module{module_}, container{core}, nvmap{core.GetNvMapFile()}, vm{}, + : nvdevice{system_, Type::GpuAddressSpace}, module{module_}, container{core}, nvmap{core.GetNvMapFile()}, vm{}, gmmu{} {} -nvhost_as_gpu::~nvhost_as_gpu() = default; +nvhost_as_gpu::~nvhost_as_gpu() { + if (m_owner_process) { m_owner_process->Close(system.Kernel()); } +} NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input, std::span<u8> output) { @@ -51,6 +57,10 @@ NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> i return WrapFixed(this, &nvhost_as_gpu::GetVARegions1, input, output); case 0x9: return WrapFixed(this, &nvhost_as_gpu::AllocAsEx, input, output); + case 0xb: + return WrapFixed(this, &nvhost_as_gpu::GetSyncpointShimInfo, input, output); + case 0xc: + return WrapFixed(this, &nvhost_as_gpu::ModifyKind, input, output); case 0x14: return WrapVariable(this, &nvhost_as_gpu::Remap, input, output); default: @@ -90,11 +100,139 @@ NvResult nvhost_as_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> i return NvResult::NotImplemented; } -void nvhost_as_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) {} +void nvhost_as_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) { + m_fd = fd; + if (const auto* session = container.GetSession(session_id)) { + m_owner_process = session->process; + m_owner_process->Open(system.Kernel()); + m_extended_abi = m_owner_process->UsesPAuthReturns(); + } +} void nvhost_as_gpu::OnClose(DeviceFD fd) {} +NvResult nvhost_as_gpu::CreateNativeSparseRegion(u64 address, u64 size) { + if (!m_extended_abi || !size || !Common::IsAligned(address, VM::YUZU_PAGESIZE) || + !Common::IsAligned(size, VM::YUZU_PAGESIZE) || (size >> VM::PAGE_SIZE_BITS) > 0xffffffffULL) { + return NvResult::BadParameter; + } + { + const std::scoped_lock lock{mutex}; + if (!vm.initialised || address < vm.va_range_start || address >= vm.va_range_split || + size > vm.va_range_split - address) { return NvResult::BadParameter; } + const auto next = allocation_map.lower_bound(address); + if ((next != allocation_map.end() && next->first < address + size) || + (next != allocation_map.begin() && std::prev(next)->first + std::prev(next)->second.size > address)) { + return NvResult::BadParameter; + } + } + IoctlAllocSpace reservation{}; + reservation.pages = static_cast<u32>(size >> VM::PAGE_SIZE_BITS); + reservation.page_size = VM::YUZU_PAGESIZE; + reservation.flags = MappingFlags::Fixed | MappingFlags::Sparse; + reservation.offset = address; + return AllocateSpace(reservation); +} + +NvResult nvhost_as_gpu::DestroyNativeSparseRegion(u64 address, u64 size) { + { + const std::scoped_lock lock{mutex}; + const auto it = allocation_map.find(address); + if (!m_extended_abi || it == allocation_map.end() || !it->second.sparse || it->second.size != size) { + return NvResult::BadParameter; + } + } + IoctlFreeSpace release{}; + release.offset = address; + release.pages = static_cast<u32>(size >> VM::PAGE_SIZE_BITS); + release.page_size = VM::YUZU_PAGESIZE; + return FreeSpace(release); +} + +NvResult nvhost_as_gpu::MapNative4K(u32 handle_id, u64 address, u64 buffer_offset, u64 size, u8 kind) { + if (!m_extended_abi || !size || !Common::IsAligned(address, VM::YUZU_PAGESIZE) || + !Common::IsAligned(size, VM::YUZU_PAGESIZE) || + !Common::IsAligned(buffer_offset, VM::YUZU_PAGESIZE)) { + return NvResult::BadParameter; + } + const auto handle = nvmap.GetHandle(handle_id); + if (!handle || buffer_offset > handle->orig_size || size > handle->orig_size - buffer_offset) { + return NvResult::BadParameter; + } + bool sparse_reservation = false; + { + const std::scoped_lock lock{mutex}; + if (!vm.initialised || address < vm.va_range_start || address >= vm.va_range_split || + size > vm.va_range_split - address) { + return NvResult::BadParameter; + } + const auto upper = allocation_map.upper_bound(address); + if (upper != allocation_map.begin()) { + const auto owner = std::prev(upper); + sparse_reservation = owner->second.sparse && address >= owner->first && + address - owner->first < owner->second.size && size <= owner->second.size - (address - owner->first); + } + const auto next = allocation_map.lower_bound(address); + if (!sparse_reservation && ((next != allocation_map.end() && next->first < address + size) || + (next != allocation_map.begin() && + std::prev(next)->first + std::prev(next)->second.size > address))) { + return NvResult::BadParameter; + } + } + IoctlAllocSpace reservation{}; + reservation.pages = static_cast<u32>(size >> VM::PAGE_SIZE_BITS); + reservation.page_size = VM::YUZU_PAGESIZE; + reservation.flags = MappingFlags::Fixed; + reservation.offset = address; + const auto reserved = sparse_reservation ? NvResult::Success : AllocateSpace(reservation); + if (reserved != NvResult::Success) { return reserved; } + IoctlMapBufferEx mapping{}; + mapping.flags = MappingFlags::Fixed; + mapping.kind = kind; + mapping.handle = handle_id; + mapping.page_size = VM::YUZU_PAGESIZE; + mapping.buffer_offset = static_cast<s64>(buffer_offset); + mapping.mapping_size = size; + mapping.offset = static_cast<s64>(address); + const auto result = MapBufferEx(mapping); + if (result != NvResult::Success && !sparse_reservation) { + nvmap.UnpinHandle(handle_id); + IoctlFreeSpace release{}; + release.offset = address; + release.pages = reservation.pages; + release.page_size = VM::YUZU_PAGESIZE; + FreeSpace(release); + } + return result; +} + +NvResult nvhost_as_gpu::UnmapNative4K(u64 address, u64 size) { + if (!m_extended_abi || !size || !Common::IsAligned(address, VM::YUZU_PAGESIZE) || + !Common::IsAligned(size, VM::YUZU_PAGESIZE)) { + return NvResult::BadParameter; + } + bool sparse_mapping = false; + { + const std::scoped_lock lock{mutex}; + const auto it = mapping_map.find(address); + sparse_mapping = it != mapping_map.end() && it->second.sparse_alloc && it->second.size == size; + } + if (sparse_mapping) { + IoctlUnmapBuffer unmap{}; + unmap.offset = address; + return UnmapBuffer(unmap); + } + IoctlFreeSpace release{}; + release.offset = address; + release.pages = static_cast<u32>(size >> VM::PAGE_SIZE_BITS); + release.page_size = VM::YUZU_PAGESIZE; + return FreeSpace(release); +} + + NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) { - LOG_DEBUG(Service_NVDRV, "called, big_page_size={:#x}", params.big_page_size); + LOG_DEBUG(Service_NVDRV, "called, flags={:#x}, as_fd={}, big_page_size={:#x}, reserved={:#x}, VA={:#x}/{:#x}/{:#x}", + params.flags, params.as_fd, params.big_page_size, params.reserved, + params.va_range_start, params.va_range_split, params.va_range_end); std::scoped_lock lock(mutex); @@ -103,21 +241,43 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) { return NvResult::InvalidState; } - if (params.big_page_size) { - if (!std::has_single_bit(params.big_page_size)) { - LOG_ERROR(Service_NVDRV, "Non power-of-2 big page size: {:#x}!", params.big_page_size); + std::shared_ptr<nvhost_as_gpu> shared_parent; + if (m_extended_abi && params.as_fd != 0) { + if (params.as_fd == m_fd || params.flags != 5 || params.big_page_size != 0x1000 || + params.reserved != 0 || params.va_range_split != 0) { return NvResult::BadParameter; } + const auto device = module.GetDevice<nvdevice>(params.as_fd); + if (!device || device->GetType() != Type::GpuAddressSpace) { return NvResult::BadParameter; } + shared_parent = std::static_pointer_cast<nvhost_as_gpu>(device); + if (!shared_parent->m_extended_abi || !shared_parent->vm.initialised || + shared_parent->m_owner_process != m_owner_process || + params.va_range_start < shared_parent->vm.va_range_start || + params.va_range_start >= params.va_range_end || + params.va_range_end > shared_parent->vm.va_range_split) { return NvResult::BadParameter; } + } + + const u32 big_page_size = GetGpuAddressSpaceBigPageSize(m_extended_abi, params.big_page_size); + if (big_page_size) { + if (!std::has_single_bit(big_page_size)) { + LOG_ERROR(Service_NVDRV, "Non power-of-2 big page size: {:#x}!", big_page_size); return NvResult::BadValue; } - if ((params.big_page_size & VM::SUPPORTED_BIG_PAGE_SIZES) == 0) { - LOG_ERROR(Service_NVDRV, "Unsupported big page size: {:#x}!", params.big_page_size); + if ((big_page_size & VM::SUPPORTED_BIG_PAGE_SIZES) == 0) { + LOG_ERROR(Service_NVDRV, "Unsupported big page size: {:#x}!", big_page_size); return NvResult::BadValue; } - vm.big_page_size = params.big_page_size; - vm.big_page_size_bits = static_cast<u32>(std::countr_zero(params.big_page_size)); + vm.big_page_size = big_page_size; + vm.big_page_size_bits = static_cast<u32>(std::countr_zero(big_page_size)); + + vm.va_range_start = big_page_size << VM::VA_START_SHIFT; + } - vm.va_range_start = params.big_page_size << VM::VA_START_SHIFT; + if (m_extended_abi && !params.va_range_start) { + const auto layout = GetGpuAddressSpaceLayout(true); + vm.va_range_start = layout.start; + vm.va_range_split = layout.split; + vm.va_range_end = layout.end; } // If this is unspecified then default values should be used @@ -127,6 +287,12 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) { vm.va_range_end = params.va_range_end; } + if (shared_parent) { + // Native flags 5 expose the parent's small-page region as a separate + // reservation view. GPU mappings must resolve through the same page table. + vm.va_range_split = params.va_range_end; + } + const u64 max_big_page_bits = Common::Log2Ceil(vm.va_range_end); const auto start_pages{static_cast<u32>(vm.va_range_start >> VM::PAGE_SIZE_BITS)}; @@ -135,11 +301,17 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) { const auto start_big_pages{static_cast<u32>(vm.va_range_split >> vm.big_page_size_bits)}; const auto end_big_pages{ - static_cast<u32>((vm.va_range_end - vm.va_range_split) >> vm.big_page_size_bits)}; + static_cast<u32>(vm.va_range_end >> vm.big_page_size_bits)}; vm.big_page_allocator.emplace(start_big_pages, end_big_pages); - gmmu = std::make_unique<Tegra::MemoryManager>(system, max_big_page_bits, vm.va_range_split, vm.big_page_size_bits, VM::PAGE_SIZE_BITS); - system.GPU().InitAddressSpace(*gmmu); + if (shared_parent) { + m_shared_parent = std::move(shared_parent); + gmmu = m_shared_parent->gmmu; + LOG_DEBUG(Service_NVDRV, "native AS view fd={} shares page table with fd={}", m_fd, params.as_fd); + } else { + gmmu = std::make_shared<Tegra::MemoryManager>(system, max_big_page_bits, vm.va_range_split, vm.big_page_size_bits, VM::PAGE_SIZE_BITS); + system.GPU().InitAddressSpace(*gmmu); + } vm.initialised = true; return NvResult::Success; @@ -159,7 +331,7 @@ NvResult nvhost_as_gpu::AllocateSpace(IoctlAllocSpace& params) { return NvResult::BadValue; } - if (params.page_size != vm.big_page_size && + if (!m_extended_abi && params.page_size != vm.big_page_size && ((params.flags & MappingFlags::Sparse) != MappingFlags::None)) { UNIMPLEMENTED_MSG("Sparse small pages are not implemented!"); return NvResult::NotImplemented; @@ -184,7 +356,7 @@ NvResult nvhost_as_gpu::AllocateSpace(IoctlAllocSpace& params) { u64 size{static_cast<u64>(params.pages) * params.page_size}; if ((params.flags & MappingFlags::Sparse) != MappingFlags::None) { - gmmu->MapSparse(params.offset, size); + gmmu->MapSparse(params.offset, size, params.page_size != VM::YUZU_PAGESIZE); } allocation_map[params.offset] = { @@ -217,6 +389,7 @@ bool nvhost_as_gpu::FreeMappingLocked(u64 offset) noexcept { gmmu->Unmap(offset, mapping.size); } mapping_map.erase(offset); + map_buffer_offsets.erase(static_cast<s64>(offset)); return true; } return false; @@ -251,6 +424,77 @@ NvResult nvhost_as_gpu::FreeSpace(IoctlFreeSpace& params) { return NvResult::BadValue; } +NvResult nvhost_as_gpu::GetNativeSyncpointShim(u64& address, u32& stride, u32& count) { + if (!m_extended_abi || !vm.initialised) { return NvResult::NotSupported; } + address = vm.va_range_split - VM::YUZU_PAGESIZE; + if (!m_syncpoint_shim_reserved) { + IoctlAllocSpace allocation{}; + allocation.pages = 1; + allocation.page_size = VM::YUZU_PAGESIZE; + allocation.flags = MappingFlags::Fixed; + allocation.offset = address; + const auto result = AllocateSpace(allocation); + if (result != NvResult::Success) { return result; } + m_syncpoint_shim_reserved = true; + gmmu->SetNativeSyncpointShim(address); + } + stride = sizeof(u32); + count = 192; + return NvResult::Success; +} + +NvResult nvhost_as_gpu::GetSyncpointShimInfo(IoctlSyncpointShimInfo& params) { + u64 address{}; + u32 stride{}, count{}; + const auto result = GetNativeSyncpointShim(address, stride, count); + params.address = address; + params.stride = stride; + params.count = count; + LOG_DEBUG(Service_NVDRV, "native syncpoint shim address={:#x}, stride={}, count={}, result={}", + address, stride, count, static_cast<u32>(result)); + return result; +} + +NvResult nvhost_as_gpu::ModifyNativeKind(u64 address, u64 size, u8 requested, u8 fallback) { + IoctlModifyKind params{}; + params.compressed_kind = requested; + params.uncompressed_kind = fallback; + params.mapping_base = address; + params.size = size; + return ModifyKind(params); +} + +NvResult nvhost_as_gpu::ModifyKind(IoctlModifyKind& params) { + const u32 requested = params.compressed_kind; + const u32 fallback = params.uncompressed_kind; + const u64 base = params.mapping_base; + const u64 offset = params.buffer_offset; + const u64 size = params.size; + LOG_DEBUG(Service_NVDRV, "native modify kind base={:#x}, offset={:#x}, size={:#x}, kinds={:#x}/{:#x}", + base, offset, size, requested, fallback); + if (!m_extended_abi || params.reserved != 0 || fallback > 255 || + (requested > 255 && requested != 0xffff) || !size || + offset > std::numeric_limits<u64>::max() - base || + !Common::IsAligned(base + offset, VM::YUZU_PAGESIZE) || + !Common::IsAligned(size, VM::YUZU_PAGESIZE)) { + return NvResult::BadParameter; + } + const auto kind = SelectNativeMemoryKind(static_cast<u8>(requested == 0xffff ? fallback : requested), + static_cast<u8>(fallback)); + if (!kind) { return NvResult::NotSupported; } + const std::scoped_lock lock{mutex}; + const u64 address = base + offset; + auto it = mapping_map.upper_bound(address); + if (!vm.initialised || it == mapping_map.begin()) { return NvResult::BadParameter; } + --it; + const auto& mapping = it->second; + const u64 delta = address - it->first; + if (delta >= mapping.size || size > mapping.size - delta) { return NvResult::BadParameter; } + gmmu->Map(address, mapping.ptr + delta, size, static_cast<Tegra::PTEKind>(kind->backend), + mapping.big_page); + return NvResult::Success; +} + NvResult nvhost_as_gpu::Remap(std::span<IoctlRemapEntry> entries) { LOG_DEBUG(Service_NVDRV, "called, num_entries={:#x}", entries.size()); @@ -414,7 +658,13 @@ NvResult nvhost_as_gpu::UnmapBuffer(IoctlUnmapBuffer& params) { NvResult nvhost_as_gpu::BindChannel(IoctlBindChannel& params) { LOG_DEBUG(Service_NVDRV, "called, fd={:X}", params.fd); - auto gpu_channel_device = module.GetDevice<nvhost_gpu>(params.fd); + const auto device = module.GetDevice<nvdevice>(params.fd); + if (!device || device->GetType() != nvdevice::Type::GpuChannel) { return NvResult::BadParameter; } + const auto gpu_channel_device = std::static_pointer_cast<nvhost_gpu>(device); + if (!vm.initialised || !gmmu || (gpu_channel_device->channel_state->initialized && + gpu_channel_device->channel_state->memory_manager != gmmu.get())) { return NvResult::InvalidState; } + gpu_channel_device->channel_state->memory_manager_owner = + std::shared_ptr<Tegra::MemoryManager>(module.GetDevice<nvhost_as_gpu>(m_fd), gmmu.get()); gpu_channel_device->channel_state->memory_manager = gmmu.get(); return NvResult::Success; } @@ -424,13 +674,13 @@ void nvhost_as_gpu::GetVARegionsImpl(IoctlGetVaRegions& params) { params.regions = std::array<VaRegion, 2>{ VaRegion{ - .offset = vm.small_page_allocator->GetVAStart() << VM::PAGE_SIZE_BITS, + .offset = GpuPageAddress(vm.small_page_allocator->GetVAStart(), VM::PAGE_SIZE_BITS), .page_size = VM::YUZU_PAGESIZE, ._pad0_{}, .pages = vm.small_page_allocator->GetVALimit() - vm.small_page_allocator->GetVAStart(), }, VaRegion{ - .offset = vm.big_page_allocator->GetVAStart() << vm.big_page_size_bits, + .offset = GpuPageAddress(vm.big_page_allocator->GetVAStart(), vm.big_page_size_bits), .page_size = vm.big_page_size, ._pad0_{}, .pages = vm.big_page_allocator->GetVALimit() - vm.big_page_allocator->GetVAStart(), diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h index b13b1c9efa..8c7306e42a 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h @@ -64,6 +64,13 @@ public: Kernel::KEvent* QueryEvent(u32 event_id) override; + NvResult CreateNativeSparseRegion(u64 address, u64 size); + NvResult DestroyNativeSparseRegion(u64 address, u64 size); + NvResult MapNative4K(u32 handle, u64 address, u64 buffer_offset, u64 size, u8 kind); + NvResult UnmapNative4K(u64 address, u64 size); + NvResult GetNativeSyncpointShim(u64& address, u32& stride, u32& count); + NvResult ModifyNativeKind(u64 address, u64 size, u8 requested, u8 fallback); + struct VaRegion { u64 offset; u32 page_size; @@ -145,6 +152,25 @@ private: static_assert(sizeof(IoctlGetVaRegions) == 16 + sizeof(VaRegion) * 2, "IoctlGetVaRegions is incorrect size"); + struct IoctlSyncpointShimInfo { + u64_le address; + u32_le stride; + u32_le count; + }; + static_assert(sizeof(IoctlSyncpointShimInfo) == 16); + NvResult GetSyncpointShimInfo(IoctlSyncpointShimInfo& params); + + struct IoctlModifyKind { + u16_le compressed_kind; + u16_le uncompressed_kind; + u32_le reserved; + u64_le buffer_offset; + u64_le size; + u64_le mapping_base; + }; + static_assert(sizeof(IoctlModifyKind) == 32); + NvResult ModifyKind(IoctlModifyKind& params); + NvResult AllocAsEx(IoctlAllocAsEx& params); NvResult AllocateSpace(IoctlAllocSpace& params); NvResult Remap(std::span<IoctlRemapEntry> params); @@ -196,6 +222,12 @@ private: std::map<u64, Allocation> allocation_map; std::mutex mutex; //!< Locks all AS operations + bool m_extended_abi{}; + bool m_syncpoint_shim_reserved{}; + DeviceFD m_fd{-1}; + Kernel::KProcess* m_owner_process{}; + std::shared_ptr<nvhost_as_gpu> m_shared_parent; + struct VM { static constexpr u32 YUZU_PAGESIZE{0x1000}; static constexpr u32 PAGE_SIZE_BITS{static_cast<u32>(std::countr_zero<u32>(YUZU_PAGESIZE))}; @@ -219,7 +251,7 @@ private: bool initialised{}; } vm; - std::unique_ptr<Tegra::MemoryManager> gmmu; + std::shared_ptr<Tegra::MemoryManager> gmmu; }; } // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp index 14cdd8fda4..6684352b00 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp @@ -4,6 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <algorithm> #include <cstring> #include "common/assert.h" #include "common/logging.h" @@ -11,6 +12,7 @@ #include "core/core_timing.h" #include "core/hle/service/nvdrv/devices/ioctl_serialization.h" #include "core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h" +#include "core/hle/service/nvdrv/devices/native_gpu_characteristics.h" #include "core/hle/service/nvdrv/nvdrv.h" namespace Service::Nvidia::Devices { @@ -94,6 +96,25 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> case 'G': switch (command.cmd) { case 0x5: + if (command.raw == 0xc0e04705) { + constexpr size_t packet_size = 0x10 + NativeGpuCharacteristicsSize; + if (input.size() < packet_size || output.size() < packet_size || + inline_output.size() < NativeGpuCharacteristicsSize) { + return NvResult::BadParameter; + } + IoctlCharacteristics legacy{}; + GetCharacteristics3(legacy, {}); + const auto native = MakeNativeGpuCharacteristics( + std::span<const u8, 0xa0>{reinterpret_cast<const u8*>(&legacy.gc), 0xa0}); + std::fill(output.begin(), output.end(), u8{}); + std::fill(inline_output.begin(), inline_output.end(), u8{}); + std::memcpy(output.data(), input.data(), 0x10); + const u64_le size{NativeGpuCharacteristicsSize}; + std::memcpy(output.data(), &size, sizeof(size)); + std::memcpy(output.data() + 0x10, native.data(), native.size()); + std::memcpy(inline_output.data(), native.data(), native.size()); + return NvResult::Success; + } return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetCharacteristics3, input, output, inline_output); case 0x6: diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp index 1e571be4c9..6b9678e440 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> +#include <fstream> #include <cstring> #include "common/assert.h" #include "common/logging.h" @@ -14,11 +16,13 @@ #include "core/hle/service/nvdrv/core/syncpoint_manager.h" #include "core/hle/service/nvdrv/devices/ioctl_serialization.h" #include "core/hle/service/nvdrv/devices/nvhost_gpu.h" +#include "core/hle/service/nvdrv/devices/native_gpu_context.h" #include "core/hle/service/nvdrv/nvdrv.h" #include "core/memory.h" #include "video_core/control/channel_state.h" #include "video_core/engines/puller.h" #include "video_core/gpu.h" +#include "video_core/memory_manager.h" #include "video_core/host1x/host1x.h" namespace Service::Nvidia::Devices { @@ -33,7 +37,7 @@ Tegra::CommandHeader BuildFenceAction(Tegra::Engines::Puller::FenceOperation op, nvhost_gpu::nvhost_gpu(Core::System& system_, EventInterface& events_interface_, NvCore::Container& core_) - : nvdevice{system_}, events_interface{events_interface_}, core{core_}, + : nvdevice{system_, Type::GpuChannel}, events_interface{events_interface_}, core{core_}, syncpoint_manager{core_.GetSyncpointManager()}, nvmap{core.GetNvMapFile()}, channel_state{system.GPU().AllocateChannel()} { channel_syncpoint = syncpoint_manager.AllocateSyncpoint(false); @@ -128,6 +132,7 @@ NvResult nvhost_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> inpu void nvhost_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) { sessions[fd] = session_id; + channel_state->native_abi = UsesNativeAbi(); } void nvhost_gpu::OnClose(DeviceFD fd) { @@ -280,6 +285,9 @@ NvResult nvhost_gpu::AllocateObjectContext(IoctlAllocObjCtx& params) { s32_le ctx_class_number_index = GetObjectContextClassNumberIndex(static_cast<CtxClasses>(params.class_num)); + if (const auto native_slot = GetNativeGpuContextSlot(params.class_num)) { + if (UsesNativeAbi()) { ctx_class_number_index = *native_slot; } + } if (ctx_class_number_index < 0) { LOG_ERROR(Service_NVDRV, "Invalid class number for object context: {:#x}", params.class_num); @@ -311,13 +319,13 @@ static boost::container::small_vector<Tegra::CommandHeader, 512> BuildWaitComman } static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementCommandList( - NvFence fence) { + NvFence fence, u32 increment_count = 2) { boost::container::small_vector<Tegra::CommandHeader, 512> result{ Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointPayload, 1, Tegra::SubmissionMode::Increasing), {}}; - for (u32 count = 0; count < 2; ++count) { + for (u32 count = 0; count < increment_count; ++count) { result.push_back(Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointOperation, 1, Tegra::SubmissionMode::Increasing)); result.push_back( @@ -328,12 +336,12 @@ static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementC } static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementWithWfiCommandList( - NvFence fence) { + NvFence fence, u32 increment_count = 2) { boost::container::small_vector<Tegra::CommandHeader, 512> result{ Tegra::BuildCommandHeader(Tegra::BufferMethods::WaitForIdle, 1, Tegra::SubmissionMode::Increasing), {}}; - auto increment_list{BuildIncrementCommandList(fence)}; + auto increment_list{BuildIncrementCommandList(fence, increment_count)}; result.insert(result.end(), increment_list.begin(), increment_list.end()); return result; } @@ -351,6 +359,41 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL system.GPU().InitChannel(*channel_state, channel_state->program_id); } + if (UsesNativeAbi() && native_submit_logs++ < 8) { + LOG_DEBUG(Service_NVDRV, "native submit channel={}, initialized={}, entries={}, flags={:#x}, fence={}:{}", + channel_state->bind_id, channel_state->initialized, entries.command_lists.size(), + params.flags.raw, params.fence.id, params.fence.value); + for (size_t i = 0; i < std::min<size_t>(entries.command_lists.size(), 4); ++i) { + const auto& entry = entries.command_lists[i]; + std::array<u32, 16> words{}; + const size_t bytes = std::min<size_t>(entry.size * sizeof(u32), sizeof(words)); + const bool mapped = channel_state->memory_manager && bytes && + channel_state->memory_manager->IsFullyMappedRange(entry.addr, bytes); + if (mapped) { channel_state->memory_manager->ReadBlockUnsafe(entry.addr, words.data(), bytes); } + if (mapped && entry.size > 1000 && std::getenv("EDEN_NATIVE_FRAME_CAPTURE")) { + std::vector<u32> contents(static_cast<size_t>(entry.size)); + channel_state->memory_manager->ReadBlockUnsafe(entry.addr, contents.data(), contents.size() * sizeof(u32)); + std::ofstream dump("native-init-channel-" + std::to_string(channel_state->bind_id) + "-" + std::to_string(i) + ".bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(contents.data()), contents.size() * sizeof(u32)); + } + LOG_DEBUG(Service_NVDRV, "native pushbuffer entry={} raw={:#x}, address={:#x}, words={}, mapped={}, data={:08x}", + i, entry.raw, static_cast<u64>(entry.addr), static_cast<u64>(entry.size), + mapped, fmt::join(words, ",")); + } + } + + if (UsesNativeAbi() && std::getenv("EDEN_NATIVE_FRAME_CAPTURE") && native_submit_logs < 100) { + for (size_t i = 0; i < entries.command_lists.size(); ++i) { + const auto& entry = entries.command_lists[i]; + const auto bytes = static_cast<size_t>(entry.size) * sizeof(u32); + if (!bytes || bytes > 0x100000 || !channel_state->memory_manager->IsFullyMappedRange(entry.addr, bytes)) continue; + std::vector<u32> contents(static_cast<size_t>(entry.size)); + channel_state->memory_manager->ReadBlockUnsafe(entry.addr, contents.data(), bytes); + std::ofstream dump("native-submit-" + std::to_string(channel_state->bind_id) + "-" + std::to_string(native_submit_logs) + "-" + std::to_string(i) + ".bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(contents.data()), bytes); + } + } + const auto bind_id = channel_state->bind_id; auto& flags = params.flags; @@ -365,6 +408,8 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL } } + const u32 native_completion = UsesNativeAbi() && flags.increment_value.Value() && !flags.fence_increment.Value() ? params.fence.value : 0; + if (native_completion > 64) return NvResult::BadParameter; params.fence.id = channel_syncpoint; u32 increment{(flags.fence_increment.Value() != 0 ? 2 : 0) + @@ -382,6 +427,14 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL } } + if (native_completion) { + // The native C56F stream reports its completion count without emitting + // host syncpoint methods. Order actual increments after that GPU work. + gpu.PushGPUEntries(bind_id, Tegra::CommandList{flags.suppress_wfi.Value() + ? BuildIncrementCommandList(params.fence, native_completion) + : BuildIncrementWithWfiCommandList(params.fence, native_completion)}); + LOG_DEBUG(Service_NVDRV, "Native ordered completion: syncpoint={}, increments={}, target={}", channel_syncpoint, native_completion, params.fence.value); + } flags.raw = 0; return NvResult::Success; @@ -463,10 +516,46 @@ Kernel::KEvent* nvhost_gpu::QueryEvent(u32 event_id) { return sm_exception_breakpoint_pause_report_event; case 3: return error_notifier_event; + case 7: + // The native SDK requests this event immediately before configuring + // CHANNEL_SET_ERROR_NOTIFIER, then wraps it in its channel error wait. + return UsesNativeAbi() ? error_notifier_event : nullptr; default: LOG_CRITICAL(Service_NVDRV, "Unknown Ctrl GPU Event {}", event_id); return nullptr; } } +NvResult nvhost_gpu::BindToTsg(DeviceFD group, Kernel::KProcess* process, std::shared_ptr<Tegra::NativeMacroContext> macros) { + const std::scoped_lock lock{tsg_mutex}; + if (tsg_group) { return NvResult::Busy; } + bool owned{}; + for (const auto& [fd, session_id] : sessions) { + const auto* session = core.GetSession(session_id); + owned |= session && session->process == process; + } + if (!owned) { return NvResult::AccessDenied; } + if (channel_state->native_abi) { + channel_state->native_macro_context = std::move(macros); + if (channel_state->payload) { + channel_state->payload->maxwell_3d.SetNativeMacroContext(channel_state->native_macro_context); + } + } + tsg_group = group; + return NvResult::Success; +} + +void nvhost_gpu::UnbindFromTsg(DeviceFD group) { + const std::scoped_lock lock{tsg_mutex}; + if (tsg_group == group) { tsg_group.reset(); } +} + +bool nvhost_gpu::UsesNativeAbi() const { + for (const auto& [fd, session_id] : sessions) { + const auto* session = core.GetSession(session_id); + if (session && session->process && session->process->UsesPAuthReturns()) { return true; } + } + return false; +} + } // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h index 70a6b1bd68..8219e4afea 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h @@ -7,6 +7,8 @@ #pragma once #include <memory> +#include <mutex> +#include <optional> #include <vector> #include "common/bit_field.h" #include "common/common_funcs.h" @@ -20,7 +22,10 @@ namespace Core::Memory { class Memory; } +namespace Kernel { class KProcess; } + namespace Tegra { +struct NativeMacroContext; namespace Control { struct ChannelState; } @@ -58,9 +63,12 @@ public: void OnClose(DeviceFD fd) override; Kernel::KEvent* QueryEvent(u32 event_id) override; + NvResult BindToTsg(DeviceFD group, Kernel::KProcess* process, std::shared_ptr<Tegra::NativeMacroContext> macros); + void UnbindFromTsg(DeviceFD group); private: friend class nvhost_as_gpu; + bool UsesNativeAbi() const; enum class CtxClasses : u32_le { Ctx2D = 0x902D, Ctx3D = 0xB197, @@ -218,6 +226,9 @@ private: NvCore::NvMap& nvmap; std::shared_ptr<Tegra::Control::ChannelState> channel_state; ::Common::unordered_map<DeviceFD, NvCore::SessionId> sessions; + std::mutex tsg_mutex; + std::optional<DeviceFD> tsg_group; + u32 native_submit_logs{}; u32 channel_syncpoint; std::mutex channel_mutex; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.cpp new file mode 100644 index 0000000000..ae9d2a50f5 --- /dev/null +++ b/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.cpp @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <algorithm> +#include <array> +#include <cstring> +#include "common/logging.h" +#include "core/core.h" +#include "video_core/macro.h" +#include "core/hle/kernel/k_process.h" +#include "core/hle/service/nvdrv/nvdrv.h" +#include "core/hle/service/nvdrv/devices/nvhost_gpu.h" +#include "core/hle/service/nvdrv/devices/nvhost_tsg_gpu.h" + +namespace Service::Nvidia::Devices { + +nvhost_tsg_gpu::nvhost_tsg_gpu(Core::System& system, Module& module_, NvCore::Container& core_) + : nvdevice{system, Type::GpuTsg}, native_macros{std::make_shared<Tegra::NativeMacroContext>()}, module{module_}, core{core_} { native_macros->engine.native_turing = true; native_macros->compute_engine.native_turing = true; } + +nvhost_tsg_gpu::~nvhost_tsg_gpu() { OnClose(0); } + +void nvhost_tsg_gpu::OnOpen(NvCore::SessionId id, DeviceFD fd) { + group_fd = fd; + if (const auto* session = core.GetSession(id)) { + process = session->process; + process->Open(system.Kernel()); + } +} + +void nvhost_tsg_gpu::OnClose(DeviceFD fd) { + for (const auto& [channel_fd, channel] : channels) { channel->UnbindFromTsg(group_fd); } + channels.clear(); + if (process) { + process->Close(system.Kernel()); + process = nullptr; + } +} + + +NvResult nvhost_tsg_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input, + std::span<u8> output) { + std::array<u32, 8> words{}; + if (!input.empty()) { std::memcpy(words.data(), input.data(), std::min(input.size(), sizeof(words))); } + LOG_DEBUG(Service_NVDRV, "native TSG ioctl={:#x}, input={}, output={}, words={:#x},{:#x},{:#x},{:#x},{:#x},{:#x},{:#x},{:#x}", + command.raw, input.size(), output.size(), words[0], words[1], words[2], words[3], + words[4], words[5], words[6], words[7]); + if (command.raw != 0xc0045401 && command.raw != 0xc0045402 && command.raw != 0xc018540b) { return NvResult::NotImplemented; } + const size_t packet_size = command.raw == 0xc018540b ? 24 : 4; + if (!process || input.size() != packet_size || output.size() != packet_size) { + return NvResult::BadParameter; + } + if (packet_size == 24 && (words[1] >= 64 || words[2] || words[3] || words[4] || words[5])) { + return NvResult::BadParameter; + } + const DeviceFD channel_fd = static_cast<s32>(words[0]); + if (command.cmd == 1 || command.cmd == 11) { + if (channels.contains(channel_fd)) { return NvResult::Busy; } + if (channels.size() >= 64) { return NvResult::InsufficientMemory; } + const auto device = module.GetDevice<nvdevice>(channel_fd); + if (!device || device->GetType() != Type::GpuChannel) { return NvResult::BadParameter; } + const auto channel = std::static_pointer_cast<nvhost_gpu>(device); + const auto result = channel->BindToTsg(fd, process, native_macros); + if (result != NvResult::Success) { return result; } + channels.emplace(channel_fd, channel); + } else { + const auto it = channels.find(channel_fd); + if (it == channels.end()) { return NvResult::BadParameter; } + it->second->UnbindFromTsg(fd); + channels.erase(it); + } + std::copy(input.begin(), input.end(), output.begin()); + return NvResult::Success; +} + +NvResult nvhost_tsg_gpu::Ioctl2(DeviceFD fd, Ioctl command, std::span<const u8> input, + std::span<const u8> inline_input, std::span<u8> output) { + return NvResult::NotImplemented; +} + +NvResult nvhost_tsg_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> input, + std::span<u8> output, std::span<u8> inline_output) { + return NvResult::NotImplemented; +} + +} // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.h new file mode 100644 index 0000000000..d64c5df3e0 --- /dev/null +++ b/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.h @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <map> +#include <memory> +#include "core/hle/service/nvdrv/devices/nvdevice.h" + +namespace Tegra { struct NativeMacroContext; } +namespace Kernel { class KProcess; } +namespace Service::Nvidia { class Module; } + +namespace Service::Nvidia::Devices { +class nvhost_gpu; + +class nvhost_tsg_gpu final : public nvdevice { +public: + nvhost_tsg_gpu(Core::System& system, Module& module, NvCore::Container& core); + ~nvhost_tsg_gpu() override; + NvResult Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input, + std::span<u8> output) override; + NvResult Ioctl2(DeviceFD fd, Ioctl command, std::span<const u8> input, + std::span<const u8> inline_input, std::span<u8> output) override; + NvResult Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> input, + std::span<u8> output, std::span<u8> inline_output) override; + void OnOpen(NvCore::SessionId session, DeviceFD fd) override; + void OnClose(DeviceFD fd) override; +private: + std::shared_ptr<Tegra::NativeMacroContext> native_macros; + Module& module; + NvCore::Container& core; + Kernel::KProcess* process{}; + DeviceFD group_fd{-1}; + std::map<DeviceFD, std::shared_ptr<nvhost_gpu>> channels; +}; + +} // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/nvdrv.cpp b/src/core/hle/service/nvdrv/nvdrv.cpp index b252a2efa5..9873d07c24 100644 --- a/src/core/hle/service/nvdrv/nvdrv.cpp +++ b/src/core/hle/service/nvdrv/nvdrv.cpp @@ -18,6 +18,8 @@ #include "core/hle/service/nvdrv/devices/nvhost_ctrl.h" #include "core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h" #include "core/hle/service/nvdrv/devices/nvhost_gpu.h" +#include "core/hle/service/nvdrv/devices/nvhost_tsg_gpu.h" +#include "core/hle/kernel/k_process.h" #include "core/hle/service/nvdrv/devices/nvhost_nvdec.h" #include "core/hle/service/nvdrv/devices/nvhost_nvdec_common.h" #include "core/hle/service/nvdrv/devices/nvhost_nvjpg.h" @@ -133,6 +135,10 @@ Module::Module(Core::System& system) auto device = std::make_shared<Devices::nvhost_gpu>(system, events_interface, container); return open_files.emplace(fd, std::move(device)).first; }; + builders["/dev/nvhost-tsg-gpu"] = [this, &system](DeviceFD fd) { + auto device = std::make_shared<Devices::nvhost_tsg_gpu>(system, *this, container); + return open_files.emplace(fd, std::move(device)).first; + }; builders["/dev/nvhost-ctrl-gpu"] = [this, &system](DeviceFD fd) { auto device = std::make_shared<Devices::nvhost_ctrl_gpu>(system, events_interface); return open_files.emplace(fd, std::move(device)).first; @@ -163,7 +169,12 @@ Module::Module(Core::System& system) }; } -Module::~Module() {} +Module::~Module() { + // Devices (including TSG-retained channels) release events and core state. + // Destroy them while those module members are still alive. + for (const auto& [fd, device] : open_files) { device->OnClose(fd); } + open_files.clear(); +} NvResult Module::VerifyFD(DeviceFD fd) const { if (fd < 0) { @@ -180,6 +191,12 @@ NvResult Module::VerifyFD(DeviceFD fd) const { } DeviceFD Module::Open(const std::string& device_name, NvCore::SessionId session_id) { + if (device_name == "/dev/nvhost-tsg-gpu") { + const auto* session = container.GetSession(session_id); + if (!session || !session->process || !session->process->UsesPAuthReturns()) { + return INVALID_NVDRV_FD; + } + } auto it = builders.find(device_name); if (it == builders.end()) { LOG_ERROR(Service_NVDRV, "Trying to open unknown device {}", device_name); diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.cpp b/src/core/hle/service/nvdrv/nvdrv_interface.cpp index 5fae68adbb..3f81720a39 100644 --- a/src/core/hle/service/nvdrv/nvdrv_interface.cpp +++ b/src/core/hle/service/nvdrv/nvdrv_interface.cpp @@ -72,6 +72,10 @@ void NVDRV::Ioctl1(HLERequestContext& ctx) { const auto input_buffer = ctx.ReadBuffer(0); const auto nv_result = nvdrv->Ioctl1(fd, command, input_buffer, output_buffer); + if (nv_result != NvResult::Success && ctx.GetThread().GetOwnerProcess()->UsesPAuthReturns()) { + LOG_DEBUG(Service_NVDRV, "native ioctl failure fd={}, ioctl={:#x}, result={}, caller={:#x}", + fd, command.raw, static_cast<u32>(nv_result), ctx.GetThread().GetContext().lr); + } if (command.is_out != 0) { ctx.WriteBuffer(output_buffer); } @@ -99,6 +103,10 @@ void NVDRV::Ioctl2(HLERequestContext& ctx) { const auto nv_result = nvdrv->Ioctl2(fd, command, input_buffer, input_inlined_buffer, output_buffer); + if (nv_result != NvResult::Success && ctx.GetThread().GetOwnerProcess()->UsesPAuthReturns()) { + LOG_DEBUG(Service_NVDRV, "native ioctl failure fd={}, ioctl={:#x}, result={}, caller={:#x}", + fd, command.raw, static_cast<u32>(nv_result), ctx.GetThread().GetContext().lr); + } if (command.is_out != 0) { ctx.WriteBuffer(output_buffer); } @@ -126,6 +134,11 @@ void NVDRV::Ioctl3(HLERequestContext& ctx) { const auto nv_result = nvdrv->Ioctl3(fd, command, input_buffer, output_buffer, inline_output_buffer); + if (command.raw == 0xc0e04705) { + LOG_DEBUG(Service_NVDRV, "native characteristics input={}, output={}, inline={}, result={}", + input_buffer.size(), output_buffer.size(), inline_output_buffer.size(), + static_cast<u32>(nv_result)); + } if (command.is_out != 0) { ctx.WriteBuffer(output_buffer, 0); ctx.WriteBuffer(inline_output_buffer, 1); @@ -185,6 +198,10 @@ void NVDRV::QueryEvent(HLERequestContext& ctx) { IPC::RequestParser rp{ctx}; const auto fd = rp.Pop<DeviceFD>(); const auto event_id = rp.Pop<u32>(); + if (ctx.GetThread().GetOwnerProcess()->UsesPAuthReturns()) { + LOG_DEBUG(Service_NVDRV, "native QueryEvent fd={}, id={}, caller={:#x}", fd, event_id, + ctx.GetThread().GetContext().lr); + } if (!is_initialized) { ServiceError(ctx, NvResult::NotInitialized); @@ -265,6 +282,14 @@ NVDRV::NVDRV(Core::System& system_, std::shared_ptr<Module> nvdrv_, const char* RegisterHandlers(functions); } +NVDRV::NVDRV(Core::System& system, std::shared_ptr<Module> module, const char* name, + Kernel::KProcess* process, u64 aruid) + : NVDRV{system, std::move(module), name} { + session_id = nvdrv->GetContainer().OpenSession(process); + pid = aruid; + is_initialized = true; +} + NVDRV::~NVDRV() { if (is_initialized) { auto& container = nvdrv->GetContainer(); diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.h b/src/core/hle/service/nvdrv/nvdrv_interface.h index f628fca7df..d6fe4c18f5 100644 --- a/src/core/hle/service/nvdrv/nvdrv_interface.h +++ b/src/core/hle/service/nvdrv/nvdrv_interface.h @@ -17,6 +17,8 @@ namespace Service::Nvidia { class NVDRV final : public ServiceFramework<NVDRV> { public: explicit NVDRV(Core::System& system_, std::shared_ptr<Module> nvdrv_, const char* name); + explicit NVDRV(Core::System& system, std::shared_ptr<Module> module, const char* name, + Kernel::KProcess* process, u64 aruid); ~NVDRV() override; std::shared_ptr<Module> GetModule() const { diff --git a/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp b/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp index c3eb2796e3..9de8abaeb6 100644 --- a/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp +++ b/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp @@ -393,6 +393,15 @@ Status BufferQueueProducer::DetachNextBuffer(std::shared_ptr<GraphicBuffer>* out return Status::NoError; } +std::optional<Fence> BufferQueueProducer::GetReleasedBufferFence(s32 slot) { + const std::scoped_lock lock{core->mutex}; + if (slot < 0 || static_cast<size_t>(slot) >= slots.size() || + slots[slot].buffer_state != BufferState::Free) { + return std::nullopt; + } + return slots[slot].fence; +} + Status BufferQueueProducer::AttachBuffer(s32* out_slot, const std::shared_ptr<GraphicBuffer>& buffer) { if (out_slot == nullptr) { diff --git a/src/core/hle/service/nvnflinger/buffer_queue_producer.h b/src/core/hle/service/nvnflinger/buffer_queue_producer.h index 697de0ac9b..bf815eb8d8 100644 --- a/src/core/hle/service/nvnflinger/buffer_queue_producer.h +++ b/src/core/hle/service/nvnflinger/buffer_queue_producer.h @@ -12,6 +12,7 @@ #include <condition_variable> #include <memory> #include <mutex> +#include <optional> #include "common/common_funcs.h" #include "common/cpu_features.h" @@ -57,6 +58,7 @@ public: Kernel::KReadableEvent* GetNativeHandle(u32 type_id) override; public: + std::optional<Fence> GetReleasedBufferFence(s32 slot); Status RequestBuffer(s32 slot, std::shared_ptr<GraphicBuffer>* buf); Status SetBufferCount(s32 buffer_count); Status DequeueBuffer(s32* out_slot, android::Fence* out_fence, bool async, u32 width, diff --git a/src/core/hle/service/service.cpp b/src/core/hle/service/service.cpp index b4fe2b3ed5..a7e7e10199 100644 --- a/src/core/hle/service/service.cpp +++ b/src/core/hle/service/service.cpp @@ -4,12 +4,16 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <chrono> +#include <limits> #include <fmt/ranges.h> #include "common/assert.h" #include "common/logging.h" #include "common/settings.h" #include "core/core.h" #include "core/hle/ipc.h" +#include "core/hle/kernel/k_process.h" +#include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/kernel.h" #include "core/hle/service/ipc_helpers.h" #include "core/hle/service/service.h" @@ -67,6 +71,26 @@ void ServiceFrameworkBase::ReportUnimplementedFunction(HLERequestContext& ctx, fmt::format_to(std::back_inserter(buf), ", [{}]={:#x}", i, cmd_buf[i]); buf.push_back('}'); + if (ctx.GetThread().GetOwnerProcess()->UsesPAuthReturns()) { + const auto& context = ctx.GetThread().GetContext(); + LOG_DEBUG(Service, "Unsupported IPC caller pc={:#x}, lr={:#x}, sp={:#x}, fp={:#x}", + context.pc, context.lr, context.sp, context.fp); + u64 frame = context.fp; + for (u32 depth = 0; depth < 8 && frame != 0; ++depth) { + if (frame > (std::numeric_limits<u64>::max)() - 15 || + !ctx.GetMemory().IsValidVirtualAddress(frame) || + !ctx.GetMemory().IsValidVirtualAddress(frame + 15)) { + break; + } + const u64 next = ctx.GetMemory().Read64(frame); + LOG_DEBUG(Service, "Unsupported IPC frame {}: return={:#x}", depth, + ctx.GetMemory().Read64(frame + 8)); + if (next <= frame) { + break; + } + frame = next; + } + } system.GetReporter().SaveUnimplementedFunctionReport(ctx, ctx.GetCommand(), function_name, service_name); UNIMPLEMENTED_MSG("Unknown / unimplemented {}", fmt::to_string(buf)); if (Settings::values.use_auto_stub) { @@ -82,7 +106,7 @@ void ServiceFrameworkBase::InvokeRequest(HLERequestContext& ctx) { if (info == nullptr || info->handler_callback == nullptr) return ReportUnimplementedFunction(ctx, info); - LOG_TRACE(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer())); + LOG_DEBUG(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer())); handler_invoker(this, info->handler_callback, ctx); } @@ -92,7 +116,7 @@ void ServiceFrameworkBase::InvokeRequestTipc(HLERequestContext& ctx) { if (info == nullptr || info->handler_callback == nullptr) return ReportUnimplementedFunction(ctx, info); - LOG_TRACE(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer())); + LOG_DEBUG(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer())); handler_invoker(this, info->handler_callback, ctx); } diff --git a/src/core/launch_params.cpp b/src/core/launch_params.cpp index a0483226ec..814a2cb553 100644 --- a/src/core/launch_params.cpp +++ b/src/core/launch_params.cpp @@ -94,7 +94,7 @@ LaunchParams ParseLaunchParams(Core::System& system, int argc, char *argv[], wch break; case 'u': { // Launch game with a specific user - bool argument_ok = isdigit(optarg[0]); + bool argument_ok = isdigit(static_cast<unsigned char>(optarg[0])) != 0; p.selected_user = atoi(optarg); if (!argument_ok) { // try to look it up by username, only finds the first username that matches. diff --git a/src/core/loader/deconstructed_rom_directory.cpp b/src/core/loader/deconstructed_rom_directory.cpp index 5e2ec7e0b9..1dce245f7f 100644 --- a/src/core/loader/deconstructed_rom_directory.cpp +++ b/src/core/loader/deconstructed_rom_directory.cpp @@ -13,6 +13,7 @@ #include "core/file_sys/control_metadata.h" #include "core/file_sys/patch_manager.h" #include "core/file_sys/romfs_factory.h" +#include "core/file_sys/romfs.h" #include "core/hle/kernel/k_page_table.h" #include "core/hle/kernel/k_process.h" #include "core/hle/service/filesystem/filesystem.h" @@ -129,10 +130,14 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory( : AppLoader(directory->GetFile("main")) , dir(std::move(directory)) , override_update(override_update_) + , standalone(false) {} FileType AppLoader_DeconstructedRomDirectory::IdentifyType(const FileSys::VirtualFile& dir_file) { - if (FileSys::IsDirectoryExeFS(dir_file->GetContainingDirectory())) { + const auto directory = dir_file->GetContainingDirectory(); + if (dir_file->GetName() == "main" && directory && + AppLoader_NSO::IdentifyType(dir_file) == FileType::NSO && + FileSys::IsDirectoryExeFS(directory)) { return FileType::DeconstructedRomDirectory; } @@ -182,11 +187,11 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect } metadata.Print(); - // Enable NCE only for applications with 39-bit address space. + // Extended ACID programs currently require the Dynarmic compatibility path. const bool is_39bit = metadata.GetAddressSpaceType() == FileSys::ProgramAddressSpaceType::Is39Bit; const bool is_application = metadata.GetPoolPartition() == FileSys::PoolPartition::Application; - Settings::SetNceEnabled(is_39bit); + Settings::SetNceEnabled(is_39bit && !metadata.UsesExtendedAcid()); const std::array static_modules = {"rtld", "main", "subsdk0", "subsdk1", "subsdk2", "subsdk3", "subsdk4", "subsdk5", "subsdk6", "subsdk7", @@ -267,11 +272,39 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect } is_loaded = true; + if (standalone) { + title_id = metadata.GetTitleID(); + system.GetFileSystemController().RegisterProcess( + process.GetProcessId(), {}, + std::make_shared<FileSys::RomFSFactory>(*this, system.GetContentProvider(), + system.GetFileSystemController())); + } return {ResultStatus::Success, LoadParameters{metadata.GetMainThreadPriority(), metadata.GetMainThreadStackSize()}}; } ResultStatus AppLoader_DeconstructedRomDirectory::ReadRomFS(FileSys::VirtualFile& out_dir) { + if (romfs == nullptr && standalone) { + const auto directory = file->GetContainingDirectory(); + for (const auto& entry : directory->GetFiles()) { + if (entry->GetExtension() == "romfs") { + romfs = entry; + break; + } + } + if (romfs == nullptr) { + auto loose = directory->GetSubdirectory("romfs"); + if (!loose && directory->GetName() == "exefs") { + const auto parent = directory->GetParentDirectory(); + if (parent) { + loose = parent->GetSubdirectory("romfs"); + } + } + if (loose) { + romfs = FileSys::CreateRomFS(std::move(loose)); + } + } + } if (romfs == nullptr) { return ResultStatus::ErrorNoRomFS; } diff --git a/src/core/loader/deconstructed_rom_directory.h b/src/core/loader/deconstructed_rom_directory.h index 7283db0662..aba89935a3 100644 --- a/src/core/loader/deconstructed_rom_directory.h +++ b/src/core/loader/deconstructed_rom_directory.h @@ -62,6 +62,7 @@ private: std::string name; u64 title_id{}; bool override_update; + bool standalone{true}; Modules modules; }; diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp index 92e2e2f602..d802d15e9f 100644 --- a/src/core/loader/loader.cpp +++ b/src/core/loader/loader.cpp @@ -125,6 +125,10 @@ bool HasApplicationProgramContent(const std::shared_ptr<FileSys::NSP>& nsp) { } // namespace FileType IdentifyFile(FileSys::VirtualFile file) { + if (const auto romdir_type = + IdentifyFileLoader<AppLoader_DeconstructedRomDirectory>(file)) { + return *romdir_type; + } if (const auto nsp_type = IdentifyFileLoader<AppLoader_NSP>(file)) { return *nsp_type; } else if (const auto xci_type = IdentifyFileLoader<AppLoader_XCI>(file)) { @@ -139,9 +143,6 @@ FileType IdentifyFile(FileSys::VirtualFile file) { return *kip_type; } else if (const auto nso_type = IdentifyFileLoader<AppLoader_NSO>(file)) { return *nso_type; - } else if (const auto romdir_type = - IdentifyFileLoader<AppLoader_DeconstructedRomDirectory>(file)) { - return *romdir_type; } else { return FileType::Unknown; } diff --git a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp index 492242fa73..af1d202edc 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp @@ -367,7 +367,11 @@ A64AddressSpace::A64AddressSpace(const A64::UserConfig& conf) void A64AddressSpace::GenerateIR(IR::Block& ir_block, IR::LocationDescriptor descriptor) const { ir_block.Reset(descriptor); const auto get_code = [this](u64 vaddr) { return conf.callbacks->MemoryReadCode(vaddr); }; - A64::Translate(ir_block, A64::LocationDescriptor{descriptor}, get_code, {conf.define_unpredictable_behaviour, conf.wall_clock_cntpct}); + A64::Translate(ir_block, A64::LocationDescriptor{descriptor}, get_code, + {.define_unpredictable_behaviour = conf.define_unpredictable_behaviour, + .wall_clock_cntpct = conf.wall_clock_cntpct, + .enable_pauth_returns = conf.enable_pauth_returns, + .address_space_bits = conf.page_table_address_space_bits}); Optimization::Optimize(ir_block, conf, {}); } diff --git a/src/dynarmic/src/dynarmic/backend/x64/a64_interface.cpp b/src/dynarmic/src/dynarmic/backend/x64/a64_interface.cpp index bb072129e7..0d5a7d3498 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/a64_interface.cpp +++ b/src/dynarmic/src/dynarmic/backend/x64/a64_interface.cpp @@ -258,7 +258,11 @@ private: // LocationDescriptor ctor() does important ops (like tflags) do not skip auto const arch_descriptor = A64::LocationDescriptor{descriptor}; ir_block.Reset(arch_descriptor); - A64::Translate(ir_block, arch_descriptor, get_code, {conf.define_unpredictable_behaviour, conf.wall_clock_cntpct}); + A64::Translate(ir_block, arch_descriptor, get_code, + {.define_unpredictable_behaviour = conf.define_unpredictable_behaviour, + .wall_clock_cntpct = conf.wall_clock_cntpct, + .enable_pauth_returns = conf.enable_pauth_returns, + .address_space_bits = conf.page_table_address_space_bits}); Optimization::Optimize(ir_block, conf, polyfill_options); return emitter.Emit(ir_block).entrypoint; } diff --git a/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc b/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc index 8a3fab7ac1..0301a007a1 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc +++ b/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc @@ -34,6 +34,16 @@ INST(IC_IVAU, "IC IVAU", "110101010000101101110101001ttttt") INST(BLR, "BLR", "1101011000111111000000nnnnn00000") INST(BR, "BR", "1101011000011111000000nnnnn00000") INST(RET, "RET", "1101011001011111000000nnnnn00000") +INST(RETA, "RETA", "110101100101111100001M1111111111") +INST(CAS, "CAS", "zz0010001L1ssssso11111nnnnnttttt") +INST(LDAPR, "LDAPR", "ss11100010111111110000nnnnnttttt") +INST(LDADD, "LDADD", "1z111000AR1sssss000000nnnnnttttt") +INST(LDCLR, "LDCLR", "1z111000AR1sssss000100nnnnnttttt") +INST(LDSET, "LDSET", "1z111000AR1sssss001100nnnnnttttt") +INST(SWPB, "SWPB", "00111000AR1sssss100000nnnnnttttt") +INST(SWPH, "SWPH", "01111000AR1sssss100000nnnnnttttt") +INST(SWP, "SWP", "1z111000AR1sssss100000nnnnnttttt") +INST(XPAC_1, "XPAC", "110110101100000101000D11111ddddd") INST(HINT, "HINT", "11010101000000110010MMMMooo11111") INST(SXTB_1, "SXTB (32-bit)", "0001001100000000000111nnnnnddddd") INST(SXTB_2, "SXTB (64-bit)", "1001001101000000000111nnnnnddddd") diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.h b/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.h index 61033eb530..5acb50df74 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.h +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.h @@ -38,6 +38,9 @@ struct TranslationOptions { /// If this is false, we treat the instruction as a NOP. /// If this is true, we emit an ExceptionRaised instruction. bool hook_hint_instructions = true; + + bool enable_pauth_returns = false; + u32 address_space_bits = 64; }; /** diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/a64_branch.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/a64_branch.cpp index 36b91f13e9..4cb89f5e86 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/a64_branch.cpp +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/a64_branch.cpp @@ -70,6 +70,14 @@ bool TranslatorVisitor::RET(Reg Rn) { return false; } +bool TranslatorVisitor::RETA([[maybe_unused]] bool M) { + if (!options.enable_pauth_returns) { + return UnallocatedEncoding(); + } + // With SCTLR.EnIA/EnIB clear, authentication leaves LR unchanged. + return RET(Reg::R30); +} + bool TranslatorVisitor::CBZ(bool sf, Imm<19> imm19, Reg Rt) { const size_t datasize = sf ? 64 : 32; const s64 offset = concatenate(imm19, Imm<2>{0}).SignExtend<s64>(); diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h index c8afef3283..fdc9fe6ac1 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h @@ -251,7 +251,7 @@ struct TranslatorVisitor final { bool CASP(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt); bool CASB(bool L, Reg Rs, bool o0, Reg Rn, Reg Rt); bool CASH(bool L, Reg Rs, bool o0, Reg Rn, Reg Rt); - bool CAS(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt); + bool CAS(Imm<2> sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt); // Loads and stores - Load register (literal) bool LDR_lit_gen(bool opc_0, Imm<19> imm19, Reg Rt); @@ -315,16 +315,16 @@ struct TranslatorVisitor final { bool LDUMINH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); bool SWPH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); bool LDAPRH(Reg Rn, Reg Rt); - bool LDADD(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); - bool LDCLR(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); + bool LDADD(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt); + bool LDCLR(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt); bool LDEOR(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); - bool LDSET(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); + bool LDSET(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt); bool LDSMAX(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); bool LDSMIN(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); bool LDUMAX(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); bool LDUMIN(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); - bool SWP(bool A, bool R, Reg Rs, Reg Rn, Reg Rt); - bool LDAPR(Reg Rn, Reg Rt); + bool SWP(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt); + bool LDAPR(Imm<2> size, Reg Rn, Reg Rt); // Loads and stores - Load/Store register (register offset) bool STRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, Reg Rt); diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp index 0e67e11b4b..20ab71623f 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp @@ -6,9 +6,117 @@ #include <optional> #include "dynarmic/frontend/A64/translate/impl/impl.h" +#include "dynarmic/ir/terminal.h" namespace Dynarmic::A64 { +bool TranslatorVisitor::CAS(Imm<2> sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) { + const size_t bits = 8U << sz.ZeroExtend<size_t>(); + const size_t reg_bits = bits == 64 ? 64 : 32; + const IR::U64 address = Rn == Reg::SP ? IR::U64{SP(64)} : IR::U64{X(64, Rn)}; + const IR::U64 original_expected = X(64, Rs); + const IR::U32U64 expected = ZeroExtend(X(bits, Rs), reg_bits); + const IR::U32U64 desired = ZeroExtend(X(bits, Rt), reg_bits); + const auto flags = ir.GetNZCVRaw(); + const auto access = L || o0 ? IR::AccType::ORDERED : IR::AccType::ATOMIC; + const auto observed = ZeroExtend(ExclusiveMem(address, bits / 8, access), reg_bits); + + ir.SetNZCV(ir.NZCVFrom(ir.SubWithCarry(observed, expected, ir.Imm1(1)))); + const auto replacement = ir.ConditionalSelect(IR::Cond::EQ, desired, observed); + ir.SetNZCVRaw(flags); + // A failed comparison may atomically write back the observed value. + IR::UAny store_value = replacement; + if (bits == 8) { + store_value = ir.LeastSignificantByte(replacement); + } else if (bits == 16) { + store_value = ir.LeastSignificantHalf(replacement); + } + const auto status = ExclusiveMem(address, bits / 8, access, store_value); + ir.SetNZCV(ir.NZCVFrom(ir.SubWithCarry(status, ir.Imm32(0), ir.Imm1(1)))); + X(64, Rs, ir.ConditionalSelect(IR::Cond::EQ, ir.ZeroExtendToLong(observed), original_expected)); + ir.SetNZCVRaw(flags); + ir.ClearExclusive(); + + // A store-exclusive failure must retry without changing the expected value. + ir.SetCheckBit(ir.IsZero(status)); + ir.SetTerm(IR::Term::CheckBit{ + IR::Term::LinkBlock{ir.current_location->AdvancePC(4)}, + IR::Term::LinkBlock{*ir.current_location}}); + return false; +} + +bool TranslatorVisitor::LDAPR(Imm<2> size, Reg Rn, Reg Rt) { + // LDAR provides ordering at least as strong as an RCpc acquire load. + return LDAR(size, Rn, Rt); +} + +enum class AtomicRegisterOperation { Add, Set, Clear, Swap }; + +static bool AtomicRegisterOperationImpl(TranslatorVisitor& v, size_t bits, bool A, bool R, + Reg Rs, Reg Rn, Reg Rt, AtomicRegisterOperation operation) { + const size_t reg_bits = bits == 64 ? 64 : 32; + const IR::U64 address = Rn == Reg::SP ? IR::U64{v.SP(64)} : IR::U64{v.X(64, Rn)}; + const IR::U64 original_result = v.X(64, Rt); + const IR::U32U64 operand = v.ZeroExtend(v.X(bits, Rs), reg_bits); + const auto flags = v.ir.GetNZCVRaw(); + const auto access = A || R ? IR::AccType::ORDERED : IR::AccType::ATOMIC; + const auto observed = v.ZeroExtend(v.ExclusiveMem(address, bits / 8, access), reg_bits); + auto replacement = operand; + switch (operation) { + case AtomicRegisterOperation::Add: + replacement = v.ir.Add(observed, operand); + break; + case AtomicRegisterOperation::Set: + replacement = v.ir.Or(observed, operand); + break; + case AtomicRegisterOperation::Clear: + replacement = v.ir.And(observed, v.ir.Not(operand)); + break; + case AtomicRegisterOperation::Swap: + break; + } + IR::UAny store_value = replacement; + if (bits == 8) { + store_value = v.ir.LeastSignificantByte(replacement); + } else if (bits == 16) { + store_value = v.ir.LeastSignificantHalf(replacement); + } + const auto status = v.ExclusiveMem(address, bits / 8, access, store_value); + v.ir.SetNZCV(v.ir.NZCVFrom(v.ir.SubWithCarry(status, v.ir.Imm32(0), v.ir.Imm1(1)))); + v.X(64, Rt, v.ir.ConditionalSelect(IR::Cond::EQ, v.ir.ZeroExtendToLong(observed), original_result)); + v.ir.SetNZCVRaw(flags); + v.ir.ClearExclusive(); + v.ir.SetCheckBit(v.ir.IsZero(status)); + v.ir.SetTerm(IR::Term::CheckBit{ + IR::Term::LinkBlock{v.ir.current_location->AdvancePC(4)}, + IR::Term::LinkBlock{*v.ir.current_location}}); + return false; +} + +bool TranslatorVisitor::LDADD(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { + return AtomicRegisterOperationImpl(*this, sz ? 64 : 32, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Add); +} + +bool TranslatorVisitor::LDCLR(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { + return AtomicRegisterOperationImpl(*this, sz ? 64 : 32, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Clear); +} + +bool TranslatorVisitor::LDSET(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { + return AtomicRegisterOperationImpl(*this, sz ? 64 : 32, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Set); +} + +bool TranslatorVisitor::SWPB(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { + return AtomicRegisterOperationImpl(*this, 8, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Swap); +} + +bool TranslatorVisitor::SWPH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { + return AtomicRegisterOperationImpl(*this, 16, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Swap); +} + +bool TranslatorVisitor::SWP(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { + return AtomicRegisterOperationImpl(*this, sz ? 64 : 32, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Swap); +} + static bool ExclusiveSharedDecodeAndOperation(TranslatorVisitor& v, bool pair, size_t size, bool L, bool o0, std::optional<Reg> Rs, std::optional<Reg> Rt2, Reg Rn, Reg Rt) { // Shared Decode diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/system.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/system.cpp index c60fdc3ae0..8425493e3f 100644 --- a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/system.cpp +++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/system.cpp @@ -34,6 +34,27 @@ enum class SystemRegisterEncoding : u32 { }; bool TranslatorVisitor::HINT([[maybe_unused]] Imm<4> CRm, [[maybe_unused]] Imm<3> op2) { + if (options.enable_pauth_returns && CRm == 0 && op2 == 7) { + return XPAC_1(false, Reg::R30); + } + return true; +} + +bool TranslatorVisitor::XPAC_1([[maybe_unused]] bool D, Reg Rd) { + if (!options.enable_pauth_returns) { + return UnallocatedEncoding(); + } + const auto bits = options.address_space_bits; + if (bits >= 64) { + return true; + } + const IR::U64 pointer = X(64, Rd); + const u64 address_mask = (u64{1} << bits) - 1; + // This compatibility model has TBI disabled; bit 55 selects the address extension. + const auto extension = ir.ArithmeticShiftRight( + ir.LogicalShiftLeft(pointer, ir.Imm8(8)), ir.Imm8(63)); + X(64, Rd, ir.Or(ir.And(pointer, ir.Imm64(address_mask)), + ir.And(extension, ir.Imm64(~address_mask)))); return true; } diff --git a/src/dynarmic/src/dynarmic/interface/A64/config.h b/src/dynarmic/src/dynarmic/interface/A64/config.h index 859aeba709..6220f5c95e 100644 --- a/src/dynarmic/src/dynarmic/interface/A64/config.h +++ b/src/dynarmic/src/dynarmic/interface/A64/config.h @@ -242,6 +242,10 @@ struct UserConfig { /// instruction is executed. bool hook_hint_instructions = false; + /// Expose authenticated returns and PAC stripping with authentication keys disabled. + /// This does not generate or verify PACs. PAC/AUT hint instructions remain NOPs. + bool enable_pauth_returns = false; + /// Determines what happens if the guest accesses an entry that is off the end of the /// page table. If true, Dynarmic will silently mirror page_table's address space. If /// false, accessing memory outside of page_table bounds will result in a call to the diff --git a/src/dynarmic/tests/A64/lse.cpp b/src/dynarmic/tests/A64/lse.cpp new file mode 100644 index 0000000000..e26b03c321 --- /dev/null +++ b/src/dynarmic/tests/A64/lse.cpp @@ -0,0 +1,526 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <catch2/catch_test_macros.hpp> +#include <catch2/generators/catch_generators.hpp> + +#include "dynarmic/tests/A64/testenv.h" +#include "dynarmic/interface/exclusive_monitor.h" + +namespace { +class AtomicTestEnv final : public A64TestEnv { +public: + Dynarmic::ExclusiveMonitor monitor{1}; + bool fail_once{}; + std::optional<u64> failure_value; + unsigned writes{}; + + bool MemoryWriteExclusive8(u64 address, u8 value, u8 expected) override { + return WriteSmallExclusive(address, value, expected, 1); + } + + bool MemoryWriteExclusive16(u64 address, u16 value, u16 expected) override { + return WriteSmallExclusive(address, value, expected, 2); + } + + bool WriteSmallExclusive(u64 address, u64 value, u64 expected, size_t bytes) { + ++writes; + if (fail_once) { + fail_once = false; + if (failure_value) { + MemoryWrite(address, *failure_value, bytes); + } + return false; + } + if (MemoryRead(address, bytes) != expected) { + return false; + } + MemoryWrite(address, value, bytes); + return true; + } + + bool MemoryWriteExclusive32(u64 address, u32 value, u32 expected) override { + ++writes; + if (fail_once) { + fail_once = false; + if (failure_value) { + MemoryWrite(address, *failure_value, 4); + } + return false; + } + if (MemoryRead(address, 4) != expected) { + return false; + } + MemoryWrite(address, value, 4); + return true; + } + + bool MemoryWriteExclusive64(u64 address, u64 value, u64 expected) override { + ++writes; + if (fail_once) { + fail_once = false; + if (failure_value) { + MemoryWrite(address, *failure_value, 8); + } + return false; + } + if (MemoryRead(address, 8) != expected) { + return false; + } + MemoryWrite(address, value, 8); + return true; + } +}; + +u32 CasInstruction(bool wide, bool acquire, bool release, u32 rs, u32 rn, u32 rt) { + return 0x88A07C00U | (u32(wide) << 30) | (u32(acquire) << 22) | + (rs << 16) | (u32(release) << 15) | (rn << 5) | rt; +} +} // namespace + +TEST_CASE("A64: CAS variants preserve flags and return the observed value", "[a64][lse]") { + const bool wide = GENERATE(false, true); + const bool acquire = GENERATE(false, true); + const bool release = GENERATE(false, true); + const bool match = GENERATE(false, true); + const bool retry = GENERATE(false, true); + const size_t bytes = wide ? 8 : 4; + const u64 observed = wide ? 0x1122334455667788ULL : 0x55667788ULL; + const u64 desired = wide ? 0x8877665544332211ULL : 0x44332211ULL; + AtomicTestEnv env; + env.code_mem = {CasInstruction(wide, acquire, release, 1, 0, 2), 0x14000000}; + env.MemoryWrite(0x1000, observed, bytes); + env.fail_once = retry; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(0, 0x1000); + jit.SetRegister(1, match ? observed : observed ^ 1); + jit.SetRegister(2, desired); + jit.SetPstate(0xA0000000); + env.ticks_left = 4; + jit.Run(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(1) == observed); + REQUIRE(jit.GetRegister(0) == 0x1000); + REQUIRE(jit.GetRegister(2) == desired); + REQUIRE(jit.GetPstate() == 0xA0000000); + REQUIRE(env.MemoryRead(0x1000, bytes) == (match ? desired : observed)); + REQUIRE(env.writes == (retry ? 2U : 1U)); +} + +TEST_CASE("A64: CAS handles SP and overlapping source registers", "[a64][lse]") { + const bool use_sp = GENERATE(false, true); + const bool overlap = GENERATE(false, true); + AtomicTestEnv env; + env.code_mem = {CasInstruction(true, true, true, 1, use_sp ? 31 : 0, overlap ? 1 : 2), + 0x14000000}; + env.MemoryWrite(0x1000, 42, 8); + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(0, 0x1000); + jit.SetSP(0x1000); + jit.SetRegister(1, 42); + jit.SetRegister(2, 84); + env.ticks_left = 2; + jit.Run(); + REQUIRE(jit.GetRegister(1) == 42); + REQUIRE(jit.GetSP() == 0x1000); + REQUIRE(env.MemoryRead(0x1000, 8) == (overlap ? 42 : 84)); +} + +TEST_CASE("A64: CAS retry preserves the full comparison register", "[a64][lse]") { + AtomicTestEnv env; + env.code_mem = {CasInstruction(false, true, false, 1, 0, 2), 0x14000000}; + env.MemoryWrite(0x1000, 0x55667788, 4); + env.fail_once = true; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(0, 0x1000); + jit.SetRegister(1, 0xFFFFFFFF55667788ULL); + jit.SetRegister(2, 42); + jit.SetPstate(0xA0000000); + env.ticks_left = 2; + jit.Step(); + REQUIRE(jit.GetPC() == 0); + REQUIRE(jit.GetRegister(1) == 0xFFFFFFFF55667788ULL); + REQUIRE(env.MemoryRead(0x1000, 4) == 0x55667788); + REQUIRE(jit.GetPstate() == 0xA0000000); + jit.Step(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(1) == 0x55667788); + REQUIRE(env.MemoryRead(0x1000, 4) == 42); + REQUIRE(jit.GetPstate() == 0xA0000000); +} + +TEST_CASE("A64: CAS supports zero-register inputs", "[a64][lse]") { + const bool wide = GENERATE(false, true); + const bool zero_expected = GENERATE(false, true); + const bool zero_desired = GENERATE(false, true); + AtomicTestEnv env; + env.code_mem = {CasInstruction(wide, false, true, zero_expected ? 31 : 1, 0, + zero_desired ? 31 : 2), 0x14000000}; + const size_t bytes = wide ? 8 : 4; + env.MemoryWrite(0x1000, zero_expected ? 0 : 42, bytes); + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(0, 0x1000); + jit.SetRegister(1, 42); + jit.SetRegister(2, 84); + env.ticks_left = 2; + jit.Run(); + REQUIRE(env.MemoryRead(0x1000, bytes) == (zero_desired ? 0 : 84)); + REQUIRE(jit.GetRegister(1) == 42); +} + +TEST_CASE("A64: LDAPR zero extends all supported widths", "[a64][rcpc]") { + const u32 size = GENERATE(0U, 1U, 2U, 3U); + AtomicTestEnv env; + env.code_mem = {0x38BFC001U | (size << 30), 0x14000000}; // LDAPR{B,H} R1, [X0] + const size_t bytes = size_t{1} << size; + const u64 value = 0xFEDCBA9876543210ULL; + env.MemoryWrite(0x1000, value, 8); + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(0, 0x1000); + jit.SetRegister(1, ~u64{0}); + jit.SetPstate(0x60000000); + env.ticks_left = 2; + jit.Run(); + const u64 mask = bytes == 8 ? ~u64{0} : (u64{1} << (bytes * 8)) - 1; + REQUIRE(jit.GetRegister(1) == (value & mask)); + REQUIRE(jit.GetRegister(0) == 0x1000); + REQUIRE(jit.GetPstate() == 0x60000000); +} + +TEST_CASE("A64: CAS retries observe an intervening write", "[a64][lse]") { + const bool wide = GENERATE(false, true); + AtomicTestEnv env; + env.code_mem = {CasInstruction(wide, true, true, 1, 0, 2), 0x14000000}; + env.MemoryWrite(0x1000, 42, wide ? 8 : 4); + env.fail_once = true; + env.failure_value = 13; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(0, 0x1000); + jit.SetRegister(1, 42); + jit.SetRegister(2, 84); + env.ticks_left = 4; + jit.Run(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(1) == 13); + REQUIRE(env.MemoryRead(0x1000, wide ? 8 : 4) == 13); + REQUIRE(env.writes == 2); +} + +TEST_CASE("A64: LDADD variants handle retries, overlap, and overflow", "[a64][lse]") { + const bool wide = GENERATE(false, true); + const bool acquire = GENERATE(false, true); + const bool release = GENERATE(false, true); + const bool retry = GENERATE(false, true); + const bool overlap = GENERATE(false, true); + const bool wrap = GENERATE(false, true); + const size_t bytes = wide ? 8 : 4; + const u64 mask = wide ? ~u64{0} : 0xFFFFFFFFULL; + const u64 observed = wrap ? mask - 1 : 42; + const u64 operand = wrap ? 5 : 7; + const u32 rt = overlap ? 1 : 2; + AtomicTestEnv env; + const u32 instruction = 0xB8200000U | (u32(wide) << 30) | (u32(acquire) << 23) | + (u32(release) << 22) | (1U << 16) | rt; + env.code_mem = {instruction, 0x14000000}; + env.MemoryWrite(0x1000, observed, bytes); + env.fail_once = retry; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(0, 0x1000); + jit.SetRegister(1, operand); + jit.SetRegister(2, 0xABCD); + jit.SetPstate(0x60000000); + env.ticks_left = 4; + jit.Run(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(rt) == observed); + REQUIRE(jit.GetRegister(1) == (overlap ? observed : operand)); + REQUIRE(jit.GetRegister(2) == (overlap ? 0xABCD : observed)); + REQUIRE(jit.GetPstate() == 0x60000000); + REQUIRE(env.MemoryRead(0x1000, bytes) == ((observed + operand) & mask)); + REQUIRE(env.writes == (retry ? 2U : 1U)); +} + +TEST_CASE("A64: small CAS truncates operands and preserves adjacent memory", "[a64][lse]") { + const bool half = GENERATE(false, true); + const bool acquire = GENERATE(false, true); + const bool release = GENERATE(false, true); + const bool match = GENERATE(false, true); + const bool retry = GENERATE(false, true); + const size_t bytes = half ? 2 : 1; + const u64 mask = half ? 0xffff : 0xff; + const u64 observed = 0x8765 & mask; + const u64 desired = 0xabcd & mask; + AtomicTestEnv env; + env.code_mem = {0x08A07C00U | (u32(half) << 30) | (u32(acquire) << 22) | + (1U << 16) | (u32(release) << 15) | 2U, + 0x14000000}; + env.MemoryWrite(0x1000, 0x1122334455667788ULL, 8); + env.MemoryWrite(0x1002, observed, bytes); + const u64 before = env.MemoryRead(0x1000, 8); + env.fail_once = retry; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(0, 0x1002); + const u64 expected = 0xfedcba9876540000ULL | (match ? observed : observed ^ 1); + jit.SetRegister(1, expected); + jit.SetRegister(2, 0x9876543210000000ULL | desired); + jit.SetPstate(0xA0000000); + if (retry) { + env.ticks_left = 1; + jit.Step(); + REQUIRE(jit.GetPC() == 0); + REQUIRE(jit.GetRegister(1) == expected); + REQUIRE(jit.GetPstate() == 0xA0000000); + } + env.ticks_left = 4; + jit.Run(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(1) == observed); + REQUIRE(jit.GetRegister(2) == (0x9876543210000000ULL | desired)); + REQUIRE(jit.GetPstate() == 0xA0000000); + const u64 after = (before & ~(mask << 16)) | ((match ? desired : observed) << 16); + REQUIRE(env.MemoryRead(0x1000, 8) == after); + REQUIRE(env.writes == (retry ? 2U : 1U)); +} + +TEST_CASE("A64: LDSET variants handle retries, overlap, and existing bits", "[a64][lse]") { + const bool wide = GENERATE(false, true); + const bool acquire = GENERATE(false, true); + const bool release = GENERATE(false, true); + const bool retry = GENERATE(false, true); + const bool overlap = GENERATE(false, true); + const bool existing_bits = GENERATE(false, true); + const size_t bytes = wide ? 8 : 4; + const u64 mask = wide ? ~u64{0} : 0xFFFFFFFFULL; + const u64 observed = existing_bits ? mask - 1 : 42; + const u64 operand = existing_bits ? 5 : 7; + const u32 rt = overlap ? 1 : 2; + AtomicTestEnv env; + const u32 instruction = 0xB8203000U | (u32(wide) << 30) | (u32(acquire) << 23) | + (u32(release) << 22) | (1U << 16) | rt; + env.code_mem = {instruction, 0x14000000}; + env.MemoryWrite(0x1000, observed, bytes); + env.fail_once = retry; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(0, 0x1000); + jit.SetRegister(1, operand); + jit.SetRegister(2, 0xABCD); + jit.SetPstate(0x60000000); + env.ticks_left = 4; + jit.Run(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(rt) == observed); + REQUIRE(jit.GetRegister(1) == (overlap ? observed : operand)); + REQUIRE(jit.GetRegister(2) == (overlap ? 0xABCD : observed)); + REQUIRE(jit.GetPstate() == 0x60000000); + REQUIRE(env.MemoryRead(0x1000, bytes) == ((observed | operand) & mask)); + REQUIRE(env.writes == (retry ? 2U : 1U)); +} + + +TEST_CASE("A64: observed LDSETL retries with a destination that is also its address", "[a64][lse]") { + const bool interference = GENERATE(false, true); + AtomicTestEnv env; + env.code_mem = {0xF8733108, 0x14000000}; + env.MemoryWrite(0x1000, 2, 8); + env.fail_once = interference; + env.failure_value = 0x10; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(8, 0x1000); + jit.SetRegister(19, 8); + jit.SetPstate(0xA0000000); + if (interference) { + env.ticks_left = 1; + jit.Step(); + REQUIRE(jit.GetRegister(8) == 0x1000); + REQUIRE(jit.GetPC() == 0); + REQUIRE(jit.GetPstate() == 0xA0000000); + } + env.ticks_left = 4; + jit.Run(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(8) == (interference ? 0x10 : 2)); + REQUIRE(jit.GetRegister(19) == 8); + REQUIRE(env.MemoryRead(0x1000, 8) == (interference ? 0x18 : 0xa)); + REQUIRE(jit.GetPstate() == 0xA0000000); +} + +TEST_CASE("A64: SWP variants handle retries and overlapping source registers", "[a64][lse]") { + const unsigned size = GENERATE(0U, 1U, 2U, 3U); + const bool acquire = GENERATE(false, true); + const bool release = GENERATE(false, true); + const bool retry = GENERATE(false, true); + const bool overlap = GENERATE(false, true); + const bool existing_bits = GENERATE(false, true); + const size_t bytes = size_t{1} << size; + const u64 mask = size == 3 ? ~u64{0} : (u64{1} << (8 * bytes)) - 1; + const u64 observed = existing_bits ? mask - 1 : 42; + const u64 operand = existing_bits ? 5 : 7; + const u32 rt = overlap ? 1 : 2; + AtomicTestEnv env; + const u32 instruction = 0x38208000U | (size << 30) | (u32(acquire) << 23) | + (u32(release) << 22) | (1U << 16) | rt; + env.code_mem = {instruction, 0x14000000}; + env.MemoryWrite(0x1000, observed, bytes); + env.fail_once = retry; + env.MemoryWrite(0x1000 + bytes, 0xA5, 1); + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(0, 0x1000); + jit.SetRegister(1, operand); + jit.SetRegister(2, 0xABCD); + jit.SetPstate(0x60000000); + env.ticks_left = 4; + jit.Run(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(rt) == observed); + REQUIRE(jit.GetRegister(1) == (overlap ? observed : operand)); + REQUIRE(jit.GetRegister(2) == (overlap ? 0xABCD : observed)); + REQUIRE(jit.GetPstate() == 0x60000000); + REQUIRE(env.MemoryRead(0x1000, bytes) == (operand & mask)); + REQUIRE(env.writes == (retry ? 2U : 1U)); + REQUIRE(env.MemoryRead(0x1000 + bytes, 1) == 0xA5); +} + + +TEST_CASE("A64: observed SWPAL retries with a destination that is also its address", "[a64][lse]") { + const bool interference = GENERATE(false, true); + AtomicTestEnv env; + env.code_mem = {0xB8EA8129, 0x14000000}; + env.MemoryWrite(0x1000, 2, 4); + env.fail_once = interference; + env.failure_value = 0x10; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(9, 0x1000); + jit.SetRegister(10, 8); + jit.SetPstate(0xA0000000); + if (interference) { + env.ticks_left = 1; + jit.Step(); + REQUIRE(jit.GetRegister(9) == 0x1000); + REQUIRE(jit.GetPC() == 0); + REQUIRE(jit.GetPstate() == 0xA0000000); + } + env.ticks_left = 4; + jit.Run(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(9) == (interference ? 0x10 : 2)); + REQUIRE(jit.GetRegister(10) == 8); + REQUIRE(env.MemoryRead(0x1000, 4) == 8); + REQUIRE(jit.GetPstate() == 0xA0000000); +} + +TEST_CASE("A64: LDCLR variants handle retries, overlap, and existing bits", "[a64][lse]") { + const bool wide = GENERATE(false, true); + const bool acquire = GENERATE(false, true); + const bool release = GENERATE(false, true); + const bool retry = GENERATE(false, true); + const bool overlap = GENERATE(false, true); + const bool existing_bits = GENERATE(false, true); + const size_t bytes = wide ? 8 : 4; + const u64 mask = wide ? ~u64{0} : 0xFFFFFFFFULL; + const u64 observed = existing_bits ? mask - 1 : 42; + const u64 operand = existing_bits ? 5 : 7; + const u32 rt = overlap ? 1 : 2; + AtomicTestEnv env; + const u32 instruction = 0xB8201000U | (u32(wide) << 30) | (u32(acquire) << 23) | + (u32(release) << 22) | (1U << 16) | rt; + env.code_mem = {instruction, 0x14000000}; + env.MemoryWrite(0x1000, observed, bytes); + env.fail_once = retry; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(0, 0x1000); + jit.SetRegister(1, operand); + jit.SetRegister(2, 0xABCD); + jit.SetPstate(0x60000000); + env.ticks_left = 4; + jit.Run(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(rt) == observed); + REQUIRE(jit.GetRegister(1) == (overlap ? observed : operand)); + REQUIRE(jit.GetRegister(2) == (overlap ? 0xABCD : observed)); + REQUIRE(jit.GetPstate() == 0x60000000); + REQUIRE(env.MemoryRead(0x1000, bytes) == ((observed & ~operand) & mask)); + REQUIRE(env.writes == (retry ? 2U : 1U)); +} + + +TEST_CASE("A64: observed LDCLRL retries with a destination that is also its address", "[a64][lse]") { + const bool interference = GENERATE(false, true); + AtomicTestEnv env; + env.code_mem = {0xF8691108, 0x14000000}; + env.MemoryWrite(0x1000, 0xf, 8); + env.fail_once = interference; + env.failure_value = 0x10; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.global_monitor = &env.monitor; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + jit.SetRegister(8, 0x1000); + jit.SetRegister(9, 1); + jit.SetPstate(0xA0000000); + if (interference) { + env.ticks_left = 1; + jit.Step(); + REQUIRE(jit.GetRegister(8) == 0x1000); + REQUIRE(jit.GetPC() == 0); + REQUIRE(jit.GetPstate() == 0xA0000000); + } + env.ticks_left = 4; + jit.Run(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(8) == (interference ? 0x10 : 0xf)); + REQUIRE(jit.GetRegister(9) == 1); + REQUIRE(env.MemoryRead(0x1000, 8) == (interference ? 0x10 : 0xe)); + REQUIRE(jit.GetPstate() == 0xA0000000); +} diff --git a/src/dynarmic/tests/A64/pauth_returns.cpp b/src/dynarmic/tests/A64/pauth_returns.cpp new file mode 100644 index 0000000000..f02e40e1af --- /dev/null +++ b/src/dynarmic/tests/A64/pauth_returns.cpp @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <catch2/catch_test_macros.hpp> +#include <catch2/generators/catch_generators.hpp> + +#include "dynarmic/tests/A64/testenv.h" + +namespace { +class ReturnTestEnv final : public A64TestEnv { +public: + Dynarmic::A64::Jit* jit{}; + std::optional<Dynarmic::A64::Exception> exception; + + void ExceptionRaised(u64, Dynarmic::A64::Exception reason) override { + exception = reason; + jit->HaltExecution(Dynarmic::HaltReason::UserDefined1); + } +}; +} // namespace + +TEST_CASE("A64: authenticated returns with disabled authentication keys", "[a64][pauth]") { + const u32 instruction = GENERATE(0xD65F0BFFU, 0xD65F0FFFU); // RETAA, RETAB + const bool supported = GENERATE(false, true); + ReturnTestEnv env; + env.code_mem = {instruction}; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.enable_pauth_returns = supported; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + env.jit = &jit; + jit.SetPC(0); + jit.SetRegister(30, 0x123400); + jit.SetSP(0x567800); + env.ticks_left = 1; + jit.Step(); + if (supported) { + REQUIRE_FALSE(env.exception.has_value()); + REQUIRE(jit.GetPC() == 0x123400); + } else { + REQUIRE(env.exception == Dynarmic::A64::Exception::UnallocatedEncoding); + } + REQUIRE(jit.GetRegister(30) == 0x123400); + REQUIRE(jit.GetSP() == 0x567800); +} + +TEST_CASE("A64: PAC hint and return agree on disabled keys", "[a64][pauth]") { + ReturnTestEnv env; + env.code_mem = {0xD503237FU, 0xD65F0FFFU}; // PACIBSP; RETAB + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.enable_pauth_returns = true; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + env.jit = &jit; + jit.SetPC(0); + jit.SetRegister(30, 0x123400); + jit.SetSP(0x567800); + env.ticks_left = 2; + jit.Step(); + REQUIRE(jit.GetPC() == 4); + REQUIRE(jit.GetRegister(30) == 0x123400); + jit.Step(); + REQUIRE_FALSE(env.exception.has_value()); + REQUIRE(jit.GetPC() == 0x123400); + REQUIRE(jit.GetRegister(30) == 0x123400); + REQUIRE(jit.GetSP() == 0x567800); +} + +TEST_CASE("A64: PAC stripping uses the configured address width", "[a64][pauth]") { + const u32 instruction = GENERATE(0xDAC143F4U, 0xDAC147F4U, 0xD50320FFU); + const u32 bits = GENERATE(32U, 36U, 39U); + const bool negative = GENERATE(false, true); + const u32 reg = instruction == 0xD50320FFU ? 30 : 20; + const u64 mask = (u64{1} << bits) - 1; + const u64 address = 0x123456789ULL & mask; + const u64 pointer = (0xAAULL << 56) | ((negative ? 0xD5ULL : 0x55ULL) << 48) | address; + ReturnTestEnv env; + env.code_mem = {instruction}; + Dynarmic::A64::UserConfig config; + config.callbacks = &env; + config.enable_pauth_returns = true; + config.page_table_address_space_bits = bits; + config.code_cache_size = 16 * 1024 * 1024; + Dynarmic::A64::Jit jit{config}; + env.jit = &jit; + jit.SetRegister(reg, pointer); + jit.SetPstate(0xA0000000); + env.ticks_left = 1; + jit.Step(); + REQUIRE_FALSE(env.exception.has_value()); + REQUIRE(jit.GetRegister(reg) == (negative ? address | ~mask : address)); + REQUIRE(jit.GetPstate() == 0xA0000000); +} diff --git a/src/dynarmic/tests/CMakeLists.txt b/src/dynarmic/tests/CMakeLists.txt index 15a77e4b5c..1b7b22f0b7 100644 --- a/src/dynarmic/tests/CMakeLists.txt +++ b/src/dynarmic/tests/CMakeLists.txt @@ -17,6 +17,8 @@ add_executable(dynarmic_tests decoder_tests.cpp # A64 A64/a64.cpp + A64/pauth_returns.cpp + A64/lse.cpp A64/fibonacci.cpp A64/fp_min_max.cpp A64/misaligned_page_table.cpp diff --git a/src/shader_recompiler/CMakeLists.txt b/src/shader_recompiler/CMakeLists.txt index cc9ce95d92..39484a1bd5 100644 --- a/src/shader_recompiler/CMakeLists.txt +++ b/src/shader_recompiler/CMakeLists.txt @@ -58,6 +58,9 @@ add_library(shader_recompiler STATIC frontend/ir/type.h frontend/ir/value.cpp frontend/ir/value.h + frontend/ampere/lower_to_maxwell.h + frontend/ampere/program_override.h + frontend/ampere/program_address_map.h frontend/maxwell/control_flow.cpp frontend/maxwell/control_flow.h frontend/maxwell/decode.cpp diff --git a/src/shader_recompiler/environment.h b/src/shader_recompiler/environment.h index 184b56ebf3..67486430c8 100644 --- a/src/shader_recompiler/environment.h +++ b/src/shader_recompiler/environment.h @@ -74,6 +74,8 @@ public: [[nodiscard]] virtual u64 ReadInstruction(u32 address) = 0; + [[nodiscard]] virtual u32 TranslateProgramAddress(u32 address) const { return address; } + [[nodiscard]] virtual u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) = 0; [[nodiscard]] virtual TextureType ReadTextureType(u32 raw_handle) = 0; diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h new file mode 100644 index 0000000000..363d1b9b31 --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h @@ -0,0 +1,1614 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +// Encoding reference: Mesa NAK sm70_encode.rs (MIT), Copyright 2022 Collabora, Ltd. +#pragma once + +#include <algorithm> +#include <array> +#include <bit> +#include <span> +#include <optional> +#include <string_view> +#include <vector> +#include "common/common_types.h" + +namespace Shader::Ampere { + +struct Instruction { + std::array<u32, 4> words; + + u32 Bits(u32 offset, u32 count) const { + u64 bits = words[offset / 32]; + if (offset / 32 + 1 < words.size()) { + bits |= u64{words[offset / 32 + 1]} << 32; + } + return static_cast<u32>((bits >> (offset % 32)) & ((u64{1} << count) - 1)); + } +}; +static_assert(sizeof(Instruction) == 16); + +struct LoweredProgram { + std::vector<u64> instructions; + std::vector<u32> source_instruction_offsets; + u32 source_size{}; + u32 required_texture_buffer = ~0U; + u32 error_offset{}; + std::string_view error; + + explicit operator bool() const { return error.empty(); } +}; + +// Lower only operations whose semantics are representable by the existing +// Maxwell frontend. These are new instructions, not reinterpretations of the +// native bytes. Scheduling/control words are added by the environment adapter. +inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { + LoweredProgram result; + struct Relocation { u32 output_index; s64 target; }; + std::vector<Relocation> relocations; + std::vector<u32> barrier_tokens; + std::vector<u32> call_targets; + std::vector<u32> call_links; + bool main_exited = false; + bool uses_uniform_bank = false; + for (const auto& instruction : code) { + const u32 op = instruction.Bits(0, 12); + uses_uniform_bank |= op == 0x3c4 || op == 0x2bd || op == 0xc0c || op == 0x986 || op == 0x9a8 || op == 0xc12 || op == 0xc17 || op == 0xe24 || op == 0xc24 || op == 0xd00 || op == 0xd09 || op == 0x9c3 || op == 0xc45 || op == 0x2a4 || op == 0x891 || op == 0x890 || op == 0x290 || op == 0x28c || op == 0x88c || op == 0x287 || op == 0x887 || op == 0x958 || op == 0xab9 || op == 0xabb || op == 0x899 || op == 0xc20 || op == 0x981 || op == 0x526 || op == 0xd08 || op == 0xc21 || op == 0xe21 || op == 0xc23 || op == 0xe23 || op == 0xc02 || + op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || (op == 0x81c && instruction.Bits(67, 1)); + } + const auto address_of = [](u32 index) { return 0x50U + index * 8 + ((index + 1) / 3) * 8; }; + const auto fail = [&](u32 index, std::string_view reason) { + result.instructions.clear(); + result.source_instruction_offsets.clear(); + result.error_offset = index * sizeof(Instruction); + result.error = reason; + return result; + }; + const auto finish = [&](u32 index) { + for (const auto& relocation : relocations) { + if (relocation.target > index) return fail(index, "native branch or call leaves captured program"); + const u32 target = result.source_instruction_offsets[static_cast<size_t>(relocation.target)]; + const s64 relative = s64{address_of(target)} - address_of(relocation.output_index) - 8; + if (relative < -0x800000 || relative > 0x7fffff) return fail(index, "lowered branch distance"); + result.instructions[relocation.output_index] |= (static_cast<u64>(relative) & 0xffffff) << 20; + } + result.source_size = (index + 1) * sizeof(Instruction); + return result; + }; + if (code.empty() || code.size() > 65536) return fail(0, "invalid native program size"); + for (u32 index = 0; index < code.size(); ++index) { + auto insn = code[index]; + result.source_instruction_offsets.push_back(static_cast<u32>(result.instructions.size())); + u32 opcode = insn.Bits(0, 12); + std::optional<u32> active_link; + u32 active_entry = 0; + if (main_exited) { + for (size_t call = 0; call < call_targets.size(); ++call) { + if (call_targets[call] <= index && call_targets[call] >= active_entry) { + active_entry = call_targets[call]; + active_link = call_links[call]; + } + } + if (active_link && insn.Bits(16, 8) == *active_link) + return fail(index, "native callee overwrites link register"); + } + // Scratch values and emulated uniform state occupy reserved high GPRs. + // Reject native accesses that could alias that state. + for (u32 bit : {16U, 24U, 32U, 64U}) { + if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66) continue; + const u32 reg = insn.Bits(bit, 8); + if (reg >= (uses_uniform_bank ? 216U : 250U) && reg < 255) return fail(index, "native program uses lowering scratch registers"); + } + const auto uniform_register = [&](u32 reg) -> u32 { + if (reg < 16) return 216 + reg; + if (reg == 63) return 255; + return ~0U; + }; + if (opcode == 0x2a4 || opcode == 0x891 || opcode == 0x890 || opcode == 0x290) { + if (insn.Bits(12, 4) != 7) return fail(index, "native uniform arithmetic predication"); + const u32 dst = uniform_register(insn.Bits(16, 8)); + const u32 a = uniform_register(insn.Bits(24, 8)); + const u32 c = uniform_register(insn.Bits(64, 8)); + if (dst == ~0U || a == ~0U || c == ~0U) return fail(index, "native uniform arithmetic register"); + insn.words[0] = (insn.words[0] & 0xffff) | (dst << 16) | (a << 24); + insn.words[2] = (insn.words[2] & ~0x080000ffU) | c; + if (opcode == 0x2a4 || opcode == 0x290) { + const u32 b = uniform_register(insn.Bits(32, 8)); + if (b == ~0U) return fail(index, "native uniform arithmetic second register"); + insn.words[1] = (insn.words[1] & ~0xffU) | b; + } + opcode = opcode == 0x2a4 ? 0x224 : opcode == 0x891 ? 0x811 : opcode == 0x890 ? 0x810 : 0x210; + } + const bool uniform_predication = opcode == 0xabb || opcode == 0xab9 || opcode == 0xc82 || opcode == 0x882; + const u32 native_predicate = insn.Bits(12, 4); + const bool guarded_uniform = uniform_predication && native_predicate != 7; + if (guarded_uniform && (native_predicate & 7) != 0) + return fail(index, "native uniform execution predicate"); + u64 predicate = u64{guarded_uniform ? ((native_predicate & 8) | 6) : native_predicate} << 16; + const u64 dest = insn.Bits(16, 8); + const auto base = [&](u64 opcode_bits) { return opcode_bits | predicate | dest; }; + const auto cbuf = [&]() -> u64 { + return (u64{insn.Bits(38, 16) / 4} << 20) | (u64{insn.Bits(54, 5)} << 34); + }; + const auto valid_cbuf = [&] { + return !insn.Bits(91, 1) && insn.Bits(54, 5) < 18 && + (insn.Bits(38, 16) % 4) == 0; + }; + std::string_view operand_error; + const auto emit = [&](u64 word) { result.instructions.push_back(word); }; + const auto save_predicate = [&] { + emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{0x40} << 20)); + }; + const auto restore_predicate = [&] { + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{0x40} << 20)); + }; + if (guarded_uniform) { + save_predicate(); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{0x40} << 20)); + } + const auto mov_imm = [&](u32 reg, u32 value) { + emit(0x0100000000000000ULL | predicate | reg | (u64{15} << 12) | (u64{value} << 20)); + }; + const auto materialize = [&](u32 kind, u32 reg, u32 scratch, bool absolute, bool negate) -> u32 { + if (kind == 3) { + if (reg < 16) reg += 216; + else if (reg == 63) reg = 255; + else { operand_error = "native uniform source register"; return 255; } + } else if (kind == 1) { + mov_imm(scratch, insn.words[1]); + reg = scratch; + } else if (kind == 2) { + if (!valid_cbuf()) { operand_error = "native constant buffer address"; return 255; } + emit(0x4c98000000000000ULL | predicate | scratch | (u64{15} << 39) | cbuf()); + reg = scratch; + } + if (absolute || negate) { + if (reg != scratch) emit(0x5c98000000000000ULL | predicate | scratch | (u64{reg} << 20) | (u64{15} << 39)); + if (absolute) emit(0x0400000000000000ULL | predicate | scratch | (u64{scratch} << 8) | (u64{0x7fffffff} << 20)); + if (negate) emit(0x0400000000000000ULL | predicate | scratch | (u64{scratch} << 8) | (u64{0x80000000} << 20) | (u64{2} << 53)); + reg = scratch; + } + return reg; + }; + const auto operand = [&](u32 which, bool floating = true) -> u32 { + const u32 form = insn.Bits(9, 3); + u32 reg{}, kind{}, abs{}, neg{}; + if (which == 0) { + reg = insn.Bits(24, 8); abs = insn.Bits(73, 1); neg = insn.Bits(72, 1); + } else { + if (form != 1 && form != 2 && form != 3 && form != 4 && form != 5 && form != 6 && form != 7) { + operand_error = "native ALU source form"; return 255; + } + const bool wide = which == 1 ? (form == 4 || form == 5 || form == 6) : (form == 2 || form == 3 || form == 7); + if (wide) { + kind = form == 6 || form == 7 ? 3 : (form == 4 || form == 2) ? 1 : 2; + if (kind == 3) reg = insn.Bits(32, 8); + if (kind == 2 || kind == 3) { abs = insn.Bits(62, 1); neg = insn.Bits(63, 1); } + } else { + const bool upper = which == 1 ? (form == 2 || form == 3 || form == 7) : true; + reg = insn.Bits(upper ? 64 : 32, 8); + abs = insn.Bits(upper ? 74 : 62, 1); neg = insn.Bits(upper ? 75 : 63, 1); + } + } + return materialize(kind, reg, 250 + which, floating && abs, floating && neg); + }; + struct HalfSource { u32 reg, swizzle; bool absolute, negate; }; + const auto half_source = [&](u32 which) -> HalfSource { + if (which == 0) return {insn.Bits(24, 8), insn.Bits(74, 2), + insn.Bits(73, 1) != 0, insn.Bits(72, 1) != 0}; + const u32 form = insn.Bits(9, 3); + const bool wide = which == 1 ? (form == 4 || form == 5) : (form == 2 || form == 3); + if (wide) { + const bool immediate = form == 2 || form == 4; + const u32 reg = materialize(immediate ? 1 : 2, 255, 250 + which, false, false); + return {reg, immediate ? 0U : insn.Bits(60, 2), + !immediate && insn.Bits(62, 1), !immediate && insn.Bits(63, 1)}; + } + const bool upper = which == 2 || form == 2 || form == 3; + return {insn.Bits(upper ? 64 : 32, 8), insn.Bits(upper ? 81 : 60, 2), + insn.Bits(upper ? 83 : 62, 1) != 0, insn.Bits(upper ? 84 : 63, 1) != 0}; + }; + const auto extract_half = [&](HalfSource source, u32 lane, u32 target) { + const bool high = source.swizzle == 3 || (source.swizzle == 0 && lane == 1); + emit(0x5ca8000000000000ULL | predicate | target | (u64{2} << 8) | + (u64{source.swizzle == 1 ? 2U : 1U} << 10) | (u64{source.reg} << 20) | + (u64{high} << 41) | (u64{source.negate} << 45) | (u64{source.absolute} << 49)); + }; + const auto xmad = [&](u32 target, u32 a, u32 b, u32 c, bool a_high, + bool b_high, u32 mode, bool immediate) { + emit((immediate ? 0x3600000000000000ULL : 0x5b00000000000000ULL) | + predicate | target | (u64{a} << 8) | + (u64{immediate ? ((b >> (b_high ? 16 : 0)) & 0xffff) : b} << 20) | + (u64{!immediate && b_high} << 35) | (u64{c} << 39) | + (u64{mode} << 50) | (u64{a_high} << 53)); + }; + const auto multiply_low = [&](u32 a, u32 b, u32 c, bool immediate) { + xmad(250, a, b, c, false, false, 0, immediate); + xmad(251, a, b, 255, true, false, 0, immediate); + xmad(251, a, b, 251, false, true, 0, immediate); + emit(base(0x5c18000000000000ULL) | (u64{251} << 8) | + (u64{250} << 20) | (u64{16} << 39)); + }; + switch (opcode) { + case 0x304: { // F2F.F16.F32, scalar round-to-nearest conversion. + if (insn.words[2] != 0x00200800 || insn.Bits(24, 8) || + (insn.words[1] & ~0xc00000ffU)) + return fail(index, "native scalar float conversion mode"); + emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(62, 1)} << 49) | + (u64{insn.Bits(63, 1)} << 45)); + break; + } + case 0x23e: case 0xa3e: { // F2FP.PACK_AB / MERGE_C, round-to-nearest + if (insn.Bits(62, 2) || insn.Bits(73, 5) || insn.Bits(79, 2) || + insn.Bits(81, 10) || insn.Bits(91, 1)) { + return fail(index, "native float packing modifiers"); + } + const bool merge = insn.Bits(78, 1); + if (insn.Bits(72, 1)) return fail(index, "native float packing selector"); + const u32 a = merge ? operand(2, false) : operand(0, false); + const u32 b = operand(1, false); + // Canonical F2F writes the converted half in its low 16 bits. + if (merge) { + emit(0x5c98000000000000ULL | predicate | 250 | (u64{a} << 20) | (u64{15} << 39)); + } else { + emit(0x5ca8000000000000ULL | predicate | 250 | (u64{1} << 8) | + (u64{2} << 10) | (u64{a} << 20)); + } + emit(0x5ca8000000000000ULL | predicate | 251 | (u64{1} << 8) | + (u64{2} << 10) | (u64{b} << 20)); + emit(base(0x5c18000000000000ULL) | (u64{250} << 8) | (u64{251} << 20) | (u64{16} << 39)); + break; + } + case 0x232: case 0x832: case 0xa32: { // HMUL2 + if (insn.Bits(78, 2) || insn.Bits(81, 10) || insn.Bits(91, 1)) + return fail(index, "native half multiply mode"); + const auto a = half_source(0), b = half_source(1); + const u32 precision = insn.Bits(76, 1) ? 2 : insn.Bits(80, 1); + emit(base(0x5d08000000000000ULL) | (u64{a.reg} << 8) | (u64{b.reg} << 20) | + (u64{b.swizzle} << 28) | (u64{b.absolute} << 30) | (u64{a.negate != b.negate} << 31) | + (u64{insn.Bits(77, 1)} << 32) | (u64{precision} << 39) | + (u64{a.absolute} << 44) | (u64{a.swizzle} << 47)); + break; + } + case 0x230: case 0x430: case 0x630: { // HADD2 + if (insn.Bits(76, 1) || insn.Bits(79, 1) || insn.Bits(81, 10) || insn.Bits(91, 1)) + return fail(index, "native half add mode"); + const auto a = half_source(0); + const auto b = half_source(opcode == 0x230 ? 1 : 2); + if (insn.Bits(78, 1)) { + if (a.reg != 255 || !a.negate || a.absolute || a.swizzle != 0 || + b.swizzle < 2 || insn.Bits(77, 1) || insn.Bits(80, 1)) + return fail(index, "native float-output half add mode"); + extract_half(b, 0, static_cast<u32>(dest)); + } else { + emit(base(0x5d10000000000000ULL) | (u64{a.reg} << 8) | (u64{b.reg} << 20) | + (u64{b.swizzle} << 28) | (u64{b.absolute} << 30) | (u64{b.negate} << 31) | + (u64{insn.Bits(77, 1)} << 32) | (u64{insn.Bits(80, 1)} << 39) | + (u64{a.negate} << 43) | (u64{a.absolute} << 44) | (u64{a.swizzle} << 47)); + } + break; + } + case 0x231: case 0x431: case 0x631: case 0x831: case 0xa31: { // HFMA2 + const bool relu = insn.Bits(79, 1) != 0; + if (insn.Bits(78, 1) || insn.Bits(85, 1) || insn.Bits(91, 1) || + (relu ? (insn.Bits(87, 4) != 7 || + insn.Bits(77, 1)) : insn.Bits(87, 4) != 0)) + return fail(index, "native half fused multiply-add mode"); + if (relu && native_predicate != 7) { + // Guard the whole expansion before its temporary P6 updates. + relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1}); + emit(0xe24000000000000fULL | (u64{native_predicate ^ 8U} << 16)); + predicate = u64{7} << 16; + } + auto a = half_source(0); + auto b = half_source(1); + const auto c = half_source(2); + if (insn.Bits(86, 1)) { // H0_NH1: negate only the upper packed component. + if (b.swizzle != 0 || b.absolute || b.reg >= 250) + return fail(index, "native half mixed-sign source mode"); + emit(0x0400000000000000ULL | predicate | 251 | (u64{b.reg} << 8) | + (u64{0x80000000} << 20) | (u64{2} << 53)); + b.reg = 251; + } + if (b.absolute || c.absolute || (a.absolute && a.reg >= 248)) + return fail(index, "native half fused multiply-add absolute source"); + if (a.absolute) { + emit(0x0400000000000000ULL | predicate | 250 | (u64{a.reg} << 8) | + (u64{a.swizzle == 1 ? 0x7fffffffU : 0x7fff7fffU} << 20)); + a.reg = 250; + a.absolute = false; + } + const u32 precision = insn.Bits(76, 1) ? 2 : insn.Bits(80, 1); + emit(0x5d00000000000000ULL | predicate | (relu ? 254U : dest) | (u64{a.reg} << 8) | (u64{b.reg} << 20) | + (u64{b.swizzle} << 28) | (u64{c.negate} << 30) | (u64{a.negate != b.negate} << 31) | + (u64{insn.Bits(77, 1)} << 32) | (u64{c.swizzle} << 35) | + (u64{precision} << 37) | (u64{c.reg} << 39) | (u64{a.swizzle} << 47)); + if (relu) { + // Clamp the rounded binary16 lanes, preserving positive infinity + // and canonicalizing either sign of NaN without saturating to one. + save_predicate(); + for (u32 lane = 0; lane < 2; ++lane) { + emit(0x3828000000070000ULL | 251 | (u64{254} << 8) | (u64{lane * 16} << 20)); + emit(0x0400000000070000ULL | 251 | (u64{251} << 8) | (u64{0xffff} << 20)); + emit(0x0400000000070000ULL | 252 | (u64{251} << 8) | (u64{0x7fff} << 20)); + mov_imm(250, 0x7c00); + emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{250} << 8) | + (u64{252} << 20) | (u64{7} << 39) | (u64{1} << 49)); + emit(0x3828000000070000ULL | 252 | (u64{251} << 8) | (u64{15} << 20)); + mov_imm(250, 1); + emit(0x5c10000000070000ULL | 252 | (u64{252} << 8) | (u64{250} << 20) | (u64{1} << 48)); + emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48)); + mov_imm(250, 0x7fff); + emit(0x5ca0000000070000ULL | 251 | (u64{250} << 8) | (u64{251} << 20) | (u64{6} << 39)); + if (lane == 0) { + emit(base(0x5c98000000000000ULL) | (u64{251} << 20) | (u64{15} << 39)); + } else { + emit(0x3848000000070000ULL | 251 | (u64{251} << 8) | (u64{16} << 20)); + emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{251} << 20) | + (u64{1} << 41) | (u64{7} << 48)); + } + } + restore_predicate(); + } + break; + } + case 0x240: case 0x840: { // HMNMX2, exact half promotion for min/max + if (insn.Bits(76, 11) || insn.Bits(91, 1)) + return fail(index, "native half min/max mode"); + const auto a = half_source(0); + const auto b = opcode == 0x840 ? HalfSource{255, 0, false, false} : half_source(1); + if (a.swizzle == 1 || b.swizzle == 1 || b.reg >= 250 && b.reg != 255) + return fail(index, "native half min/max source mode"); + for (u32 lane = 0; lane < 2; ++lane) { + extract_half(a, lane, 250); + if (opcode == 0x840) { + const u32 half = (insn.words[1] >> (lane * 16)) & 0xffff; + mov_imm(251, half); + extract_half(HalfSource{251, 2, false, false}, 0, 251); + } else { + extract_half(b, lane, 251); + } + const u32 target = lane == 0 ? 252 : 250; + emit(0x5c60000000000000ULL | predicate | target | (u64{250} << 8) | (u64{251} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(0x5ca8000000000000ULL | predicate | target | (u64{1} << 8) | + (u64{2} << 10) | (u64{target} << 20)); + } + emit(base(0x5c18000000000000ULL) | (u64{250} << 8) | (u64{252} << 20) | (u64{16} << 39)); + break; + } + case 0xc24: case 0xe24: { // IMAD low word with a uniform multiplier or addend. + if ((insn.words[2] & ~0x080002ffU) != 0x078e0000 || (insn.words[1] & ~0xffU)) + return fail(index, "native uniform-addend multiply mode"); + const u32 a = operand(0, false), b = operand(1, false), c = operand(2, false); + multiply_low(a, b, c, false); + break; + } + case 0x224: { // IMAD register / zero-product move + if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 && insn.Bits(75, 1) && + (insn.words[2] & ~0xaffU) == 0x078e0000) { + emit(base(0x5c10000000000000ULL) | (u64{255} << 8) | + (u64{insn.Bits(64, 8)} << 20) | (u64{1} << 48)); + break; + } + if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 && + (insn.words[2] & ~0xffU) == 0x078e0000) { + emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.Bits(64, 8)} << 20)); + break; + } + if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (insn.words[1] & ~0xffU)) + return fail(index, "native integer register multiply mode"); + multiply_low(insn.Bits(24, 8), insn.Bits(32, 8), insn.Bits(64, 8), false); + break; + } + case 0x827: { // IMAD.HI.U32, captured zero addend + if (insn.words[2] != 0x078e00ff) return fail(index, "native high multiply mode"); + const u32 a = insn.Bits(24, 8), b = insn.words[1]; + // Base-2^16 long multiplication. Each intermediate sum fits U32. + xmad(250, a, b, 255, false, false, 0, true); + xmad(251, a, b, 250, true, false, 2, true); + xmad(250, a, b, 251, false, true, 1, true); + xmad(252, a, b, 251, true, true, 2, true); + emit(0x3828000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{16} << 20)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20)); + break; + } + case 0xa24: { // IMAD, constant-buffer multiplier + if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || !valid_cbuf() || insn.Bits(62, 2)) + return fail(index, "native integer constant-buffer multiply mode"); + emit(0x4c98000000000000ULL | predicate | 252 | (u64{15} << 39) | cbuf()); + multiply_low(insn.Bits(24, 8), 252, insn.Bits(64, 8), false); + break; + } + case 0x424: case 0x624: { // IMAD.MOV.U32: zero product plus immediate/cbuf + if (opcode == 0x624 && insn.Bits(24, 8) == 255 && + (insn.words[2] & ~0x07800000U) == 0x000e06ffU) { + const u32 source = operand(2, false); + mov_imm(250, 1); + emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(base(0x5c10000000000000ULL) | (u64{source} << 8) | (u64{250} << 20)); + break; + } + if (insn.Bits(24, 8) != 255 || insn.Bits(64, 8) != 255 || insn.words[2] != 0x078e00ff) { + if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (opcode == 0x624 && insn.Bits(62, 2))) + return fail(index, "native multiply wide addend mode"); + const u32 a = operand(0, false), b = operand(1, false), c = operand(2, false); + multiply_low(a, b, c, false); + break; + } + if (insn.Bits(24, 8) != 255 || insn.Bits(64, 8) != 255 || + insn.words[2] != 0x078e00ff) + return fail(index, "native integer move mode"); + if (opcode == 0x624) { + if (!valid_cbuf() || insn.Bits(62, 2)) + return fail(index, "native integer move constant buffer"); + emit(base(0x4c98000000000000ULL) | (u64{15} << 39) | cbuf()); + } else { + emit(base(0x0100000000000000ULL) | (u64{15} << 12) | + (u64{insn.words[1]} << 20)); + } + break; + } + case 0x824: { // Captured low-word IMAD forms + if (insn.words[1] == 1 && (insn.words[2] & 0xffffff00U) == 0x078e0200) { + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(64, 8)} << 20)); + break; + } + if (insn.words[1] != 48 || insn.words[2] != 0x078e02ff) { + if ((insn.words[2] & ~0x2ffU) != 0x078e0000) + return fail(index, "native integer multiply mode"); + multiply_low(insn.Bits(24, 8), insn.words[1], insn.Bits(64, 8), true); + break; + } + const u64 source = insn.Bits(24, 8); + emit(0x5c18000000000000ULL | predicate | 250 | (source << 8) | (source << 20) | (u64{1} << 39)); + emit(base(0x3848000000000000ULL) | (u64{250} << 8) | (u64{4} << 20)); + break; + } + case 0x886: { // VOTEU ballot and independent uniform predicate output. + const u32 output = insn.Bits(81, 3), target = insn.Bits(16, 8); + if ((target >= 16 && target != 63) || insn.words[1] || + (output > 1 && output != 7) || insn.Bits(72, 2) > 2 || native_predicate != 7) + return fail(index, "native uniform vote mode"); + const u32 temporary = output == 7 ? 7 : 6 - output; + const u32 mask = 1U << temporary; + if (output != 7) emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + emit(0x50d8000000070000ULL | uniform_register(target) | + (u64{insn.Bits(87, 4)} << 39) | (u64{temporary} << 45) | (u64{insn.Bits(72, 2)} << 48)); + if (output != 7) { + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20)); + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + } + break; + } + case 0x3c2: { // R2UR: broadcast the first lane satisfying the execution predicate. + if (dest >= 16 || insn.words[1] || insn.words[2] != 0x000e0000) + return fail(index, "native register-to-uniform mode"); + save_predicate(); + emit(0x50d8000000070000ULL | 250 | (u64{native_predicate} << 39) | + (u64{6} << 45) | (u64{1} << 48)); + emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49)); + emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48)); + emit(0x5c30000000070000ULL | 250 | (u64{250} << 20)); + emit(0xef10000000070000ULL | 251 | (u64{insn.Bits(24, 8)} << 8) | + (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x5ca0000000070000ULL | (dest + 216) | (u64{251} << 8) | + (u64{dest + 216} << 20) | (u64{6} << 39)); + restore_predicate(); + break; + } + case 0x81c: { // Observed predicate LUT constants and UP0-to-predicate copy. + const u32 target = insn.Bits(81, 3); + const u32 lut = insn.Bits(64, 3) | (insn.Bits(72, 5) << 3); + if ((native_predicate != 7 && insn.Bits(67, 1)) || target == 7 || insn.Bits(84, 3) != 7 || + insn.Bits(16, 8) || insn.words[1]) + return fail(index, "native predicate LUT mode"); + if (!insn.Bits(67, 1) && (lut == 0xfe || lut == 0xa8)) { + emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 | + (u64{insn.Bits(87, 4)} << 12) | (u64{1} << 24) | + (u64{insn.Bits(77, 4)} << 29) | (u64{insn.Bits(68, 4)} << 39) | + (u64{lut == 0xfe} << 45)); + break; + } + if (!insn.Bits(67, 1) && (lut == 0x80 || + (lut == 8 && insn.Bits(68, 4) == 7 && insn.Bits(77, 4) == 7))) { + emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 | + (u64{insn.Bits(87, 4) ^ (lut == 8 ? 8U : 0U)} << 12) | (u64{insn.Bits(77, 4)} << 29) | + (u64{insn.Bits(68, 4)} << 39)); + break; + } + if (insn.Bits(77, 4) != 7 || insn.Bits(87, 4) != 7) + return fail(index, "native predicate LUT source mode"); + if (insn.Bits(67, 1) && insn.Bits(68, 4) <= 1 && lut == 0x80) { + emit(0x3828000000070000ULL | 250 | (u64{239} << 8) | (u64{6 - insn.Bits(68, 4)} << 20)); + emit(0x3848000000070000ULL | 250 | (u64{250} << 8) | (u64{target} << 20)); + } else if (!insn.Bits(67, 1) && insn.Bits(68, 4) == 7 && (lut == 0x80 || lut == 8)) { + emit(0x010000000007f000ULL | 250 | ((lut == 0x80 ? u64{1} << target : 0ULL) << 20)); + } else return fail(index, "native predicate LUT truth table"); + emit(0x38f0000000070000ULL | (u64{250} << 8) | (u64{1} << (target + 20))); + break; + } + case 0x882: { // UMOV immediate. + if (dest >= 16 || insn.words[2]) return fail(index, "native uniform immediate move mode"); + mov_imm(static_cast<u32>(dest) + 216, insn.words[1]); + break; + } + case 0xc82: { // UMOV UR, UR, including uniform execution predicates. + const u32 source = uniform_register(insn.words[1]); + if (dest >= 16 || source == ~0U || insn.words[2] != 0x08000000) + return fail(index, "native uniform move mode"); + emit(0x5c98000000000000ULL | predicate | (dest + 216) | + (u64{source} << 20) | (u64{15} << 39)); + break; + } + case 0xab9: case 0xabb: { // ULDC U32/U64, direct or indexed. + const u32 uniform_dest = insn.Bits(16, 8), index_reg = insn.Bits(24, 8); + const u32 width = insn.Bits(73, 3); + if (uniform_dest >= 16 || (width != 4 && width != 5) || + (width == 5 && ((uniform_dest & 1) || uniform_dest >= 15)) || + insn.Bits(54, 5) >= 18 || insn.Bits(38, 16) % 4 || insn.Bits(62, 2) || + (insn.words[2] & ~0xe00U) != (opcode == 0xab9 ? 0U : 0x08000000U) || + (opcode == 0xabb && index_reg >= 16)) + return fail(index, "native uniform constant load mode"); + if (opcode == 0xab9 && width == 4) { + emit(0x4c98000000000000ULL | predicate | (uniform_dest + 216) | (u64{15} << 39) | cbuf()); + } else { + emit(0xef90000000000000ULL | predicate | (uniform_dest + 216) | + (u64{opcode == 0xab9 ? 255U : index_reg + 216} << 8) | (u64{insn.Bits(38, 16)} << 20) | + (u64{insn.Bits(54, 5)} << 36) | (u64{width} << 48)); + } + break; + } + case 0x899: { // USHF.L.U32; accept a zero word outside the selected half. + const u32 dst = insn.Bits(16, 8), low = insn.Bits(24, 8), high = insn.Bits(64, 8); + if (insn.words[2] == (0x08011400U | high) && low == 63 && high < 16 && + dst < 16 && native_predicate == 7 && insn.words[1] <= 32) { + emit(0x3828000000000000ULL | predicate | (dst + 216) | + (u64{high + 216} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48)); + break; + } + const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high < 16; + const bool low_result = insn.words[2] == 0x0800063f && low < 16; + if (dst >= 16 || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result)) + return fail(index, "native uniform shift mode"); + emit(0x3848000000000000ULL | predicate | (dst + 216) | + (u64{(high_result ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20)); + break; + } + case 0xb82: { // LDC.64 indexed byte offset + const u32 width = insn.Bits(73, 3); + if ((width != 4 && width != 5) || insn.Bits(78, 2) || insn.Bits(91, 1) || + insn.Bits(54, 5) >= 18 || + (width == 5 && ((dest & 1) || dest >= 254))) return fail(index, "native indexed constant load mode"); + emit(base(0xef90000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(38, 16)} << 20) | (u64{insn.Bits(54, 5)} << 36) | (u64{width} << 48)); + break; + } + case 0xf89: { // Quad-local SHFL.BFLY with immediate lane and segmentation mask. + const u32 lane = insn.Bits(53, 5), mask = insn.Bits(40, 13); + if (insn.words[2] != 0x000e0000 || insn.Bits(32, 8) || + insn.Bits(58, 6) != 3 || mask != 0x1c03 || (lane != 1 && lane != 2)) + return fail(index, "native quad butterfly mode"); + emit(base(0xef10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{lane} << 20) | (u64{1} << 28) | (u64{1} << 29) | + (u64{3} << 30) | (u64{mask} << 34) | (u64{7} << 48)); + break; + } + case 0x822: { // FSWZADD, bounded quad derivative sign patterns. + const u32 swizzle = insn.words[1]; + if ((swizzle != 0x66 && swizzle != 0x5a) || insn.words[2] > 255) + return fail(index, "native swizzled add mode"); + // Maxwell encodes operand signs rather than the native operation enum. + emit(base(0x50f8000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(64, 8)} << 20) | (u64{swizzle == 0x66 ? 0x99U : 0xa5U} << 28)); + break; + } + case 0xb6c: { // TXD ARRAY_2D, four channels and consecutive split destinations. + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const u32 coordinates = insn.Bits(24, 8), derivatives = insn.Bits(32, 8); + if ((insn.words[2] & ~0xffU) != 0x001e0f00 || insn.Bits(59, 2) || + insn.Bits(61, 3) != 5 || binding >= 18 || offset > 0x1fff || + (dest & 1) || dest >= 248 || insn.Bits(64, 8) != dest + 2 || + coordinates >= 248 || derivatives >= 247) + return fail(index, "native array gradient texture mode"); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native texture descriptor buffer conflict"); + result.required_texture_buffer = binding; + emit(base(0xde00000000000000ULL) | (u64{coordinates} << 8) | + (u64{derivatives} << 20) | (u64{3} << 28) | (u64{15} << 31) | + (u64{offset} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51)); + break; + } + case 0x589: { // SHFL.IDX, register lane and immediate segmentation mask. + if (insn.words[2] != 0x000e0000 || insn.Bits(53, 11)) + return fail(index, "native shuffle mode"); + emit(0xef10000000000000ULL | predicate | dest | + (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) | + (u64{1} << 29) | (u64{insn.Bits(40, 13)} << 34) | (u64{7} << 48)); + break; + } + case 0x98e: { // RED.E U32, direct register address; ADD through XOR. + if ((insn.words[2] & ~0x03800000U) != 0x0010e100 || + insn.Bits(40, 24) || (insn.Bits(24, 8) & 1)) + return fail(index, "native global reduction mode"); + emit(0xebf8000000000000ULL | predicate | insn.Bits(32, 8) | + (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(87, 3)} << 23) | (u64{1} << 48)); + break; + } + case 0x2bd: // UFLO.U32, return the highest set-bit index. + if (dest >= 16 || insn.words[1] >= 16 || insn.words[2] != 0x080e0000 || native_predicate != 7) + return fail(index, "native uniform leading bit mode"); + emit(0x5c30000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 20)); + break; + case 0x3a1: { // MATCH.ALL.U64 with only the all-equal predicate retained. + const u32 source = insn.Bits(24, 8), target = insn.Bits(81, 3); + if (dest != 255 || source >= 254 || (source & 1) || target >= 6 || + native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0000U) != 0x200) + return fail(index, "native warp match mode"); + save_predicate(); + emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); + emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49)); + emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48)); + emit(0x5c30000000070000ULL | 250 | (u64{250} << 20)); + for (u32 word = 0; word < 2; ++word) + emit(0xef10000000070000ULL | (251 + word) | (u64{source + word} << 8) | + (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source} << 8) | + (u64{251} << 20) | (u64{7} << 39) | (u64{2} << 49)); + emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source + 1} << 8) | + (u64{252} << 20) | (u64{6} << 39) | (u64{2} << 49)); + emit(0x50d8000000070000ULL | 255 | (u64{6} << 39) | (u64{target} << 45)); + restore_predicate(); + break; + } + case 0x3c4: { // REDUX.SUM over the actual active lanes, modulo 2^32. + const u32 source = insn.Bits(24, 8); + if (dest >= 16 || insn.words[1] || insn.words[2] != 0xc000 || native_predicate != 7) + return fail(index, "native warp sum mode"); + emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); + mov_imm(static_cast<u32>(dest) + 216, 0); + for (u32 lane = 0; lane < 32; ++lane) { + emit(0xef10000000070000ULL | 251 | (u64{source} << 8) | (u64{lane} << 20) | + (u64{1} << 28) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); + emit(0x3828000000070000ULL | 252 | (u64{250} << 8) | (u64{lane} << 20)); + emit(0x0400000000070000ULL | 252 | (u64{252} << 8) | (u64{1} << 20)); + emit(0x5c10000000070000ULL | 252 | (u64{255} << 8) | (u64{252} << 20) | (u64{1} << 48)); + emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48)); + emit(0x5c10000000070000ULL | (dest + 216) | (u64{dest + 216} << 8) | (u64{251} << 20)); + } + break; + } + case 0xf8c: { // ATOMS.POPC.INC.32 with no returned value: one increment per active lane. + const u32 stride = insn.Bits(78, 2); + if (dest != 255 || insn.words[1] || (insn.words[2] & ~0xc000U) != 0x0d00003f) + return fail(index, "native shared increment mode"); + u32 address = insn.Bits(24, 8); + if (stride && address != 255) { + emit(0x3848000000000000ULL | predicate | 251 | (u64{address} << 8) | + (u64{stride + 1} << 20)); + address = 251; + } + mov_imm(250, 1); + emit(0xec00000000000000ULL | predicate | 255 | (u64{address} << 8) | (u64{250} << 20)); + break; + } + case 0x38c: { // ATOMS.ADD.U32, naturally aligned shared byte offset. + const u32 offset = insn.Bits(40, 24); + if (insn.words[2] || (offset & 3)) return fail(index, "native shared atomic mode"); + emit(0xec00000000000000ULL | predicate | dest | + (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) | + (u64{offset >> 2} << 30)); + break; + } + case 0x9a8: { // ATOMG.E.ADD.STRONG.GPU U32 through a uniform pointer. + const u32 address = uniform_register(insn.Bits(64, 6)); + if (insn.Bits(24, 8) != 255 || insn.Bits(40, 24) || + (insn.words[2] & ~0x3fU) != 0x081ee140 || address == ~0U || (address & 1)) + return fail(index, "native global atomic mode"); + emit(0xed00000000000000ULL | predicate | dest | (u64{address} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{1} << 48)); + break; + } + case 0xb97: { // SULD.P.2D.R.IGN, direct descriptor and red channel only. + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const u32 coordinate = insn.Bits(24, 8); + if (insn.words[2] != 0x001e0100 || insn.Bits(61, 3) != 3 || + binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || dest == 255 || coordinate > 253) + return fail(index, "native formatted surface load mode"); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native surface descriptor buffer conflict"); + result.required_texture_buffer = binding; + emit(base(0xeb00000000000000ULL) | (u64{coordinate} << 8) | + (u64{1} << 20) | (u64{3} << 33) | (u64{offset} << 36) | (u64{1} << 51)); + break; + } + case 0xb9b: { // SUST.P.2D/3D.IGN, direct descriptor and all four channels. + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const u32 data = insn.Bits(32, 8), coordinate = insn.Bits(24, 8); + const u32 dimension = insn.Bits(61, 3); + if (insn.words[2] != 0x00100f00 || (dimension != 3 && dimension != 5) || + binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || data > 252 || coordinate > (dimension == 5 ? 252U : 253U)) + return fail(index, "native formatted surface store mode"); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native surface descriptor buffer conflict"); + result.required_texture_buffer = binding; + emit(0xeb20000000000000ULL | predicate | data | (u64{coordinate} << 8) | + (u64{15} << 20) | (u64{dimension} << 33) | (u64{offset} << 36) | (u64{1} << 51)); + break; + } + case 0x301: // BREV register. + if (insn.words[2] || insn.words[1] > 255) return fail(index, "native bit reverse mode"); + emit(base(0x3828000000000000ULL) | (u64{insn.words[1]} << 8) | + (u64{1} << 39) | (u64{1} << 40)); + break; + case 0x219: { // U32 register shifts with a zero word outside the selected result. + const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8), shift = insn.Bits(32, 8); + const bool right = insn.Bits(76, 1) != 0, high_result = insn.Bits(80, 1) != 0; + if (insn.words[1] > 255 || insn.Bits(73, 2) != 3 || + (insn.words[2] & ~0x000118ffU) != 0x00000600U || + (right ? (!high_result || low != 255) : (high_result || high != 255))) + return fail(index, "native register shift mode"); + emit(base(right ? 0x5cf8000000000000ULL : 0x5bf8000000000000ULL) | + (u64{right ? high : 255U} << 8) | (u64{shift} << 20) | + (u64{right ? 255U : low} << 39) | (u64{insn.Bits(75, 1)} << 50)); + break; + } + case 0x300: case 0xd00: { // FLO, including a uniform source. + if ((insn.words[2] & ~(0x600U | (opcode == 0xd00 ? 0x08000000U : 0U))) != 0x000e0000U) + return fail(index, "native leading bit mode"); + const u32 source = operand(1, false); + emit(base(0x5c30000000000000ULL) | (u64{source} << 20) | + (u64{insn.Bits(63, 1)} << 40) | (u64{insn.Bits(74, 1)} << 41) | + (u64{insn.Bits(73, 1)} << 48)); + break; + } + case 0x309: case 0xd09: { // POPC, including a uniform source. + if (insn.words[2] != (opcode == 0xd09 ? 0x08000000U : 0U)) + return fail(index, "native population count mode"); + const u32 source = operand(1, false); + emit(base(0x5c08000000000000ULL) | (u64{source} << 20) | (u64{insn.Bits(63, 1)} << 40)); + break; + } + case 0x918: + if (insn.words[1] || insn.words[2]) return fail(index, "native NOP mode"); + emit(0x50b0000000000f00ULL | predicate); + break; + case 0x992: { // MEMBAR.SC: preserve workgroup/device visibility. + const u32 scope = insn.Bits(76, 3); + if (insn.words[1] || insn.words[2] != (scope << 12) || + (scope != 0 && scope != 2 && scope != 3 && scope != 5)) + return fail(index, "native memory barrier scope"); + const u32 legacy_scope = scope == 0 ? 0 : scope == 2 ? 1 : scope == 3 ? 2 : 3; + emit(0xef98000000000000ULL | predicate | (u64{legacy_scope} << 8)); + break; + } + case 0x9ab: // ERRBAR: wait for prior memory operations on the host backend. + if (insn.words[1] || insn.words[2]) return fail(index, "native error barrier mode"); + emit(0xef98000000000000ULL | predicate | (u64{1} << 8)); + break; + case 0xb1d: // BAR.SYNC 0, full workgroup. + if (insn.words[1] || insn.words[2] || native_predicate != 7) + return fail(index, "native workgroup barrier mode"); + emit(0xf0a8000000000000ULL | predicate | (u64{1} << 43) | + (u64{1} << 44) | (u64{1} << 39) | (u64{7} << 39)); + break; + case 0x388: case 0x984: { // Shared memory load/store with scaled register offsets. + const u32 width = insn.Bits(73, 3), stride = insn.Bits(78, 2); + if (width > 6 || insn.words[2] != ((width << 9) | (stride << 14)) || + (opcode == 0x984 && insn.Bits(32, 8))) + return fail(index, "native shared memory addressing mode"); + u32 address = insn.Bits(24, 8); + if (stride && address != 255) { + emit(0x3848000000000000ULL | predicate | 250 | + (u64{address} << 8) | (u64{stride + 1} << 20)); + address = 250; + } + const u32 data = opcode == 0x388 ? insn.Bits(32, 8) : static_cast<u32>(dest); + emit((opcode == 0x388 ? 0xef58000000000000ULL : 0xef48000000000000ULL) | + predicate | data | (u64{address} << 8) | (u64{insn.Bits(40, 24)} << 20) | + (u64{width} << 48)); + break; + } + case 0x919: case 0x9c3: { // S2R / S2UR thread and workgroup identities. + const u32 special = insn.Bits(72, 8); + if (insn.words[1] || insn.words[2] != (special << 8) || + (special != 0 && special != 33 && special != 34 && special != 35 && + special != 37 && special != 38 && special != 39 && + special != 56 && special != 57 && special != 58 && special != 59 && special != 60)) + return fail(index, "native special register"); + const u32 target = opcode == 0x9c3 ? uniform_register(static_cast<u32>(dest)) : static_cast<u32>(dest); + if (target == ~0U) return fail(index, "native uniform special destination"); + emit(0xf0c8000000000000ULL | predicate | target | (u64{special} << 20)); + break; + } + case 0x28c: case 0x88c: { // UISETP U32/I32 with UPT accumulation. + const u32 output = insn.Bits(81, 3); + const u32 a = uniform_register(insn.Bits(24, 8)), b = opcode == 0x88c ? 252U : uniform_register(insn.Bits(32, 8)); + if (output > 1 || insn.Bits(84, 3) != 7 || insn.Bits(87, 4) != 7 || + (insn.Bits(72, 4) & ~2U) || insn.Bits(12, 4) != 7 || a == ~0U || b == ~0U) + return fail(index, "native uniform comparison mode"); + const u32 temporary = 6 - output, mask = 1U << temporary; + if (opcode == 0x88c) mov_imm(252, insn.words[1]); + emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + emit(0x5b60000000070000ULL | 7 | (u64{temporary} << 3) | + (u64{a} << 8) | (u64{b} << 20) | (u64{7} << 39) | + (u64{insn.Bits(73, 1)} << 48) | (u64{insn.Bits(76, 3)} << 49)); + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20)); + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + break; + } + case 0x287: case 0x887: { // USEL, UP0/UP1; preserve ordinary predicates. + const u32 condition = insn.Bits(87, 3); + const u32 target = uniform_register(static_cast<u32>(dest)); + const u32 a = uniform_register(insn.Bits(24, 8)), b = opcode == 0x887 ? 252U : uniform_register(insn.Bits(32, 8)); + if (condition > 1 || target == ~0U || a == ~0U || b == ~0U || + insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x07800000U) != 0x08000000U) + return fail(index, "native uniform select mode"); + if (opcode == 0x887 && index > 0 && condition == 0 && insn.words[1] <= 0x7ffff && + code[index - 1].Bits(0, 12) == 0x88c && + code[index - 1].Bits(24, 8) == insn.Bits(24, 8) && + code[index - 1].words[1] == insn.words[1] && code[index - 1].words[2] == 0x0bf01070 && + insn.words[2] == 0x08000000) { + // Preserve the unsigned clamp idiom for indirect-table analysis. + emit(0x3820000000070000ULL | target | (u64{a} << 8) | + (u64{insn.words[1]} << 20) | (u64{7} << 39)); + break; + } + if (opcode == 0x887) mov_imm(252, insn.words[1]); + const u32 temporary = 6 - condition, mask = 1U << temporary; + emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{mask} << 20)); + emit(0x5ca0000000070000ULL | target | (u64{a} << 8) | (u64{b} << 20) | + (u64{temporary} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + break; + } + case 0x981: { // LDG through a uniform 64-bit base plus a register offset. + const u32 uniform_address = uniform_register(insn.Bits(32, 6)); + u32 address = insn.Bits(24, 8); + const u32 width = insn.Bits(73, 3); + if ((insn.words[2] & ~(0x04000000U | 0xe00U)) != 0x081ea100 || + insn.Bits(38, 2) || uniform_address == ~0U || + (uniform_address != 255 && (uniform_address & 1)) || width > 6 || + (width == 5 && ((dest & 1) || dest >= 254)) || + (width == 6 && ((dest & 3) || dest >= 252))) + return fail(index, "native global load uniform addressing mode"); + if (address == 255) address = uniform_address; + else { + if (insn.Bits(90, 1) && ((address & 1) || address >= 254)) + return fail(index, "native global load register address pair"); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{uniform_address} << 8) | + (u64{address} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 251 | + (u64{uniform_address == 255 ? 255U : uniform_address + 1} << 8) | + (u64{insn.Bits(90, 1) ? address + 1 : 255} << 20) | (u64{1} << 43)); + address = 250; + } + emit(base(0xeed0000000000000ULL) | (u64{address} << 8) | + (u64{insn.Bits(40, 24)} << 20) | (u64{1} << 45) | (u64{width} << 48)); + break; + } + case 0x386: case 0x986: { // Global store, original data and 64-bit address. + const u32 data = insn.Bits(32, 8), width = insn.Bits(73, 3); + if (width > 6 || (width == 5 && ((data & 1) || data >= 254)) || + (width == 6 && ((data & 3) || data >= 252))) + return fail(index, "native global store width/alignment"); + u32 address = insn.Bits(24, 8); + bool extended = insn.Bits(72, 1) != 0; + if (opcode == 0x986) { + const u32 uniform_address = uniform_register(insn.Bits(64, 6)); + if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U)) != 0x0810a100 || + uniform_address == ~0U || (uniform_address & 1) || !extended) + return fail(index, "native global store uniform addressing mode"); + if (address == 255) address = uniform_address; + else { + if (insn.Bits(90, 1) && ((address & 1) || address >= 254)) + return fail(index, "native global store register address pair"); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{uniform_address} << 8) | + (u64{address} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 251 | (u64{uniform_address + 1} << 8) | + (u64{insn.Bits(90, 1) ? address + 1 : 255} << 20) | (u64{1} << 43)); + address = 250; + } + } else if (insn.words[2] != 0x0010a900) { + return fail(index, "native global store register addressing mode"); + } + emit(0xeed8000000000000ULL | predicate | data | + (u64{address} << 8) | (u64{insn.Bits(40, 24)} << 20) | + (u64{extended} << 45) | (u64{width} << 48)); + break; + } + case 0x95b: // Fragment discard; execution predicate is preserved. + if (insn.words[1] || insn.words[2] != 0x03800000) + return fail(index, "native fragment discard mode"); + emit(0xe33000000000000fULL | predicate); + break; + case 0x305: case 0xb05: { // F2I NTZ, register/constant-buffer source + const bool half = opcode == 0x305 && insn.words[2] == 0x0010f000; + const bool small = insn.words[2] == 0x00212800; + const bool floor = (insn.words[2] & ~0x100U) == 0x00207000; + const bool nearest = (insn.words[2] & ~0x100U) == 0x00203000; + if ((!half && !small && !floor && !nearest && (insn.words[2] & ~0x100U) != 0x0021f000) || (opcode == 0x305 && (insn.words[1] & 0xffffff00U))) { + return fail(index, "native float-to-integer conversion mode"); + } + const bool is_signed = insn.Bits(72, 1); + u32 source = opcode == 0xb05 ? materialize(2, 0, 252, false, false) : insn.Bits(32, 8); + if (half) { + extract_half({source, 2, false, false}, 0, 252); + source = 252; + } + if (!is_signed) { + // Clear NaNs without consuming a guest predicate. Clamp the + // conversion below 2^32, retaining an overflow mask for saturation. + emit(0x5800000000000000ULL | predicate | 250 | (u64{source} << 8) | + (u64{source} << 20) | (u64{7} << 39) | (u64{7} << 48)); + emit(0x5c40000000000000ULL | predicate | 250 | (u64{source} << 8) | + (u64{250} << 20) | (u64{7} << 48)); + if (!small) { + mov_imm(252, 0x4f800000); + emit(0x5800000000000000ULL | predicate | 251 | (u64{250} << 8) | + (u64{252} << 20) | (u64{7} << 39) | (u64{6} << 48)); + mov_imm(252, 0x4f7fffff); + emit(0x5c60000000000000ULL | predicate | 250 | (u64{250} << 8) | + (u64{252} << 20) | (u64{7} << 39)); + } + source = 250; + } + emit(base(0x5cb0000000000000ULL) | (u64{small ? 1U : 2U} << 8) | (u64{2} << 10) | + (u64{is_signed} << 12) | (u64{source} << 20) | + (u64{small || nearest ? 0U : floor ? 1U : 3U} << 39) | (u64{insn.Bits(80, 1)} << 44)); + if (!is_signed && !small) { + emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{251} << 20) | + (u64{1} << 41) | (u64{7} << 48)); + } + break; + } + case 0xb6f: { // TXQ texture dimensions, two packed scalar outputs. + if (insn.Bits(59, 5) || insn.Bits(64, 8) != 255 || insn.Bits(72, 4) != 3 || + insn.Bits(76, 14) || insn.Bits(91, 1) || insn.Bits(54, 5) != 2 || + insn.Bits(40, 14) > 0x1fff || (dest & 1) || dest >= 254) + return fail(index, "native texture dimension query mode"); + result.required_texture_buffer = 2; + emit(0xdf48000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | + (u64{1} << 22) | (u64{3} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49)); + break; + } + case 0xb66: { // TLD.SCR.LZ, direct texel fetch + const u32 mask = insn.Bits(72, 4); + const u32 dimension = insn.Bits(61, 3); + const bool array = dimension == 5; + const bool volume = dimension == 2; + u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + const u32 second_dest = insn.Bits(64, 8); + const bool split = second_dest != 255; + constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12}; + constexpr std::array<u32, 5> rgba_masks{7, 11, 13, 14, 15}; + u32 swizzle = ~0U; + if (split) { + for (u32 i = 0; i < rgba_masks.size(); ++i) if (rgba_masks[i] == mask) swizzle = i; + } else { + for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i; + } + if (!insn.Bits(60, 1) || (dimension != 0 && dimension != 1 && !array && !volume) || + swizzle == ~0U || insn.Bits(76, 5) || + insn.Bits(81, 3) != 7 || insn.Bits(87, 3) != 1 || + insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 || + dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) || + (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) || + (array && ((source_a & 1) || source_a >= 249)) || + (volume && ((source_a & 1) || source_a >= 254))) { + return fail(index, "native texel fetch mode or descriptor binding"); + } + const u32 binding = insn.Bits(54, 5); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native texture descriptor buffer conflict"); + result.required_texture_buffer = binding; + if (array) { + emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a + 1} << 20)); + emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{source_b} << 20)); + source_b = 250; + } + emit(0xda00000000000000ULL | predicate | dest | (u64{source_a} << 8) | + (u64{source_b} << 20) | (u64{insn.Bits(64, 8)} << 28) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | + (u64{swizzle} << 50) | (u64{array ? 8U : volume ? 7U : dimension == 0 ? 0U : 2U} << 53)); + break; + } + case 0x91a: // DEPBAR.LE SB5, 1: scoreboard wait, represented by IR data dependencies. + if (insn.words[0] != 0x791a || (insn.words[1] & ~0xfc0U) != 0xd000 || insn.words[2]) { + return fail(index, "native dependency barrier mode"); + } + break; + case 0xb63: { // TLD4, scalar 2D gather with split RG/BA destinations. + const u32 second_dest = insn.Bits(64, 8); + if (!insn.Bits(60, 1) || insn.Bits(61, 3) != 1 || insn.Bits(72, 4) != 15 || + insn.Bits(76, 3) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || + (dest & 1) || dest >= 254 || (second_dest & 1) || second_dest >= 254) + return fail(index, "native 2D texture gather mode"); + result.required_texture_buffer = 2; + emit(0xdf00000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{second_dest} << 28) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | + (u64{insn.Bits(87, 2)} << 52)); + break; + } + case 0xb60: { // TEX, scalar 2D implicit-LOD sample + const u32 mask = insn.Bits(72, 4); + const u32 second_dest = insn.Bits(64, 8); + constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12}; + constexpr std::array<u32, 5> rgba_masks{7, 11, 13, 14, 15}; + u32 swizzle = ~0U; + const bool split = second_dest != 255; + const bool depth_compare = insn.Bits(78, 1); + const u32 dimension = insn.Bits(61, 3), lod = insn.Bits(87, 3); + const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + if (!insn.Bits(60, 1) && dimension == 7 && lod == 3) { + const u32 components = std::popcount(mask); + if (depth_compare || !components || split != (components > 2) || + insn.Bits(76, 1) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || + source_a >= 246 || source_b >= 250 || dest == 255 || + (components > 1 && ((dest & 1) || dest >= 254)) || + (components == 4 && ((second_dest & 1) || second_dest >= 254))) + return fail(index, "native array-cube vector sample mode"); + result.required_texture_buffer = 2; + // Vector coordinates already use layer,X,Y,Z order. LOD is + // supplied separately, matching the Maxwell TEX operand ABI. + emit(0xc000000000000000ULL | predicate | 250 | (u64{source_a} << 8) | + (u64{source_b} << 20) | (u64{7} << 28) | (u64{mask} << 31) | + (u64{insn.Bits(77, 1)} << 35) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | (u64{lod} << 55)); + for (u32 i = 0; i < components; ++i) + emit(0x5c98000000000000ULL | predicate | + (i < 2 ? dest + i : second_dest + i - 2) | + (u64{15} << 39) | (u64{250 + i} << 20)); + break; + } + if (dimension == 5 && depth_compare && lod == 0 && !split && mask == 1) { + if (!insn.Bits(60, 1) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 || + insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || + (source_a & 1) || (source_b & 1) || source_a >= 248 || source_b >= 248 || dest >= 248) + return fail(index, "native array depth comparison sample mode"); + result.required_texture_buffer = 2; + for (u32 i = 0; i < 3; ++i) { + const u32 source = i < 2 ? source_a + i : source_b; + emit(0x5c98000000000000ULL | predicate | (250 + i) | + (u64{15} << 39) | (u64{source} << 20)); + } + emit(0xc000000000000000ULL | predicate | dest | (u64{250} << 8) | + (u64{source_b + 1} << 20) | (u64{3} << 28) | (u64{1} << 31) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | + (u64{1} << 50) | (u64{7} << 51)); + break; + } + if ((dimension == 3 && lod == 3) || (dimension == 2 && lod <= 1)) { + const u32 components = std::popcount(mask); + if (!insn.Bits(60, 1) || depth_compare || !components || + split != (components > 2) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 || + insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || + (source_a & 1) || (dimension == 3 && (source_b & 1)) || source_a >= 249 || source_b >= 249 || + dest == 255 || (components > 1 && ((dest & 1) || dest >= 254)) || + (components == 4 && ((second_dest & 1) || second_dest >= 254))) + return fail(index, "native volume/cube sample mode"); + result.required_texture_buffer = 2; + const auto copy = [&](u32 target, u32 source) { + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20)); + }; + copy(250, source_a); + copy(251, source_a + 1); + copy(252, source_b); + emit(0xc000000000000000ULL | predicate | 250 | (u64{250} << 8) | + (u64{dimension == 3 ? source_b + 1 : 255U} << 20) | (u64{dimension == 3 ? 6U : 4U} << 28) | (u64{mask} << 31) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | + (u64{7} << 51) | (u64{lod} << 55)); + for (u32 i = 0; i < components; ++i) + copy(i < 2 ? static_cast<u32>(dest) + i : second_dest + i - 2, 250 + i); + break; + } + if (split) { + for (u32 i = 0; i < rgba_masks.size(); ++i) if (rgba_masks[i] == mask) swizzle = i; + } else { + for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i; + } + // SM70 scalar coordinates and split RG/BA destinations match TEXS. + // Descriptor offsets are words in both encodings, but only native + // explicitly encodes the constant-buffer binding. + if (!insn.Bits(60, 1) || (dimension != 1 && dimension != 5) || + swizzle == ~0U || insn.Bits(76, 2) || + insn.Bits(81, 3) != 7 || (lod > 1 && (lod != 3 || dimension != 1 || depth_compare)) || + (lod == 3 && ((source_a & 1) || source_a >= 249)) || + insn.Bits(59, 1) || + insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || + dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) || + (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) || + (depth_compare && (dimension != 1 || split || mask != 1 || (source_a & 1) || source_a >= 249)) || + (dimension == 5 && ((source_a & 1) || source_a >= 249))) { + return fail(index, "native texture mode or descriptor binding"); + } + result.required_texture_buffer = 2; + emit(0xd800000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(64, 8)} << 28) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | + (u64{swizzle} << 50) | (u64{dimension == 5 ? (lod ? 8U : 7U) : depth_compare ? (lod ? 6U : 4U) : (lod == 3 ? 3U : lod ? 2U : 1U)} << 53)); + break; + } + case 0x816: { // Captured PRMT half-word concatenation + if ((insn.words[1] != 0x5410 && insn.words[1] != 0x5432 && insn.words[1] != 0x7610 && insn.words[1] != 0x7632) || + (insn.words[2] & ~0xffU)) return fail(index, "native permutation mode"); + const u32 a = insn.Bits(24, 8), b = insn.Bits(64, 8); + if (insn.words[1] != 0x5432 && insn.words[1] != 0x7632) { + emit(0x0400000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{0xffff} << 20)); + } else { + emit(0x3828000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{16} << 20)); + } + if (insn.words[1] == 0x7610 || insn.words[1] == 0x7632) { + emit(0x0400000000000000ULL | predicate | 251 | (u64{b} << 8) | (u64{0xffff0000} << 20)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{251} << 20)); + } else { + emit(base(0x5c18000000000000ULL) | (u64{b} << 8) | (u64{250} << 20) | (u64{16} << 39)); + } + break; + } + case 0x307: { // FRND.F32, round to an integral floating-point value + if (insn.Bits(75, 2) != 2 || insn.Bits(84, 2) != 2 || insn.Bits(81, 3) || + insn.Bits(86, 6)) return fail(index, "native floating round format"); + emit(base(0x5ca8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | + (u64{insn.Bits(32, 8)} << 20) | (u64{8 + insn.Bits(78, 2)} << 39) | + (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(63, 1)} << 45) | + (u64{insn.Bits(62, 1)} << 49)); + break; + } + case 0x321: // ALD + case 0x322: { // AST + if (insn.Bits(76, 2) || insn.Bits(79, 1)) return fail(index, "native indexed/patch attribute access"); + const u32 address = insn.Bits(40, 10); + const u32 components = insn.Bits(74, 2) + 1; + if (address % 4 || address + components * 4 > 0x400 || insn.Bits(24, 8) != 255) { + return fail(index, "native attribute address or indexing"); + } + const u32 value_reg = opcode == 0x321 ? static_cast<u32>(dest) : insn.Bits(32, 8); + if (value_reg + components > 255) return fail(index, "native attribute register range"); + const u32 vertex = opcode == 0x321 ? insn.Bits(32, 8) : insn.Bits(64, 8); + const u64 translated = (opcode == 0x321 ? 0xefd8000000000000ULL : 0xeff0000000000000ULL) | + predicate | value_reg | (u64{255} << 8) | (u64{address} << 20) | + (u64{vertex} << 39) | (u64{components - 1} << 47); + result.instructions.push_back(translated); + break; + } + case 0x387: case 0x983: { // Captured scalar local spill at byte offset zero. + const bool store = opcode == 0x387; + if (insn.Bits(24, 8) != 255 || (store && (insn.Bits(16, 8) || insn.words[1] > 247)) || + (!store && (insn.words[1] || dest >= 248)) || + insn.words[2] != (store ? 0x00100800U : 0x00300800U)) + return fail(index, "native local scalar spill mode"); + emit((store ? 0xef50000000000000ULL : 0xef40000000000000ULL) | predicate | + (store ? insn.words[1] : dest) | (u64{255} << 8) | (u64{4} << 48)); + break; + } + case 0x803: { // P2R PR, register merge source, immediate mask + if (insn.words[1] > 0x7f || insn.words[2]) + return fail(index, "native predicate-to-register mode"); + emit(base(0x38e8000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.words[1]} << 20)); + break; + } + case 0x804: { // R2P PR, register byte, immediate mask + if (insn.Bits(16, 8) || insn.words[1] > 0x7f || (insn.words[2] & ~0x3000U)) + return fail(index, "native register-to-predicate mode"); + emit(0x38f0000000000000ULL | predicate | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.words[1]} << 20) | (u64{insn.Bits(76, 2)} << 41)); + break; + } + case 0xc02: // MOV uniform register + case 0x202: // MOV register + case 0x802: // MOV immediate + case 0xa02: { // MOV constant buffer + if (insn.Bits(72, 4) != 15 || (insn.Bits(76, 16) & ~(opcode == 0xc02 ? 0x8000U : 0U))) return fail(index, "native MOV modifiers"); + if (opcode == 0x802) { + result.instructions.push_back(base(0x0100000000000000ULL) | (u64{15} << 12) | + (u64{insn.words[1]} << 20)); + } else if (opcode == 0xc02) { + if (insn.words[1] >= 16 || insn.words[2] != 0x08000f00) return fail(index, "native uniform move source"); + emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.words[1] + 216} << 20)); + } else if (opcode == 0x202) { + if (insn.Bits(40, 24)) return fail(index, "native MOV register modifiers"); + result.instructions.push_back(base(0x5c98000000000000ULL) | (u64{15} << 39) | + (u64{insn.Bits(32, 8)} << 20)); + } else { + if (!valid_cbuf() || insn.Bits(62, 2)) return fail(index, "native MOV constant buffer"); + result.instructions.push_back(base(0x4c98000000000000ULL) | (u64{15} << 39) | cbuf()); + } + break; + } + case 0x223: case 0x423: case 0x623: case 0x823: case 0xa23: case 0xc23: case 0xe23: { // FFMA + if ((insn.Bits(76, 1) && !insn.Bits(80, 1)) || + (insn.Bits(81, 11) & ~((opcode == 0xc23 || opcode == 0xe23) ? 0x400U : 0U))) return fail(index, "native FFMA denormal/source mode"); + const u32 a = operand(0), b = operand(1), c = operand(2); + emit(base(0x5980000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | (u64{c} << 39) | + (u64{insn.Bits(77, 1)} << 50) | (u64{insn.Bits(78, 2)} << 51) | + (u64{insn.Bits(76, 1) ? 2U : insn.Bits(80, 1)} << 53)); + break; + } + case 0x220: case 0x820: case 0xa20: case 0xc20: { // FMUL + const u32 scale = insn.Bits(84, 3); + if (insn.Bits(76, 1) || (scale != 3 && scale != 4 && scale != 5 && scale != 6) || insn.Bits(81, 3) || + insn.Bits(87, 4) || (insn.Bits(91, 1) != 0) != (opcode == 0xc20) || + (scale != 4 && (!insn.Bits(80, 1) || insn.Bits(78, 2)))) + return fail(index, "native FMUL scale/mode"); + const u32 a = operand(0), b = operand(1); + emit(base(0x5c68000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(78, 2)} << 39) | (u64{scale == 3 ? 1U : scale == 5 ? 6U : scale == 6 ? 5U : 0U} << 41) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(77, 1)} << 50)); + break; + } + case 0x221: case 0x421: case 0x621: case 0x821: case 0xa21: case 0xe21: { // FADD + if (insn.Bits(81, 10) || (insn.Bits(91, 1) != 0) != (opcode == 0xe21)) return fail(index, "native FADD mode"); + const u32 a = operand(0), b = operand(insn.Bits(9, 3) == 2 || insn.Bits(9, 3) == 3 || insn.Bits(9, 3) == 7 ? 2 : 1); + emit(base(0x5c58000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(78, 2)} << 39) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(77, 1)} << 50)); + break; + } + case 0x209: case 0x809: case 0xa09: { // FMNMX + if (insn.Bits(74, 6) || insn.Bits(81, 6) || insn.Bits(91, 1)) + return fail(index, "native floating min/max mode"); + const u32 a = operand(0), b = operand(1); + emit(base(0x5c60000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | + (u64{insn.Bits(80, 1)} << 44)); + break; + } + case 0x308: case 0xb08: case 0xd08: { // MUFU, including uniform sources + const u32 operation = insn.Bits(74, 6); + if (operation > 8 || operation == 6 || operation == 7 || insn.Bits(80, 11) || + (insn.Bits(91, 1) != 0) != (opcode == 0xd08)) return fail(index, "native MUFU mode"); + if (insn.Bits(73, 1)) { + if (opcode != 0x308 || insn.Bits(60, 2) > 1) + return fail(index, "native half special-function source"); + emit(0x5ca8000000000000ULL | predicate | 250 | (u64{2} << 8) | (u64{1} << 10) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(60, 1)} << 41) | + (u64{insn.Bits(63, 1)} << 45) | (u64{insn.Bits(62, 1)} << 49)); + emit(0x5080000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{operation} << 20)); + emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) | (u64{250} << 20)); + } else { + const u32 src = operand(1); + emit(base(0x5080000000000000ULL) | (u64{src} << 8) | (u64{operation} << 20)); + } + break; + } + case 0x526: { // IPA with a uniform byte offset into the varying array. + const u32 source = uniform_register(insn.Bits(24, 8)); + if (source == ~0U || insn.words[1] != 255 || + (insn.Bits(76, 4) != 0 && insn.Bits(76, 4) != 4) || + insn.Bits(81, 3) != 7 || insn.Bits(84, 7) || !insn.Bits(91, 1)) + return fail(index, "native indexed interpolation mode"); + const u32 offset = insn.Bits(64, 8) * 4; + emit(0x1c00000000000000ULL | predicate | 250 | (u64{source} << 8) | (u64{offset} << 20)); + emit(base(0xe000000000000000ULL) | (u64{250} << 8) | (u64{255} << 20) | + (u64{1} << 38) | (u64{insn.Bits(78, 1)} << 55)); + break; + } + case 0x326: { // IPA.PASS / CONSTANT, default sample location + if (insn.Bits(32, 8) != 255 || (insn.Bits(76, 4) != 0 && insn.Bits(76, 4) != 4) || insn.Bits(81, 3) != 7) return fail(index, "native IPA interpolation mode"); + emit(base(0xe000000000000000ULL) | (u64{255} << 8) | (u64{255} << 20) | (u64{insn.Bits(64, 8)} << 30) | (u64{insn.Bits(78, 1)} << 55)); + break; + } + case 0x805: { // CS2R SRZ + if (insn.Bits(72, 8) != 255 || !insn.Bits(80, 1) || dest >= 249) return fail(index, "native special-register read"); + mov_imm(static_cast<u32>(dest), 0); + mov_imm(static_cast<u32>(dest) + 1, 0); + break; + } + case 0x306: { // I2F.F32.U8, low byte, round-to-nearest + if (insn.words[2] != 0x1000 || (insn.words[1] & 0xffffff00U)) + return fail(index, "native byte-to-float conversion mode"); + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{insn.Bits(32, 8)} << 20)); + break; + } + case 0x245: case 0xa45: case 0xc45: { // I2FP.F32.U32 + if ((insn.words[2] & ~(0x400U | (opcode == 0xc45 ? 0x08000000U : 0U))) != 0x00201000) return fail(index, "native packed integer-to-float format"); + const u32 src = operand(1, false); + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | (u64{insn.Bits(74, 1)} << 13) | (u64{src} << 20)); + break; + } + case 0x207: case 0x807: case 0xa07: + case 0x208: case 0x808: case 0xa08: { // Bitwise SEL / FSEL + const u32 a = operand(0, false), b = operand(1, false); + emit(base(0x5ca0000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + break; + } + case 0x20a: case 0x80a: case 0xa0a: { // FSET.BF with PT/AND + if (insn.Bits(64, 8) || insn.Bits(74, 2) || insn.Bits(81, 6) || + insn.Bits(87, 4) != 7 || insn.Bits(91, 1)) + return fail(index, "native floating comparison output mode"); + const u32 a = operand(0), b = operand(1); + emit(base(0x5800000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | + (u64{7} << 39) | (u64{insn.Bits(76, 4)} << 48) | (u64{1} << 52) | + (u64{insn.Bits(80, 1)} << 55)); + break; + } + case 0x20b: case 0x80b: case 0xa0b: // FSETP + case 0x20c: case 0x80c: case 0xa0c: case 0xc0c: { // ISETP + const bool floating = (opcode & 0x1ff) == 0xb; + if (!floating && insn.Bits(72, 1)) return fail(index, "native extended integer comparison"); + const u32 a = operand(0, floating), b = operand(1, floating); + const u32 combine = insn.Bits(74, 2); + if (combine > 2) return fail(index, "native predicate combination"); + u64 out = (floating ? 0x5bb0000000000000ULL : 0x5b60000000000000ULL) | predicate | + insn.Bits(84, 3) | (u64{insn.Bits(81, 3)} << 3) | (u64{a} << 8) | (u64{b} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | (u64{combine} << 45); + if (floating) out |= (u64{insn.Bits(80, 1)} << 47) | (u64{insn.Bits(76, 4)} << 48); + else out |= (u64{insn.Bits(73, 1)} << 48) | (u64{insn.Bits(76, 3)} << 49); + emit(out); + break; + } + case 0x212: case 0x812: case 0xc12: { // Bounded LOP3 bitwise forms, no predicate output + const u32 lut = insn.Bits(72, 8); + if ((lut != 0xc0 && lut != 0xfc && lut != 0xf8 && lut != 0x48 && lut != 0xe2) || insn.Bits(80, 1) || + insn.Bits(87, 4) != 15 || + (opcode == 0x212 && insn.Bits(40, 24))) return fail(index, "native LUT/predicate mode"); + u32 b = opcode == 0xc12 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + if (b == ~0U) return fail(index, "native uniform logic source"); + if (opcode == 0x812) { + mov_imm(251, insn.words[1]); + b = 251; + } + const u32 a = insn.Bits(24, 8), c = insn.Bits(64, 8); + const auto logical = [&](u32 target, u32 lhs, u32 rhs, u32 operation, u32 pred_target = 7) { + emit(0x5c40000000000000ULL | predicate | target | (u64{lhs} << 8) | + (u64{rhs} << 20) | (u64{operation} << 41) | + (u64{pred_target == 7 ? 0U : 3U} << 44) | (u64{pred_target} << 48)); + }; + const u32 pred_target = insn.Bits(81, 3); + if (lut == 0xe2) { + logical(250, a, b, 0); + logical(252, c, b, 0); + result.instructions.back() |= u64{1} << 40; + logical(static_cast<u32>(dest), 250, 252, 1, pred_target); + } else if (lut == 0xf8) { + logical(250, b, c, false); + logical(static_cast<u32>(dest), a, 250, 1, pred_target); + } else if (lut == 0x48) { + logical(250, a, c, 2); + logical(static_cast<u32>(dest), 250, b, 0, pred_target); + } else { + logical(static_cast<u32>(dest), a, b, lut == 0xfc ? 1U : 0U, pred_target); + } + break; + } + case 0x217: case 0x817: case 0xc17: { // IMNMX register, immediate, or uniform operand. + if ((insn.words[2] & ~(0x04000200U | (opcode == 0xc17 ? 0x08000000U : 0U))) != 0x03800000 || + insn.words[1] > (opcode == 0x817 ? 0x7ffffU : 255U)) + return fail(index, "native integer min/max mode"); + const u32 source = opcode == 0xc17 ? uniform_register(insn.words[1]) : insn.words[1]; + if (source == ~0U) return fail(index, "native uniform min/max operand"); + emit(base(opcode == 0x817 ? 0x3820000000000000ULL : 0x5c20000000000000ULL) | + (u64{insn.Bits(24, 8)} << 8) | (u64{source} << 20) | + (u64{insn.Bits(87, 4)} << 39) | (u64{insn.Bits(73, 1)} << 48)); + break; + } + case 0x819: { // SHF.HI, immediate shift + const bool is_signed = insn.Bits(73, 2) == 2; + if (!insn.Bits(80, 1) && !insn.Bits(76, 1) && insn.Bits(64, 8) == 255 && + insn.Bits(73, 2) == 3 && !insn.Bits(75, 1) && insn.words[1] <= 32) { + emit(base(0x3848000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20)); + break; + } + if ((insn.Bits(73, 2) != 3 && !is_signed) || insn.Bits(75, 1) || !insn.Bits(80, 1) || insn.words[1] > 32) return fail(index, "native funnel-shift mode"); + const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8), shift = insn.words[1]; + if (insn.Bits(76, 1)) { + if (low != 255) return fail(index, "native nonzero low right-shift word"); + emit(base(0x3828000000000000ULL) | (u64{high} << 8) | (u64{shift} << 20) | + (u64{is_signed} << 48)); + } else { + if (is_signed) return fail(index, "native signed left funnel shift"); + emit(base(0x36f8000000000000ULL) | (u64{low} << 8) | (u64{high} << 39) | (u64{shift} << 20)); + } + break; + } + case 0xa11: { // LEA with constant base and carry predicate. + if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || + insn.Bits(81, 3) == 7 || insn.Bits(62, 2)) + return fail(index, "native constant LEA carry mode"); + const u32 b = operand(1, false); + emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) | + (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); + break; + } + case 0x211: case 0x811: { // LEA low word, no carry result + if (opcode == 0x211 && insn.Bits(80, 1)) { + if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(81, 3) != 7 || + insn.Bits(32, 8) != 255 || insn.Bits(64, 8) != 255 || insn.Bits(62, 2)) + return fail(index, "native high effective address mode"); + emit(base(0x3828000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{32 - insn.Bits(75, 5)} << 20)); + break; + } + if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || insn.Bits(81, 3) != 7 || + (opcode == 0x211 && insn.Bits(62, 2))) return fail(index, "native LEA carry/high mode"); + u32 b = insn.Bits(32, 8); + if (opcode == 0x811) { mov_imm(251, insn.words[1]); b = 251; } + emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39)); + break; + } + case 0x210: case 0x810: case 0xa10: { // IADD3 with zero third source and no carry outputs + if (opcode == 0xa10 && insn.Bits(81, 3) != 7) { + if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(84, 7) != 0x7f || + insn.Bits(77, 4) != 15 || insn.Bits(63, 1) || insn.Bits(72, 1)) + return fail(index, "native constant add carry mode"); + const u32 b = operand(1, false); + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) | + (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); + break; + } + if (opcode == 0xa10 && insn.Bits(74, 1)) { + if (insn.Bits(24, 8) != 255 || (insn.words[2] & ~0x07800000U) != 0x007fe4ffU) + return fail(index, "native extended constant add mode"); + const u32 b = operand(1, false); + mov_imm(250, 1); + emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(base(0x5c10000000000000ULL) | (u64{b} << 8) | (u64{250} << 20)); + break; + } + if (insn.Bits(64, 8) != 255 && !insn.Bits(73, 4) && insn.Bits(77, 7) == 0x7f && insn.Bits(84, 7) == 0x7f) { + const u32 b = operand(1, false); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | + (u64{b} << 20) | (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{insn.Bits(64, 8)} << 20)); + break; + } + if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(77, 7) != 0x7f || insn.Bits(84, 7) != 0x7f) return fail(index, "native integer carry/third source"); + if (opcode == 0xa10) { + const u32 b = operand(1, false); + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20) | + (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49)); + } else if (opcode == 0x210) { + if (insn.words[1] & ~0x800000ffU) return fail(index, "native integer add register modifiers"); + emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49)); + } else { + emit(base(0x1c00000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20) | (u64{insn.Bits(72, 1)} << 56)); + } + break; + } + case 0x949: case 0x958: { // Captured BRX / BRXU with absolute native-PC table values + if (insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x3ffffU) != (opcode == 0x958 ? 0x0b800000U : 0x03800000U)) + return fail(index, "native computed branch mode"); + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + const s64 relative = static_cast<s64>(raw << 14) >> 14; + if (s64{index} * 16 + 16 + relative != 0) + return fail(index, "native computed branch relative base"); + const s64 offset = -s64{address_of(static_cast<u32>(result.instructions.size()))} - 8; + if (offset < -0x800000) return fail(index, "native computed branch range"); + const u32 source = opcode == 0x958 ? uniform_register(insn.Bits(24, 8)) : insn.Bits(24, 8); + if (source == ~0U) return fail(index, "native uniform computed branch source"); + emit(0xe25000000000000fULL | predicate | (u64{source} << 8) | + ((static_cast<u64>(offset) & 0xffffff) << 20)); + break; + } + case 0x944: { // Verified CALL.REL.NOINC with R55 holding the return PC + // A predicated NOINC forward transfer can terminate a loop at its + // reconvergence point without a return-link setup or matching RET. + if (native_predicate != 7 && insn.words[1] == 16 && insn.words[2] == 0x03c00000 && + index + 2 < code.size() && code[index + 1].Bits(0, 12) == 0x947) { + relocations.push_back({static_cast<u32>(result.instructions.size()), index + 2}); + emit(0xe24000000000000fULL | predicate); + break; + } + if (main_exited || index == 0 || insn.words[0] != 0x7944 || + (insn.words[2] & ~0x3ffffU) != 0x03c00000 || + (code[index - 1].words[0] & ~0x00ff0000U) != 0x00007802 || + code[index - 1].Bits(16, 8) >= 250 || + code[index - 1].words[1] != (index + 1) * 16 || + code[index - 1].words[2] != 0xf00) + return fail(index, "native call linkage mode"); + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + const s64 relative = static_cast<s64>(raw << 14) >> 14; + const s64 target = s64{index} * 16 + 16 + relative; + if (target < 0 || target % 16) return fail(index, "native call destination alignment"); + const u32 link = code[index - 1].Bits(16, 8); + for (size_t call = 0; call < call_targets.size(); ++call) { + if (call_targets[call] == target / 16 && call_links[call] != link) + return fail(index, "native inconsistent call linkage"); + } + call_targets.push_back(static_cast<u32>(target / 16)); + call_links.push_back(link); + relocations.push_back({static_cast<u32>(result.instructions.size()), target / 16}); + emit(0xe26000000007000fULL); + break; + } + case 0x950: { // Verified RET.REL.NODEC epilogue restoring the call link + const u32 return_reg = insn.Bits(24, 8); + if (!main_exited || index < 2 || !active_link || + (insn.words[0] & 0x00ffffffU) != 0x00007950 || return_reg >= 249 || + (insn.words[2] & ~0x3ffffU) != 0x03c00000 || + code[index - 2].words[0] != (0x00007202U | (return_reg << 16)) || + code[index - 2].words[1] != *active_link || code[index - 2].words[2] != 0xf00 || + code[index - 1].words[0] != (0x00007802U | ((return_reg + 1) << 16)) || + code[index - 1].words[1] || code[index - 1].words[2] != 0xf00 || !barrier_tokens.empty()) + return fail(index, "native return linkage mode"); + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + const s64 relative = static_cast<s64>(raw << 14) >> 14; + if (s64{index} * 16 + 16 + relative != 0) + return fail(index, "native return relative base"); + for (const auto& relocation : relocations) { + if (relocation.target == index || relocation.target == index - 1) + return fail(index, "native branch bypasses return linkage"); + } + emit(0xe32000000007000fULL); + bool complete = true; + for (u32 target : call_targets) complete &= target <= index; + if (complete) return finish(index); + break; + } + case 0x947: case 0x945: case 0x941: { // BRA / BSSY B0 / BSYNC B0 + u64 branch_predicate = predicate; + if (opcode == 0x947 && insn.Bits(87, 4) != 7) { + if (insn.Bits(12, 4) != 7) { + relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1}); + emit(0xe24000000000000fULL | (u64{insn.Bits(12, 4) ^ 8U} << 16)); + } + branch_predicate = u64{insn.Bits(87, 4)} << 16; + } else if (insn.Bits(87, 4) != 7) { + return fail(index, "native extra branch condition"); + } + if (opcode == 0x941) { + if (barrier_tokens.empty() || barrier_tokens.back() != insn.Bits(16, 4) || insn.Bits(12, 4) != 7) return fail(index, "native reconvergence token"); + barrier_tokens.pop_back(); + emit(0xf0f800000007000fULL); + break; + } + if (opcode == 0x945) { + if (barrier_tokens.size() >= 16 || insn.Bits(16, 4) != barrier_tokens.size() || insn.Bits(12, 4) != 7) return fail(index, "native nested reconvergence token"); + barrier_tokens.push_back(insn.Bits(16, 4)); + } + s64 relative = static_cast<s32>(insn.words[1]); + if (opcode == 0x947) { + const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32); + relative = static_cast<s64>(raw << 14) >> 14; + } + const s64 target = s64{index} * 16 + 16 + relative; + if (target < 0 || target % 16) return fail(index, "native branch destination alignment"); + relocations.push_back({static_cast<u32>(result.instructions.size()), target / 16}); + emit((opcode == 0x947 ? 0xe24000000000000fULL | branch_predicate : 0xe29000000007000fULL)); + break; + } + case 0x94d: { // EXIT + if (insn.Bits(84, 2) || insn.Bits(87, 4) != 7) { + return fail(index, "native conditional/special EXIT"); + } + result.instructions.push_back(0xe30000000000000fULL | predicate); + if (insn.Bits(12, 4) != 7) break; + if (!barrier_tokens.empty()) return fail(index, "native unclosed reconvergence scope"); + if (call_targets.empty()) { + const bool pending_branch = std::any_of(relocations.begin(), relocations.end(), + [&](const Relocation& relocation) { return relocation.target > index; }); + if (!pending_branch) return finish(index); + break; + } + if (main_exited) return fail(index, "native library exits without returning"); + for (u32 target : call_targets) { + if (target <= index) return fail(index, "native call target overlaps main program"); + } + main_exited = true; + break; + } + default: + return fail(index, "unsupported native opcode"); + } + if (!operand_error.empty()) return fail(index, operand_error); + if (guarded_uniform) restore_predicate(); + } + return fail(static_cast<u32>(code.size()), main_exited ? "native program lacks captured library return" : "native program has no unconditional EXIT"); +} + +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ampere/program_address_map.h b/src/shader_recompiler/frontend/ampere/program_address_map.h new file mode 100644 index 0000000000..bee32a821c --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/program_address_map.h @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once +#include <optional> +#include <span> +#include <vector> +#include "common/common_types.h" + +namespace Shader::Ampere { +constexpr u64 NativeAddressMapMagic = 0x50414d4e4544454eULL; +inline bool HasNativeAddressMap(std::span<const u64> code) { + return code.size() >= 2 && code.back() == NativeAddressMapMagic; +} +inline void AppendNativeAddressMap(std::vector<u64>& code, std::span<const u32> offsets) { + if (offsets.empty()) return; + for (u32 index : offsets) code.push_back(0x50U + index * 8 + ((index + 1) / 3) * 8); + code.push_back(offsets.size()); + code.push_back(NativeAddressMapMagic); +} +inline std::optional<u32> NativeProgramAddress(std::span<const u64> code, u32 address) { + if (!HasNativeAddressMap(code) || address % 16) return {}; + const u64 count = code[code.size() - 2]; + if (!count || count > 65536 || count + 2 > code.size() || address / 16 >= count) return {}; + const size_t start = code.size() - static_cast<size_t>(count) - 2; + const u64 target = code[start + address / 16]; + if (target < 0x50 || target >= start * 8 || target % 8 || target % 32 == 0) return {}; + return static_cast<u32>(target); +} +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ampere/program_capture.h b/src/shader_recompiler/frontend/ampere/program_capture.h new file mode 100644 index 0000000000..a7f2351254 --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/program_capture.h @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h" + +namespace Shader::Ampere { +class ProgramCaptureState { +public: + enum class Result { Continue, Complete, InvalidTarget }; + static constexpr u32 MaximumInstructions = 65536; + + Result Observe(const Instruction& instruction, u32 index) { + const u32 opcode = instruction.Bits(0, 12); + if (opcode == 0x944 || opcode == 0x947 || opcode == 0x945) { + const u64 raw = instruction.words[1] | (u64{instruction.words[2] & 0x3ffff} << 32); + const s64 relative = opcode == 0x945 ? static_cast<s32>(instruction.words[1]) + : static_cast<s64>(raw << 14) >> 14; + const s64 target = s64{index} * 16 + 16 + relative; + if (target < 0 || target % 16 || target >= MaximumInstructions * 16) + return Result::InvalidTarget; + // Predicated NOINC transfers to the following reconvergence + // point are loop exits, not calls into a trailing function library. + const bool loop_exit = opcode == 0x944 && instruction.Bits(12, 4) != 7 && + instruction.words[1] == 16 && instruction.words[2] == 0x03c00000; + if (opcode == 0x944 && !loop_exit) { + last_call_target = std::max(last_call_target, static_cast<u32>(target / 16)); + } else { + last_forward_target = std::max(last_forward_target, static_cast<u32>(target / 16)); + } + } + if (opcode == 0x94d && instruction.Bits(12, 4) == 7 && index >= last_forward_target) { + if (!last_call_target) return Result::Complete; + main_exited = true; + } + if (opcode == 0x950 && instruction.Bits(12, 4) == 7 && main_exited && + index >= last_call_target && index >= last_forward_target) + return Result::Complete; + return Result::Continue; + } + +private: + u32 last_call_target{}; + u32 last_forward_target{}; + bool main_exited{}; +}; +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ampere/program_override.h b/src/shader_recompiler/frontend/ampere/program_override.h new file mode 100644 index 0000000000..0943028b0c --- /dev/null +++ b/src/shader_recompiler/frontend/ampere/program_override.h @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include <cstring> +#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h" + +namespace Shader::Ampere { + +inline u64 NativeProgramKey(std::span<const Instruction> source) { + u64 hash = 14695981039346656037ULL; + for (const auto& instruction : source) { + for (u32 word : instruction.words) { + for (u32 byte = 0; byte < 4; ++byte) hash = (hash ^ ((word >> (byte * 8)) & 255)) * 1099511628211ULL; + } + } + return hash; +} + +// Private iteration format: version, lengths, texture bank, exact original +// instructions, then the newly generated canonical instructions. Source bytes +// are compared directly; the filename hash is only an index. +inline std::vector<u8> EncodeProgramOverride(std::span<const Instruction> source, + const LoweredProgram& program) { + if (!program || program.source_size != source.size_bytes() || program.source_instruction_offsets.size() != source.size()) return {}; + const std::array<u32, 6> header{0x4e454445, 0x3230564f, + static_cast<u32>(source.size_bytes()), static_cast<u32>(program.instructions.size()), + program.required_texture_buffer, static_cast<u32>(program.source_instruction_offsets.size())}; + std::vector<u8> bytes(sizeof(header) + source.size_bytes() + program.instructions.size() * 8 + program.source_instruction_offsets.size() * 4); + std::memcpy(bytes.data(), header.data(), sizeof(header)); + std::memcpy(bytes.data() + sizeof(header), source.data(), source.size_bytes()); + std::memcpy(bytes.data() + sizeof(header) + source.size_bytes(), program.instructions.data(), program.instructions.size() * 8); + std::memcpy(bytes.data() + sizeof(header) + source.size_bytes() + program.instructions.size() * 8, + program.source_instruction_offsets.data(), program.source_instruction_offsets.size() * 4); + return bytes; +} + +inline u32 ProgramOverrideSourceSize(std::span<const u8> bytes) { + std::array<u32, 6> header{}; + if (bytes.size() < sizeof(header)) return 0; + std::memcpy(header.data(), bytes.data(), sizeof(header)); + const bool mapped = header[1] == 0x3230564f; + if (header[0] != 0x4e454445 || (!mapped && header[1] != 0x3130564f) || + (mapped ? header[5] != header[2] / 16 : header[5] != 0) || + header[2] == 0 || header[2] % sizeof(Instruction) || header[2] > 1048576 || + header[3] == 0 || header[3] > 1048576 || (header[4] != ~0U && header[4] >= 18)) return 0; + if (bytes.size() != sizeof(header) + size_t{header[2]} + size_t{header[3]} * 8 + size_t{header[5]} * 4) return 0; + return header[2]; +} + +inline LoweredProgram DecodeProgramOverride(std::span<const u8> bytes, + std::span<const Instruction> source) { + LoweredProgram result; + const auto fail = [&result](std::string_view message) { + result.instructions.clear(); + result.source_instruction_offsets.clear(); + result.error = message; + return result; + }; + std::array<u32, 6> header{}; + if (!ProgramOverrideSourceSize(bytes)) return fail("invalid native override header or length"); + std::memcpy(header.data(), bytes.data(), sizeof(header)); + if (header[2] != source.size_bytes()) return fail("native override source length mismatch"); + if (std::memcmp(bytes.data() + sizeof(header), source.data(), source.size_bytes())) + return fail("native override source mismatch"); + result.instructions.resize(header[3]); + std::memcpy(result.instructions.data(), bytes.data() + sizeof(header) + source.size_bytes(), header[3] * 8); + if (header[5]) { + result.source_instruction_offsets.resize(header[5]); + std::memcpy(result.source_instruction_offsets.data(), + bytes.data() + sizeof(header) + source.size_bytes() + size_t{header[3]} * 8, + size_t{header[5]} * 4); + u32 previous = 0; + for (u32 offset : result.source_instruction_offsets) { + if (offset >= result.instructions.size() || offset < previous) + return fail("native override address map mismatch"); + previous = offset; + } + } + result.source_size = header[2]; + result.required_texture_buffer = header[4]; + return result; +} + +} // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/maxwell/control_flow.cpp b/src/shader_recompiler/frontend/maxwell/control_flow.cpp index 7dabfbf16e..9fb30df664 100644 --- a/src/shader_recompiler/frontend/maxwell/control_flow.cpp +++ b/src/shader_recompiler/frontend/maxwell/control_flow.cpp @@ -457,7 +457,7 @@ CFG::AnalysisState CFG::AnalyzeBRX(Block* block, Location pc, Instruction inst, block->indirect_branches.reserve(targets.size()); for (const u32 target : targets) { - Block* const branch{AddLabel(block, block->stack, target, function_id)}; + Block* const branch{AddLabel(block, block->stack, env.TranslateProgramAddress(target), function_id)}; block->indirect_branches.push_back({ .block = branch, .address = target, diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 875c40604c..1bcb954334 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -17,7 +17,15 @@ add_executable(tests common/undefined_fix.cpp common/unique_function.cpp core/core_timing.cpp + core/program_metadata.cpp + core/native_gpu_characteristics.cpp + core/graphics_supervisor.cpp core/internal_network/network.cpp + video_core/native_semaphore.cpp + video_core/native_shader.cpp + video_core/native_swizzle.cpp + video_core/image_overlap.cpp + video_core/pending_download.cpp video_core/memory_tracker.cpp input_common/calibration_configuration_job.cpp ) diff --git a/src/tests/core/graphics_supervisor.cpp b/src/tests/core/graphics_supervisor.cpp new file mode 100644 index 0000000000..08e272f09a --- /dev/null +++ b/src/tests/core/graphics_supervisor.cpp @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <array> +#include "core/hle/service/am/service/gpu_memory_registration.h" +#include <catch2/catch_test_macros.hpp> +#include "core/hle/service/am/service/graphics_supervisor_protocol.h" +#include "core/hle/service/am/service/graphics_syncpoint_cache.h" +#include "core/hle/service/am/service/native_image_layout.h" + +using Service::AM::GraphicsSupervisorProtocol::DecodeBootstrap; + +TEST_CASE("Graphics supervisor decodes the captured TIPC bootstrap", "[graphics-supervisor]") { + const std::array<u32, 16> packet{0x14, 0x8000000c, 2, 0xffff8001, + 0x51, 0, 5, 0x80000000}; + const auto request = DecodeBootstrap(packet); + REQUIRE(request.has_value()); + REQUIRE(request->process_handle == 0xffff8001); + REQUIRE(request->applet_resource_user_id == 0x51); + REQUIRE(request->flags == 0x8000000000000005ULL); + REQUIRE(request->parameters == std::array<u64, 4>{}); + for (size_t length = 0; length < packet.size(); ++length) { + REQUIRE_FALSE(DecodeBootstrap(std::span{packet}.first(length))); + } +} + +TEST_CASE("Graphics supervisor rejects incompatible packet layouts", "[graphics-supervisor]") { + const std::array<u32, 16> original{0x14, 0x8000000c, 2, 0xffff8001, + 0x51, 0, 5, 0x80000000}; + for (const auto replacement : {0U, 0x15U, 0x10014U}) { + auto packet = original; + packet[0] = replacement; + REQUIRE_FALSE(DecodeBootstrap(packet)); + } + for (const auto replacement : {0U, 0x8000000bU, 0x8000000dU}) { + auto packet = original; + packet[1] = replacement; + REQUIRE_FALSE(DecodeBootstrap(packet)); + } + for (const auto replacement : {0U, 1U, 3U, 0x20U}) { + auto packet = original; + packet[2] = replacement; + REQUIRE_FALSE(DecodeBootstrap(packet)); + } +} + + +TEST_CASE("Graphics syncpoint cache extends real counter values across wraparound", "[graphics-supervisor]") { + using namespace Service::AM::GraphicsSyncpointCache; + REQUIRE(ExtendCounter(Unallocated, 7) == 7); + REQUIRE(ExtendCounter(7, 12) == 12); + REQUIRE(ExtendCounter(0xfffffffeULL, 0xffffffffU) == 0xffffffffULL); + REQUIRE(ExtendCounter(0xffffffffULL, 0) == 0x100000000ULL); + REQUIRE(ExtendCounter(0x1ffffffffULL, 3) == 0x200000003ULL); + REQUIRE(Header[0] == 1); + REQUIRE(Header[2] + 1 == CounterCount); + REQUIRE(Header[3] % 8 == 0); + REQUIRE(Header[3] + CounterCount * 8 <= Header[5]); + REQUIRE(Header[7] + GroupCount * 8 <= 0x2000); +} + +TEST_CASE("Graphics syncpoint cache retains wrap history across unallocated snapshots", "[graphics-supervisor]") { + using namespace Service::AM::GraphicsSyncpointCache; + CounterValue counter; + REQUIRE(counter.Observe(std::nullopt) == Unallocated); + REQUIRE(counter.Observe(0xfffffffeU) == 0xfffffffeULL); + REQUIRE(counter.Observe(1U) == 0x100000001ULL); + REQUIRE(counter.Observe(std::nullopt) == Unallocated); + REQUIRE(counter.Observe(3U) == 0x100000003ULL); +} + + +TEST_CASE("Native block image storage covers every padded GOB", "[graphics-supervisor]") { + using Service::AM::CalculateNativeImageRequirements; + std::array<u32, 16> preference{0, 2, 0x0437077f, 0, 7, 120, 1, 1}; + const auto requirements = CalculateNativeImageRequirements(preference, 1); + REQUIRE(requirements.has_value()); + REQUIRE(requirements->size == 8847360); + REQUIRE(requirements->alignment == 8192); + REQUIRE(requirements->size % requirements->alignment == 0); + preference[1] = 1; // The GPU descriptor uses the same block storage. + REQUIRE(CalculateNativeImageRequirements(preference, 1)->size == requirements->size); + preference[2] = 0; // A one-pixel image still occupies a complete block. + preference[5] = 1; + REQUIRE(CalculateNativeImageRequirements(preference, 1)->size == 8192); +} + +TEST_CASE("Native block image sizing rejects unsupported layouts and undersized pitches", "[graphics-supervisor]") { + using Service::AM::CalculateNativeImageRequirements; + const std::array<u32, 16> original{0, 2, 0x0437077f, 0, 7, 120, 1, 1}; + for (const auto count : {0U, 2U, 0xffffffffU}) { + REQUIRE_FALSE(CalculateNativeImageRequirements(original, count)); + } + for (const auto index : {0, 1, 3, 6, 7, 8, 15}) { + auto preference = original; + preference[index] ^= 4; + REQUIRE_FALSE(CalculateNativeImageRequirements(preference, 1)); + } + for (const auto block : {0U, 2U, 9U, 32U, 0xffffffffU}) { + auto preference = original; + preference[4] = block; + REQUIRE_FALSE(CalculateNativeImageRequirements(preference, 1)); + } + for (const auto pitch : {0U, 119U, 4097U, 0xffffffffU}) { + auto preference = original; + preference[5] = pitch; + REQUIRE_FALSE(CalculateNativeImageRequirements(preference, 1)); + } + REQUIRE_FALSE(CalculateNativeImageRequirements(std::span{original}.first(15), 1)); +} + +TEST_CASE("GPU memory registration ignores padding around its byte selectors", "[graphics-supervisor]") { + std::array<u32, 8> packet{0x75, 6, 0x1fdd6000, 0x21, 0x1fddcfff, 0x21, 0xaabbcc00, 0x101}; + const auto request = Service::AM::DecodeGpuMemoryRegistration(packet); + REQUIRE(request.has_value()); + REQUIRE(request->begin == 0x211fdd6000ULL); + REQUIRE(request->end == 0x211fddcfffULL); + REQUIRE(request->flags == 0); + REQUIRE(request->device == 1); + packet[7] = 0xffffff02; + REQUIRE(Service::AM::DecodeGpuMemoryRegistration(packet)->device == 2); + packet[0] = 0x76; + REQUIRE_FALSE(Service::AM::DecodeGpuMemoryRegistration(packet)); + REQUIRE_FALSE(Service::AM::DecodeGpuMemoryRegistration(std::span<const u32>{packet.data(), 7})); +} + +TEST_CASE("Native presentation slots ignore leftover IPC padding", "[graphics-supervisor]") { + using Service::AM::GraphicsSupervisorProtocol::DecodeBufferSlot; + REQUIRE(DecodeBufferSlot(0) == 0); + REQUIRE(DecodeBufferSlot(0x4f434601) == 1); + REQUIRE(DecodeBufferSlot(0xffffff0e) == 14); + REQUIRE(DecodeBufferSlot(0x4f43460f) == 15); +} diff --git a/src/tests/core/native_gpu_characteristics.cpp b/src/tests/core/native_gpu_characteristics.cpp new file mode 100644 index 0000000000..cb79853ac2 --- /dev/null +++ b/src/tests/core/native_gpu_characteristics.cpp @@ -0,0 +1,389 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <algorithm> +#include <array> +#include <cstring> +#include <memory> +#include "video_core/macro.h" +#include "video_core/native_qmd.h" +#include "video_core/dirty_flags.h" +#include <catch2/catch_test_macros.hpp> +#include "core/hle/kernel/k_memory_block.h" +#include "core/hle/service/nvdrv/devices/native_gpu_kind.h" +#include "core/hle/service/nvdrv/devices/native_gpu_context.h" +#include "core/hle/service/nvdrv/devices/native_gpu_characteristics.h" +#include "core/hle/service/nvdrv/devices/gpu_address_space_layout.h" + +TEST_CASE("Native GPU characteristics replace topology with ioctl interface levels", + "[native-gpu]") { + std::array<u8, 0xa0> legacy; + legacy.fill(0xcc); + const u32_le arch{0x120}; + const u32_le impl{0xb}; + std::memcpy(legacy.data(), &arch, sizeof(arch)); + std::memcpy(legacy.data() + 4, &impl, sizeof(impl)); + const u32_le available{0x30000}; + std::memcpy(legacy.data() + 0x34, &available, sizeof(available)); + const auto native = Service::Nvidia::Devices::MakeNativeGpuCharacteristics(legacy); + REQUIRE(native.size() == 0xd0); + u32_le native_arch, native_impl; + std::memcpy(&native_arch, native.data(), sizeof(native_arch)); + std::memcpy(&native_impl, native.data() + 4, sizeof(native_impl)); + REQUIRE(static_cast<u32>(native_arch) == 0x170); + REQUIRE(static_cast<u32>(native_impl) == 0xe); + REQUIRE(std::equal(legacy.begin() + 8, legacy.begin() + 0x28, native.begin() + 8)); + REQUIRE(std::equal(legacy.begin() + 0x2c, legacy.begin() + 0x34, native.begin() + 0x2c)); + REQUIRE(std::equal(legacy.begin() + 0x38, legacy.begin() + 0x3c, native.begin() + 0x38)); + REQUIRE(std::equal(legacy.begin() + 0x44, legacy.begin() + 0x50, native.begin() + 0x44)); + REQUIRE(std::equal(legacy.begin() + 0x58, legacy.begin() + 0x70, native.begin() + 0x58)); + u32_le sm_version, spa_version; + std::memcpy(&sm_version, native.data() + 0x3c, sizeof(sm_version)); + std::memcpy(&spa_version, native.data() + 0x40, sizeof(spa_version)); + REQUIRE(static_cast<u32>(sm_version) == 0x807); + REQUIRE(static_cast<u32>(spa_version) == 0x807); + u32_le available_sizes; + std::memcpy(&available_sizes, native.data() + 0x34, sizeof(available_sizes)); + REQUIRE(static_cast<u32>(available_sizes) == 0x11000); + u32_le big_page_size; + std::memcpy(&big_page_size, native.data() + 0x28, sizeof(big_page_size)); + REQUIRE(static_cast<u32>(big_page_size) == 0x1000); + const auto level = [&](size_t offset) { + u16_le value; + std::memcpy(&value, native.data() + offset, sizeof(value)); + return static_cast<s16>(static_cast<u16>(value)); + }; + REQUIRE(level(0x70) == 0x1c); + REQUIRE(level(0x72) == 2); + u64_le flags; + std::memcpy(&flags, native.data() + 0x50, sizeof(flags)); + REQUIRE((static_cast<u64>(flags) & (1ULL << 8)) != 0); + REQUIRE((static_cast<u64>(flags) & ~((1ULL << 8) | (1ULL << 26) | (1ULL << 34) | (0xfULL << 40))) == (0xccccccccccccccccULL & ~((1ULL << 8) | (1ULL << 26) | (1ULL << 34) | (0xfULL << 40)))); + REQUIRE((static_cast<u64>(flags) & (0xfULL << 40)) == (0xfULL << 40)); + REQUIRE((static_cast<u64>(flags) & (1ULL << 34)) != 0); + REQUIRE(level(0x74) == -1); + REQUIRE(level(0x76) == 0x1d); + // The captured SDK rejects the old topology value 2 here, requiring >= 6. + REQUIRE(level(0x78) == 0x14); + REQUIRE(std::all_of(native.begin() + 0x7a, native.end(), [](u8 b) { return b == 0; })); +} + +TEST_CASE("Native GPU VA layout accommodates the captured NVN bootstrap reservation", + "[native-gpu]") { + using Service::Nvidia::Devices::GetGpuAddressSpaceLayout; + constexpr auto native = GetGpuAddressSpaceLayout(true); + constexpr auto legacy = GetGpuAddressSpaceLayout(false); + REQUIRE(native.split - native.start >= 0x7ff8000000ULL + (1ULL << 32)); + REQUIRE(native.start == legacy.start); + REQUIRE(native.end == 1ULL << 40); + REQUIRE(native.end > native.split); + REQUIRE(legacy.split == 1ULL << 34); + REQUIRE(legacy.end == 1ULL << 37); +} + +TEST_CASE("GPU region addresses retain bits above the 32-bit page index", "[native-gpu]") { + using Service::Nvidia::Devices::GpuPageAddress; + REQUIRE(GpuPageAddress(0x400000, 17) == 0x8000000000ULL); + REQUIRE(GpuPageAddress(0x20000, 17) == 0x400000000ULL); + REQUIRE(GpuPageAddress(0x8000, 12) == 0x8000000ULL); +} + +TEST_CASE("GPU memory eligibility survives splitting and remains separate from cache attributes", + "[native-gpu]") { + using namespace Kernel; + KMemoryBlock right{KProcessAddress{0x10000}, 4, KMemoryState::Normal, + KMemoryPermission::UserReadWrite, KMemoryAttribute::Uncached}; + right.UpdateAttribute(KMemoryAttribute::GpuAccessible, KMemoryAttribute::GpuAccessible); + KMemoryBlock left; + right.Split(&left, KProcessAddress{0x12000}); + REQUIRE(left.GetNumPages() == 2); + REQUIRE(right.GetNumPages() == 2); + REQUIRE(left.GetMemoryInfo().GetSvcMemoryInfo().attribute == + (Svc::MemoryAttribute::Uncached | Svc::MemoryAttribute::GpuAccessible)); + REQUIRE(right.GetAttribute() == left.GetAttribute()); + right.UpdateAttribute(KMemoryAttribute::GpuAccessible, KMemoryAttribute::None); + REQUIRE(right.GetAttribute() == KMemoryAttribute::Uncached); + REQUIRE(left.GetAttribute() == + (KMemoryAttribute::Uncached | KMemoryAttribute::GpuAccessible)); + REQUIRE(right.GetPermission() == KMemoryPermission::UserReadWrite); + REQUIRE(right.GetState() == KMemoryState::Normal); +} + +TEST_CASE("Native 4 KiB allocation preference retains a separate large-page GPU pool", "[native-gpu]") { + using Service::Nvidia::Devices::GetGpuAddressSpaceBigPageSize; + REQUIRE(GetGpuAddressSpaceBigPageSize(true, 0x1000) == 0x10000); + REQUIRE(GetGpuAddressSpaceBigPageSize(true, 0x20000) == 0x20000); + REQUIRE(GetGpuAddressSpaceBigPageSize(false, 0x1000) == 0x1000); + REQUIRE(GetGpuAddressSpaceBigPageSize(false, 0) == 0); +} + +TEST_CASE("Native generic memory kind is translated rather than treated as legacy depth memory", + "[native-gpu]") { + using Service::Nvidia::Devices::TranslateNativeUncompressedKind; + REQUIRE(TranslateNativeUncompressedKind(0, 0) == static_cast<u8>(Tegra::PTEKind::PITCH)); + REQUIRE(TranslateNativeUncompressedKind(6, 6) == static_cast<u8>(Tegra::PTEKind::GENERIC_16BX2)); + REQUIRE(TranslateNativeUncompressedKind(6, 6) != static_cast<u8>(Tegra::PTEKind::Z16_MS16_2C)); + REQUIRE_FALSE(TranslateNativeUncompressedKind(8, 6)); + REQUIRE_FALSE(TranslateNativeUncompressedKind(7, 7)); + REQUIRE_FALSE(TranslateNativeUncompressedKind(0xfe, 0xfe)); +} + +TEST_CASE("Native mappings report the actual selected memory kind", "[native-gpu]") { + using Service::Nvidia::Devices::SelectNativeMemoryKind; + const auto normal = SelectNativeMemoryKind(6, 6); + REQUIRE(normal.has_value()); + REQUIRE(normal->selected == 6); + REQUIRE(normal->backend == 0xfe); + const auto fallback = SelectNativeMemoryKind(8, 6); + REQUIRE(fallback.has_value()); + REQUIRE(fallback->selected == 6); + REQUIRE(fallback->backend == normal->backend); + REQUIRE(SelectNativeMemoryKind(0, 0)->selected == 0); + const auto depth = SelectNativeMemoryKind(0xc, 3); + REQUIRE(depth.has_value()); + REQUIRE(depth->selected == 3); + REQUIRE(depth->backend == static_cast<u8>(Tegra::PTEKind::S8Z24)); + REQUIRE(SelectNativeMemoryKind(3, 3)->backend == depth->backend); + const auto depth16 = SelectNativeMemoryKind(0xb, 1); + REQUIRE(depth16.has_value()); + REQUIRE(depth16->selected == 1); + REQUIRE(depth16->backend == static_cast<u8>(Tegra::PTEKind::Z16)); + REQUIRE(SelectNativeMemoryKind(1, 1)->backend == depth16->backend); + REQUIRE_FALSE(SelectNativeMemoryKind(0xb, 6)); + REQUIRE_FALSE(SelectNativeMemoryKind(0xc, 6)); + REQUIRE_FALSE(SelectNativeMemoryKind(8, 8)); + REQUIRE_FALSE(SelectNativeMemoryKind(8, 0)); + REQUIRE_FALSE(SelectNativeMemoryKind(9, 6)); +} + +TEST_CASE("Native GPU context allocations classify explicit known engine identities", "[native-gpu]") { + using Service::Nvidia::Devices::GetNativeGpuContextSlot; + REQUIRE(GetNativeGpuContextSlot(0xc797) == 1U); + REQUIRE(GetNativeGpuContextSlot(0xc7c0) == 2U); + REQUIRE_FALSE(GetNativeGpuContextSlot(0xa143)); + REQUIRE(GetNativeGpuContextSlot(0xc7b5) == 4U); + REQUIRE(GetNativeGpuContextSlot(0xc56f) == 5U); + REQUIRE_FALSE(GetNativeGpuContextSlot(0)); + REQUIRE_FALSE(GetNativeGpuContextSlot(0xb197)); + REQUIRE_FALSE(GetNativeGpuContextSlot(0xffffc797)); + REQUIRE_FALSE(GetNativeGpuContextSlot(0xdeadbeef)); +} + +TEST_CASE("Native MME uploads use instruction addresses and bypass Maxwell code storage", "[native-gpu]") { + Tegra::MacroEngine engine{true}; + engine.native_turing = true; + engine.ClearCode(8); + engine.AddCode(8, 0x11223344); + engine.AddCode(8, 0x55667788); + engine.AddCode(8, 0x99aabbcc); + REQUIRE(engine.native_code[8] == 0); + REQUIRE(engine.native_code[24] == 0x11223344); + REQUIRE(engine.native_code[25] == 0x55667788); + REQUIRE(engine.native_code[26] == 0x99aabbcc); + REQUIRE(engine.uploaded_macro_code.empty()); + engine.ClearCode(0xffffffff); + engine.AddCode(0xffffffff, 1); + REQUIRE(engine.native_code[24] == 0x11223344); + engine.native_turing = false; + engine.AddCode(8, 7); + REQUIRE(engine.uploaded_macro_code.at(8).front() == 7); +} + +TEST_CASE("Native MME loops consume the exact parameter count, including zero iterations", "[native-gpu]") { + Tegra::TuringMacroInterpreter interpreter; + constexpr std::array<u32, 15> code{ + 0x00000003U, 0x18c00000U, 0x31c00300U, + 0x00000003U, 0x18c00000U, 0xb00c5f00U, + 0x00740003U, 0x18c00740U, 0x318c0300U, + 0x00000003U, 0x18c00000U, 0x318c0301U, + 0x00000003U, 0x18c00000U, 0x318c0300U, + }; + std::vector<u32> output; + auto emit = [&](u32 method, u32 value) { REQUIRE(method == 0xd00); output.push_back(value); return true; }; + auto read = [](u32) { return 0U; }; + const std::array<u32, 1> zero{0}; + auto result = interpreter.Execute(code, 0, zero, read, emit); + REQUIRE(result.success); + REQUIRE(result.consumed == 1); + REQUIRE(output.empty()); + const std::array<u32, 3> two{2, 99, 88}; + result = interpreter.Execute(code, 0, two, read, emit); + REQUIRE(result.success); + REQUIRE(result.consumed == 3); + REQUIRE(output == std::vector<u32>{99, 88}); + result = interpreter.Execute(code, 0, std::span<const u32>{}, read, emit); + REQUIRE_FALSE(result.success); + REQUIRE(result.error == "parameter FIFO underflow"); + result = interpreter.Execute(code, 5, zero, read, emit); + REQUIRE_FALSE(result.success); + REQUIRE(result.error == "instruction outside uploaded RAM"); +} + +TEST_CASE("Native MME pairs ALU carry with two incrementing outputs", "[native-gpu]") { + Tegra::TuringMacroInterpreter interpreter; + constexpr std::array<u32, 6> code{ + 0x20340003U, 0x18084740U, 0x3bc00301U, + 0x00000003U, 0x18c00000U, 0x318c0300U, + }; + const std::array<u32, 2> params{0xffffffffU, 1}; + std::vector<std::pair<u32, u32>> writes; + const auto result = interpreter.Execute(code, 0, params, [](u32) { return 0U; }, + [&](u32 method, u32 value) { writes.emplace_back(method, value); return true; }); + REQUIRE(result.success); + REQUIRE(result.consumed == 2); + REQUIRE(writes == std::vector<std::pair<u32, u32>>{{0xd00, 0}, {0xd01, 1}}); +} + +TEST_CASE("Native empty QMDs release only after their dependency signals", "[native-gpu]") { + Tegra::NativeQmdScheduler scheduler; + Tegra::NativeQmdScheduler::Descriptor descriptor{}; + descriptor[4] = 0x8040; // Captured QMD: two dependencies, no CTAs. + descriptor[18] = 0x00010030; + descriptor[24] = 0x100000; + descriptor[25] = 0x40880021; // One-word release, sysmembar, upper address 0x21. + descriptor[26] = 0; + u32 releases{}; + const auto read = [&](u64, auto& out) { out = descriptor; return true; }; + const auto release = [&](u64 address, u32 value) { + REQUIRE(address == 0x2100100000ULL); + REQUIRE(value == 42); + ++releases; + return true; + }; + REQUIRE(scheduler.Signal(0x2100200000ULL, 3, read, release).empty()); + REQUIRE(releases == 0); + REQUIRE(scheduler.Signal(0x2100200000ULL, 7, read, release).empty()); + REQUIRE(releases == 0); + descriptor[26] = 42; + REQUIRE(scheduler.Signal(0x2100200000ULL, 7, read, release).empty()); + REQUIRE(releases == 1); +} + +TEST_CASE("Native QMD scheduler rejects real shader work without reporting completion", "[native-gpu]") { + Tegra::NativeQmdScheduler scheduler; + Tegra::NativeQmdScheduler::Descriptor descriptor{}; + descriptor[4] = 1U << 14; + descriptor[18] = 0x30; + descriptor[12] = descriptor[13] = descriptor[14] = 1; + u32 releases{}; + const auto result = scheduler.Signal(0x10000, 3, + [&](u64, auto& out) { out = descriptor; return true; }, + [&](u64, u32) { ++releases; return true; }); + REQUIRE(result == "native shader dispatch required"); + REQUIRE(releases == 0); +} + +TEST_CASE("Native dependent QMD signals survive until the child is uploaded", "[native-gpu]") { + Tegra::NativeQmdScheduler scheduler; + Tegra::NativeQmdScheduler::Descriptor parent{}, child{}; + parent[4] = 1U << 14; + parent[18] = 0x30; + parent[15] = 0x200; + parent[16] = 9; // Enabled child decrement-dependence operation. + child[4] = 2U << 14; + child[18] = 0x30; + child[24] = 0x30000; + child[25] = 0x40880000; + child[26] = 99; + u32 releases{}; + const auto read = [&](u64 address, auto& out) { out = address == 0x10000 ? parent : child; return true; }; + const auto release = [&](u64 address, u32 value) { + REQUIRE(address == 0x30000); + REQUIRE(value == 99); + ++releases; + return true; + }; + REQUIRE(scheduler.Signal(0x10000, 3, read, release).empty()); + REQUIRE(scheduler.Signal(0x10000, 7, read, release).empty()); + REQUIRE(releases == 0); + REQUIRE(scheduler.Signal(0x20000, 3, read, release).empty()); + REQUIRE(releases == 0); + REQUIRE(scheduler.Signal(0x20000, 7, read, release).empty()); + REQUIRE(releases == 1); +} + +TEST_CASE("Native MME dual ALUs sample registers before either writeback", "[native-gpu]") { + Tegra::TuringMacroInterpreter interpreter; + constexpr std::array<u32, 12> code{ + 0x00000003U, 0x18c00000U, 0x31c00300U, + 0x00000003U, 0x40c7c001U, 0xf1c00300U, + 0x00380483U, 0x18c00000U, 0xf190fb01U, + 0x00000003U, 0x18c00000U, 0x318c0300U, + }; + const std::array<u32, 2> parameters{3, 0x1000000}; + std::vector<std::pair<u32, u32>> writes; + const auto result = interpreter.Execute(code, 0, parameters, [](u32) { return 0U; }, + [&](u32 method, u32 value) { writes.emplace_back(method, value); return true; }); + REQUIRE(result.success); + REQUIRE(result.consumed == 2); + REQUIRE(writes == std::vector<std::pair<u32, u32>>{{0x120, 7}}); +} + +TEST_CASE("Native MME taken back edges suppress fallthrough termination", "[native-gpu]") { + Tegra::TuringMacroInterpreter interpreter; + constexpr std::array<u32, 12> code{ + 0x00000003U, 0x18c00000U, 0x31c00300U, + 0x00383403U, 0x18c00000U, 0x72000b00U, + 0x00000003U, 0x18c017ffU, 0xf00c7701U, + 0x00000003U, 0x18c00000U, 0x318c0300U, + }; + for (const u32 count : {1U, 3U, 16U}) { + std::vector<u32> writes; + const std::array<u32, 1> parameters{count}; + const auto result = interpreter.Execute(code, 0, parameters, [](u32) { return 0U; }, + [&](u32 method, u32 value) { REQUIRE(method == 0xd00); writes.push_back(value); return true; }); + REQUIRE(result.success); + REQUIRE(result.consumed == 1); + REQUIRE(writes.size() == count); + REQUIRE(writes.front() == count - 1); + REQUIRE(writes.back() == 0); + } +} + +TEST_CASE("Native compute QMD releases follow successful dispatch and retain reduction flags", "[native-gpu]") { + Tegra::NativeQmdScheduler scheduler; + Tegra::NativeQmdScheduler::Descriptor descriptor{}; + descriptor[18] = 0x30; + descriptor[12] = descriptor[13] = descriptor[14] = 1; + descriptor[24] = 0x10000; + descriptor[25] = 0x44800021; + descriptor[26] = 1; + std::vector<int> order; + const auto read = [&](u64, auto& out) { out = descriptor; return true; }; + const auto release = [&](u64 address, u32 value, u32 flags) { + REQUIRE(address == 0x2100010000ULL); + REQUIRE(value == 1); + REQUIRE(flags == 0x44800021); + order.push_back(2); + return true; + }; + REQUIRE(scheduler.Signal(0x20000, 3, read, release, + [&](const auto&) { order.push_back(1); return false; }) == "native shader dispatch required"); + REQUIRE(order == std::vector<int>{1}); + order.clear(); + REQUIRE(scheduler.Signal(0x20000, 3, read, release, + [&](const auto&) { order.push_back(1); return true; }).empty()); + REQUIRE(order == std::vector<int>{1, 2}); +} + +TEST_CASE("Native vertex stream sizes use byte counts and dirty the selected stream", "[native-gpu]") { + using Maxwell3D = Tegra::Engines::Maxwell3D; + auto regs = std::make_unique<Maxwell3D::Regs>(); + regs->reg_array.fill(0); + regs->reg_array[0x600 / 4 + 1] = 0x1b4e8; + REQUIRE(regs->NativeVertexStreamSize(0) == 111848); + REQUIRE(regs->vertex_stream_limits[0].Address() == 0); + regs->reg_array[0x600 / 4 + 31 * 2] = 0xffffff81; + regs->reg_array[0x600 / 4 + 31 * 2 + 1] = 0x76543210; + REQUIRE(regs->NativeVertexStreamSize(31) == 0x8176543210ULL); + REQUIRE(regs->NativeVertexStreamSize(1) == 0); + Maxwell3D::DirtyState::Tables tables{}; + VideoCommon::Dirty::SetupDirtyFlags(tables); + for (u32 stream = 0; stream < 32; ++stream) { + for (u32 word = 0; word < 2; ++word) { + const u32 method = 0x600 / 4 + stream * 2 + word; + REQUIRE(tables[0][method] == VideoCommon::Dirty::VertexBuffer0 + stream); + REQUIRE(tables[1][method] == VideoCommon::Dirty::VertexBuffers); + } + } +} diff --git a/src/tests/core/program_metadata.cpp b/src/tests/core/program_metadata.cpp new file mode 100644 index 0000000000..b5af9953f4 --- /dev/null +++ b/src/tests/core/program_metadata.cpp @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <cstring> +#include <memory> +#include <vector> + +#include <catch2/catch_test_macros.hpp> +#include <catch2/generators/catch_generators.hpp> + +#include "common/swap.h" +#include "core/file_sys/program_metadata.h" +#include "core/file_sys/romfs.h" +#include "core/file_sys/vfs/vfs_vector.h" +#include "core/loader/deconstructed_rom_directory.h" +#include "core/loader/loader.h" + +namespace { +template <typename T> +void Put(std::vector<u8>& bytes, std::size_t offset, T value) { + std::memcpy(bytes.data() + offset, &value, sizeof(value)); +} + +std::vector<u8> MakeMetadata(bool extended) { + std::vector<u8> bytes(0x600); + std::memcpy(bytes.data(), "META", 4); + bytes[0xC] = 7; + Put(bytes, 0x1C, u32_le{0x100000}); + Put(bytes, 0x70, u32_le{0x500}); + Put(bytes, 0x74, u32_le{0x100}); + Put(bytes, 0x78, u32_le{0x80}); + Put(bytes, 0x7C, u32_le{0x380}); + const std::size_t acid = 0x80 + (extended ? 0x300 : 0x200); + const u32 fac = extended ? 0x340 : 0x240; + std::memcpy(bytes.data() + acid, "ACID", 4); + Put(bytes, acid + 0x20, u32_le{fac}); + Put(bytes, acid + 0x24, u32_le{0x2C}); + Put(bytes, 0x80 + fac + 4, u64_le{0x1234}); + std::memcpy(bytes.data() + 0x500, "ACI0", 4); + Put(bytes, 0x510, u64_le{0x0100123456780000}); + Put(bytes, 0x520, u32_le{0x40}); + Put(bytes, 0x524, u32_le{0x1C}); + Put(bytes, 0x530, u32_le{0x80}); + Put(bytes, 0x534, u32_le{4}); + Put(bytes, 0x544, u64_le{0x5678}); + Put(bytes, 0x580, u32_le{0x030043F7}); + return bytes; +} +} // namespace + +TEST_CASE("Program metadata reads both ACID layouts and relative access offsets", "[loader][npdm]") { + const bool extended = GENERATE(false, true); + FileSys::ProgramMetadata metadata; + REQUIRE(metadata.Load(std::make_shared<FileSys::VectorVfsFile>(MakeMetadata(extended))) == + Loader::ResultStatus::Success); + REQUIRE(metadata.UsesExtendedAcid() == extended); + REQUIRE(metadata.GetTitleID() == 0x0100123456780000); + REQUIRE(metadata.GetFilesystemPermissions() == 0x5678); + REQUIRE(metadata.GetMainThreadStackSize() == 0x100000); + REQUIRE(metadata.GetKernelCapabilities() == std::vector<u32>{0x030043F7}); +} + +TEST_CASE("Program metadata rejects invalid sections", "[loader][npdm]") { + const bool extended = GENERATE(false, true); + auto bytes = MakeMetadata(extended); + const std::size_t acid = 0x80 + (extended ? 0x300 : 0x200); + Loader::ResultStatus expected{}; + SECTION("NPDM magic") { + bytes[0] = 0; + expected = Loader::ResultStatus::ErrorBadNPDMHeader; + } + SECTION("ACID past EOF") { + Put(bytes, 0x78, u32_le{0xFFFFFFF0}); + expected = Loader::ResultStatus::ErrorBadACIDHeader; + } + SECTION("ACID magic") { + bytes[acid] = 0; + expected = Loader::ResultStatus::ErrorBadACIDHeader; + } + SECTION("ACI magic") { + bytes[0x500] = 0; + expected = Loader::ResultStatus::ErrorBadACIHeader; + } + SECTION("FAH outside its section") { + Put(bytes, 0x520, u32_le{0xF0}); + expected = Loader::ResultStatus::ErrorBadFileAccessHeader; + } + SECTION("FAC outside its section") { + Put(bytes, acid + 0x20, u32_le{0xFFFFFFF0}); + expected = Loader::ResultStatus::ErrorBadFileAccessControl; + } + SECTION("Truncated FAH") { + Put(bytes, 0x524, u32_le{0x10}); + expected = Loader::ResultStatus::ErrorBadFileAccessHeader; + } + SECTION("Capabilities outside their section") { + Put(bytes, 0x530, u32_le{0x100}); + expected = Loader::ResultStatus::ErrorBadKernelCapabilityDescriptors; + } + SECTION("Unaligned capability count") { + Put(bytes, 0x534, u32_le{3}); + expected = Loader::ResultStatus::ErrorBadKernelCapabilityDescriptors; + } + FileSys::ProgramMetadata metadata; + REQUIRE(metadata.Load(std::make_shared<FileSys::VectorVfsFile>(std::move(bytes))) == expected); +} + +TEST_CASE("Reloading legacy metadata clears extended ACID state", "[loader][npdm]") { + FileSys::ProgramMetadata metadata; + REQUIRE(metadata.Load(std::make_shared<FileSys::VectorVfsFile>(MakeMetadata(true))) == + Loader::ResultStatus::Success); + REQUIRE(metadata.UsesExtendedAcid()); + REQUIRE(metadata.Load(std::make_shared<FileSys::VectorVfsFile>(MakeMetadata(false))) == + Loader::ResultStatus::Success); + REQUIRE_FALSE(metadata.UsesExtendedAcid()); + REQUIRE(metadata.Load(nullptr) == Loader::ResultStatus::ErrorBadNPDMHeader); +} + +TEST_CASE("ExeFS main takes precedence over standalone NSO", "[loader][exefs]") { + auto directory = std::make_shared<FileSys::VectorVfsDirectory>(); + auto main = std::make_shared<FileSys::VectorVfsFile>(std::vector<u8>{'N', 'S', 'O', '0'}, + "main", directory); + directory->AddFile(main); + directory->AddFile(std::make_shared<FileSys::VectorVfsFile>(MakeMetadata(false), "main.npdm")); + REQUIRE(Loader::IdentifyFile(main) == Loader::FileType::DeconstructedRomDirectory); + directory->DeleteFile("main.npdm"); + REQUIRE(Loader::AppLoader_DeconstructedRomDirectory::IdentifyType(main) == Loader::FileType::Error); +} + +TEST_CASE("Extracted ExeFS exposes loose RomFS", "[loader][exefs]") { + const bool sibling = GENERATE(false, true); + auto root = std::make_shared<FileSys::VectorVfsDirectory>(); + auto exefs = std::make_shared<FileSys::VectorVfsDirectory>( + std::vector<FileSys::VirtualFile>{}, std::vector<FileSys::VirtualDir>{}, "exefs", root); + root->AddDirectory(exefs); + auto main = std::make_shared<FileSys::VectorVfsFile>(std::vector<u8>{'N', 'S', 'O', '0'}, + "main", exefs); + exefs->AddFile(main); + exefs->AddFile(std::make_shared<FileSys::VectorVfsFile>(MakeMetadata(false), "main.npdm")); + auto romfs = std::make_shared<FileSys::VectorVfsDirectory>( + std::vector<FileSys::VirtualFile>{}, std::vector<FileSys::VirtualDir>{}, "romfs"); + romfs->AddFile(std::make_shared<FileSys::VectorVfsFile>(std::vector<u8>{1, 2, 3}, "asset.bin")); + (sibling ? root : exefs)->AddDirectory(romfs); + Loader::AppLoader_DeconstructedRomDirectory loader(main); + FileSys::VirtualFile image; + REQUIRE(loader.ReadRomFS(image) == Loader::ResultStatus::Success); + const auto extracted = FileSys::ExtractRomFS(image); + REQUIRE(extracted != nullptr); + const auto asset = extracted->GetFile("asset.bin"); + REQUIRE(asset != nullptr); + REQUIRE(asset->ReadAllBytes() == std::vector<u8>{1, 2, 3}); +} diff --git a/src/tests/video_core/image_overlap.cpp b/src/tests/video_core/image_overlap.cpp new file mode 100644 index 0000000000..a4a08c72ea --- /dev/null +++ b/src/tests/video_core/image_overlap.cpp @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <catch2/catch_test_macros.hpp> +#include <catch2/generators/catch_generators.hpp> +#include "video_core/texture_cache/image_base.h" +#include "video_core/texture_cache/image_info.h" +#include "video_core/texture_cache/image_view_info.h" +#include "video_core/texture_cache/util.h" + +namespace { +VideoCommon::ImageInfo ArrayInfo(bool native) { + VideoCommon::ImageInfo info; + info.type = VideoCommon::ImageType::e2D; + info.format = VideoCore::Surface::PixelFormat::A8B8G8R8_UNORM; + info.size = {64, 64, 1}; + info.native_tiling = native; + info.layer_stride = VideoCommon::CalculateLayerStride(info); + info.maybe_unaligned_layer_stride = info.layer_stride; + return info; +} +} + +TEST_CASE("Repeated views inside an existing array do not grow its storage", "[texture-overlap]") { + const auto single = ArrayInfo(GENERATE(false, true)); + auto storage = single; + storage.resources.layers = 3; + constexpr GPUVAddr gpu = 0x100000; + constexpr VAddr cpu = 0x200000; + for (u32 frame = 0; frame < 100; ++frame) { + const VideoCommon::ImageBase existing{storage, gpu, cpu}; + const u32 offset = (1 + frame % 2) * single.layer_stride; + const auto joined = VideoCommon::ResolveOverlap(single, gpu + offset, cpu + offset, + existing, true, false, true); + REQUIRE(joined); + REQUIRE(joined->gpu_addr == gpu); + REQUIRE(joined->cpu_addr == cpu); + REQUIRE(joined->resources.layers == 3); + storage.resources = joined->resources; + REQUIRE(VideoCommon::CalculateUnswizzledSizeBytes(storage) == 64 * 64 * 4 * 3); + } +} + +TEST_CASE("Array overlap extends only to the end of the new view", "[texture-overlap]") { + auto storage = ArrayInfo(GENERATE(false, true)); + storage.resources.layers = 3; + auto view = storage; + view.resources.layers = 4; + constexpr GPUVAddr gpu = 0x100000; + constexpr VAddr cpu = 0x200000; + const VideoCommon::ImageBase existing{storage, gpu, cpu}; + const u32 offset = 2 * storage.layer_stride; + const auto joined = VideoCommon::ResolveOverlap(view, gpu + offset, cpu + offset, + existing, true, false, true); + REQUIRE(joined); + REQUIRE(joined->resources.layers == 6); +} + +TEST_CASE("Native texture views retain their direct address without a legacy array prefix", "[texture-overlap]") { + Tegra::Texture::TICEntry raw{}; + raw.format.Assign(Tegra::Texture::TextureFormat::A8B8G8R8); + raw.r_type.Assign(Tegra::Texture::ComponentType::UNORM); + raw.g_type.Assign(Tegra::Texture::ComponentType::UNORM); + raw.b_type.Assign(Tegra::Texture::ComponentType::UNORM); + raw.a_type.Assign(Tegra::Texture::ComponentType::UNORM); + raw.header_version.Assign(Tegra::Texture::TICHeaderVersion::BlockLinear); + raw.block_height.Assign(4); + raw.width_minus_one.Assign(1919); + raw.height_minus_1.Assign(1079); + const bool array = GENERATE(false, true); + raw.texture_type.Assign(array ? Tegra::Texture::TextureType::Texture2DArray : Tegra::Texture::TextureType::Texture2D); + raw.depth_minus_1.Assign(array ? 2 : 0); + raw.normalized_coords.Assign(1); + raw.address_high.Assign(0x81); + raw.address_low = 0x870000; + raw.layer_base_0_2.Assign(1); + raw.layer_base_3_7.Assign(4); + REQUIRE(raw.BaseLayer() == 33); + REQUIRE(VideoCommon::TextureViewHeader(raw, false) == raw); + const auto native = VideoCommon::TextureViewHeader(raw, true); + REQUIRE(native.BaseLayer() == 0); + REQUIRE(native.Address() == raw.Address()); + const VideoCommon::ImageInfo info(native); + REQUIRE(info.size.width == 1920); + REQUIRE(info.size.height == 1080); + REQUIRE(info.resources.layers == (array ? 3 : 1)); + REQUIRE(VideoCommon::CalculateUnswizzledSizeBytes(info) == size_t{1920} * 1080 * 4 * (array ? 3U : 1U)); + const VideoCommon::ImageViewInfo view(native, 0); + REQUIRE(view.range.base.layer == 0); + REQUIRE(view.type == (array ? VideoCommon::ImageViewType::e2DArray : VideoCommon::ImageViewType::e2D)); + REQUIRE(view.range.extent.layers == (array ? 3 : 1)); + REQUIRE(VideoCommon::ImageInfo(raw).resources.layers == (array ? 36 : 34)); +} diff --git a/src/tests/video_core/native_semaphore.cpp b/src/tests/video_core/native_semaphore.cpp new file mode 100644 index 0000000000..8dcafd91bc --- /dev/null +++ b/src/tests/video_core/native_semaphore.cpp @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <array> +#include <optional> +#include <vector> +#include <catch2/catch_test_macros.hpp> +#include "common/common_types.h" +#include "video_core/native_semaphore.h" + +TEST_CASE("Native semaphore predicates require the actual producer value", "[native-gpu]") { + const Tegra::NativeSemaphoreWait equal{0x1000, 840, 1}; + REQUIRE_FALSE(equal.Matches(0)); + REQUIRE_FALSE(equal.Matches(839)); + REQUIRE(equal.Matches(840)); + REQUIRE_FALSE(equal.Matches(841)); + const Tegra::NativeSemaphoreWait gequal{0x1000, 840, 4}; + REQUIRE_FALSE(gequal.Matches(0)); + REQUIRE_FALSE(gequal.Matches(839)); + REQUIRE(gequal.Matches(840)); + REQUIRE(gequal.Matches(841)); + const Tegra::NativeSemaphoreWait mask{0x1000, 0x24, 8}; + REQUIRE_FALSE(mask.Matches(0)); + REQUIRE_FALSE(mask.Matches(0x10)); + REQUIRE(mask.Matches(4)); + REQUIRE(mask.Matches(0x20)); + REQUIRE_FALSE(Tegra::NativeSemaphoreWait::Supported(0)); + REQUIRE_FALSE(Tegra::NativeSemaphoreWait::Supported(2)); + REQUIRE_FALSE(Tegra::NativeSemaphoreWait{0x1000, 0, 2}.Matches(0)); +} + +TEST_CASE("Native channels resume in dependency order without completing blocked work", "[native-gpu]") { + const std::array<size_t, 3> channels{0, 1, 2}; + std::array<std::optional<Tegra::NativeSemaphoreWait>, 3> waits{ + Tegra::NativeSemaphoreWait{0x1000, 1, 4}, + Tegra::NativeSemaphoreWait{0x2000, 1, 1}, + Tegra::NativeSemaphoreWait{0x3000, 1, 4}, + }; + std::array<u32, 3> words{}; + std::array<u32, 3> following_commands{}; + std::vector<size_t> order; + const auto ready = [&](size_t channel) { + return waits[channel] && waits[channel]->Matches(words[channel]); + }; + const auto dispatch = [&](size_t channel) { + REQUIRE(ready(channel)); + waits[channel].reset(); + ++following_commands[channel]; + order.push_back(channel); + if (channel) words[channel - 1] = 1; + }; + REQUIRE(Tegra::ResumeReadyNativeChannels(channels, ready, dispatch) == 0); + REQUIRE(order.empty()); + REQUIRE(following_commands == std::array<u32, 3>{0, 0, 0}); + // A real independent producer releases the last channel. Its remaining + // FIFO commands release the next channel, then the original consumer. + words[2] = 1; + REQUIRE(Tegra::ResumeReadyNativeChannels(channels, ready, dispatch) == 3); + REQUIRE(order == std::vector<size_t>{2, 1, 0}); + REQUIRE(following_commands == std::array<u32, 3>{1, 1, 1}); + REQUIRE(Tegra::ResumeReadyNativeChannels(channels, ready, dispatch) == 0); + REQUIRE(following_commands == std::array<u32, 3>{1, 1, 1}); +} diff --git a/src/tests/video_core/native_shader.cpp b/src/tests/video_core/native_shader.cpp new file mode 100644 index 0000000000..76424b7dbf --- /dev/null +++ b/src/tests/video_core/native_shader.cpp @@ -0,0 +1,2841 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <bit> +#include <algorithm> +#include <cmath> +#include <cstdlib> +#include <cstring> +#include <filesystem> +#include <fstream> +#include <limits> +#include <unordered_map> +#include <catch2/catch_test_macros.hpp> +#include <catch2/generators/catch_generators.hpp> +#include "video_core/texture_cache/format_lookup_table.h" +#include "shader_recompiler/environment.h" +#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h" +#include "shader_recompiler/frontend/ampere/program_override.h" +#include "shader_recompiler/frontend/ampere/program_capture.h" +#include "shader_recompiler/frontend/ampere/program_address_map.h" +#include "shader_recompiler/frontend/ir/basic_block.h" +#include "shader_recompiler/frontend/ir/modifiers.h" +#include "shader_recompiler/frontend/maxwell/decode.h" +#include "shader_recompiler/frontend/maxwell/instruction.h" +#include "shader_recompiler/frontend/maxwell/translate_program.h" +#include "shader_recompiler/host_translate_info.h" +#include "shader_recompiler/ir_opt/passes.h" +#include "shader_recompiler/frontend/maxwell/translate/impl/impl.h" + +namespace { +namespace IR = Shader::IR; +using Native = Shader::Ampere::Instruction; + +class EmptyEnvironment : public Shader::Environment { +public: + u64 ReadInstruction(u32) override { throw std::logic_error("unexpected instruction read"); } + u32 ReadCbufValue(u32, u32) override { throw std::logic_error("unexpected constant folding"); } + Shader::TextureType ReadTextureType(u32) override { return {}; } + Shader::TexturePixelFormat ReadTexturePixelFormat(u32) override { return {}; } + bool IsTexturePixelFormatInteger(u32) override { return false; } + u32 ReadViewportTransformState() override { return 1; } + u32 TextureBoundBuffer() const override { return 0; } + u32 LocalMemorySize() const override { return 0; } + u32 SharedMemorySize() const override { return 0; } + std::array<u32, 3> WorkgroupSize() const override { return {}; } + bool HasHLEMacroState() const override { return false; } + std::optional<Shader::ReplaceConstant> GetReplaceConstBuffer(u32, u32) override { return {}; } + void Dump(u64, u64) override {} +}; + +constexpr std::array<Native, 7> VertexTransform{{ + {{0xff007321, 0x000080ff, 0x00000400, 0x000e2200}}, // load two input components + {{0x00037a02, 0x00c00000, 0x00000f00, 0x000fe400}}, // scale X + {{0x00027a02, 0x00c00100, 0x00000f00, 0x000fe200}}, // scale Y + {{0x00007623, 0x00c00200, 0x00010003, 0x001fe200}}, // X * scale X + offset X + {{0x01017623, 0x00c00300, 0x00010002, 0x000fc400}}, // Y * scale Y + offset Y + {{0xff007322, 0x00007000, 0x000004ff, 0x000fe200}}, // store position X/Y + {{0x0000794d, 0x00000000, 0x03800000, 0x000fea00}}, +}}; + +// Evaluate the existing frontend's emitted IR, independently of native bit +// decoding, to check actual attribute/constant/register data flow. +std::array<float, 2> EvaluateVertex(IR::Block& block, std::array<float, 2> input, + std::array<float, 4> constants, s32 vertex_id = 0) { + std::unordered_map<const IR::Inst*, u32> values; + std::unordered_map<const IR::Inst*, std::array<u32, 2>> pairs; + std::array<u32, 256> regs{}; + std::array<float, 2> output{}; + const auto get = [&](const IR::Value& value) { + if (!value.IsImmediate()) return values.at(value.Inst()); + if (value.Type() == IR::Type::U1) return static_cast<u32>(value.U1()); + return value.Type() == IR::Type::F32 ? std::bit_cast<u32>(value.F32()) : value.U32(); + }; + for (const auto& inst : block.Instructions()) { + const auto arg = [&](size_t i) { return get(inst.Arg(i)); }; + switch (inst.GetOpcode()) { + case IR::Opcode::GetRegister: + values[&inst] = regs[static_cast<u32>(inst.Arg(0).Reg())]; + break; + case IR::Opcode::SetRegister: + if (inst.Arg(0).Reg() != IR::Reg::RZ) regs[static_cast<u32>(inst.Arg(0).Reg())] = arg(1); + break; + case IR::Opcode::GetAttributeU32: + REQUIRE(static_cast<u32>(inst.Arg(0).Attribute()) == 0x2fc / 4); + values[&inst] = static_cast<u32>(vertex_id); + break; + case IR::Opcode::ShiftLeftLogical32: + values[&inst] = arg(0) << arg(1); + break; + case IR::Opcode::ShiftRightArithmetic32: + values[&inst] = static_cast<u32>(static_cast<s32>(arg(0)) >> arg(1)); + break; + case IR::Opcode::SLessThan: + values[&inst] = static_cast<s32>(arg(0)) < static_cast<s32>(arg(1)); + break; + case IR::Opcode::ULessThan: + values[&inst] = arg(0) < arg(1); + break; + case IR::Opcode::SelectU32: + values[&inst] = arg(0) ? arg(1) : arg(2); + break; + case IR::Opcode::SMin32: + values[&inst] = static_cast<u32>(std::min(static_cast<s32>(arg(0)), static_cast<s32>(arg(1)))); + break; + case IR::Opcode::SMax32: + values[&inst] = static_cast<u32>(std::max(static_cast<s32>(arg(0)), static_cast<s32>(arg(1)))); + break; + case IR::Opcode::INeg32: + values[&inst] = 0U - arg(0); + break; + case IR::Opcode::IAdd32: + values[&inst] = arg(0) + arg(1); + break; + case IR::Opcode::ConvertF32S32: + values[&inst] = std::bit_cast<u32>(static_cast<float>(static_cast<s32>(arg(0)))); + break; + case IR::Opcode::GetAttribute: { + const auto attr = static_cast<u32>(inst.Arg(0).Attribute()); + if (attr == 0x2fc / 4) { + values[&inst] = static_cast<u32>(vertex_id); + break; + } + REQUIRE(attr >= 0x80 / 4); + REQUIRE(attr < 0x88 / 4); + values[&inst] = std::bit_cast<u32>(input[attr - 0x80 / 4]); + break; + } + case IR::Opcode::GetCbufU32x2: + REQUIRE(arg(0) == 3); + REQUIRE(arg(1) % 4 == 0); + REQUIRE(arg(1) + 8 <= sizeof(constants)); + pairs[&inst] = {std::bit_cast<u32>(constants[arg(1) / 4]), std::bit_cast<u32>(constants[arg(1) / 4 + 1])}; + break; + case IR::Opcode::CompositeExtractU32x2: + values[&inst] = pairs.at(inst.Arg(0).Inst()).at(arg(1)); + break; + case IR::Opcode::GetCbufF32: + case IR::Opcode::GetCbufU32: + REQUIRE(arg(0) == 3); + REQUIRE(arg(1) < 16); + values[&inst] = std::bit_cast<u32>(constants[arg(1) / 4]); + break; + case IR::Opcode::BitCastF32U32: + case IR::Opcode::BitCastU32F32: + values[&inst] = arg(0); + break; + case IR::Opcode::LogicalAnd: + values[&inst] = arg(0) && arg(1); + break; + case IR::Opcode::LogicalOr: + values[&inst] = arg(0) || arg(1); + break; + case IR::Opcode::LogicalNot: + values[&inst] = !arg(0); + break; + case IR::Opcode::FPIsNan32: + values[&inst] = std::isnan(std::bit_cast<float>(arg(0))); + break; + case IR::Opcode::FPOrdGreaterThanEqual32: + values[&inst] = std::bit_cast<float>(arg(0)) >= std::bit_cast<float>(arg(1)); + break; + case IR::Opcode::FPRoundEven32: + values[&inst] = std::bit_cast<u32>(std::nearbyint(std::bit_cast<float>(arg(0)))); + break; + case IR::Opcode::FPFloor32: + values[&inst] = std::bit_cast<u32>(std::floor(std::bit_cast<float>(arg(0)))); + break; + case IR::Opcode::FPTrunc32: + values[&inst] = std::bit_cast<u32>(std::trunc(std::bit_cast<float>(arg(0)))); + break; + case IR::Opcode::FPClamp32: + values[&inst] = std::bit_cast<u32>(std::clamp(std::bit_cast<float>(arg(0)), + std::bit_cast<float>(arg(1)), std::bit_cast<float>(arg(2)))); + break; + case IR::Opcode::ConvertU32F32: + values[&inst] = static_cast<u32>(std::bit_cast<float>(arg(0))); + break; + case IR::Opcode::ConvertF32U32: + values[&inst] = std::bit_cast<u32>(static_cast<float>(arg(0))); + break; + case IR::Opcode::BitFieldUExtract: + values[&inst] = (arg(0) >> arg(1)) & ((1U << arg(2)) - 1); + break; + case IR::Opcode::BitwiseOr32: + values[&inst] = arg(0) | arg(1); + break; + case IR::Opcode::FPOrdLessThan32: + values[&inst] = std::bit_cast<float>(arg(0)) < std::bit_cast<float>(arg(1)); + break; + case IR::Opcode::BitwiseXor32: + values[&inst] = arg(0) ^ arg(1); + break; + case IR::Opcode::BitwiseNot32: + values[&inst] = ~arg(0); + break; + case IR::Opcode::BitwiseAnd32: + values[&inst] = arg(0) & arg(1); + break; + case IR::Opcode::FPAdd32: + values[&inst] = std::bit_cast<u32>(std::bit_cast<float>(arg(0)) + std::bit_cast<float>(arg(1))); + break; + case IR::Opcode::FPMul32: + values[&inst] = std::bit_cast<u32>(std::bit_cast<float>(arg(0)) * std::bit_cast<float>(arg(1))); + break; + case IR::Opcode::FPMin32: + values[&inst] = std::bit_cast<u32>(std::fmin(std::bit_cast<float>(arg(0)), std::bit_cast<float>(arg(1)))); + break; + case IR::Opcode::FPMax32: + values[&inst] = std::bit_cast<u32>(std::fmax(std::bit_cast<float>(arg(0)), std::bit_cast<float>(arg(1)))); + break; + case IR::Opcode::SelectF32: + values[&inst] = arg(0) ? arg(1) : arg(2); + break; + case IR::Opcode::FPFma32: { + const auto control = inst.Flags<IR::FpControl>(); + REQUIRE(control.fmz_mode == IR::FmzMode::FTZ); + REQUIRE(control.rounding == IR::FpRounding::RN); + values[&inst] = std::bit_cast<u32>(std::fma(std::bit_cast<float>(arg(0)), + std::bit_cast<float>(arg(1)), std::bit_cast<float>(arg(2)))); + break; + } + case IR::Opcode::SetAttribute: { + const auto attr = static_cast<u32>(inst.Arg(0).Attribute()); + REQUIRE(attr >= 0x70 / 4); + REQUIRE(attr < 0x78 / 4); + output[attr - 0x70 / 4] = std::bit_cast<float>(arg(1)); + break; + } + default: + FAIL("Unexpected IR operation " << IR::NameOf(inst.GetOpcode())); + } + } + return output; +} +} // namespace + +TEST_CASE("Native vertex lowering preserves attribute transforms through existing shader IR", "[native-shader]") { + const auto lowered = Shader::Ampere::LowerToMaxwell(VertexTransform); + REQUIRE(lowered); + REQUIRE(lowered.source_size == VertexTransform.size() * 16); + REQUIRE(lowered.instructions.size() >= VertexTransform.size()); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const u64 word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::ALD: visitor.ALD(word); break; + case Opcode::AST: visitor.AST(word); break; + case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break; + case Opcode::FFMA_rc: visitor.FFMA_rc(word); break; + case Opcode::FFMA_reg: visitor.FFMA_reg(word); break; + case Opcode::MOV_reg: visitor.MOV_reg(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected lowered vertex instruction"); + } + } + REQUIRE(EvaluateVertex(block, {2, 4}, {3, 7, 5, 11}) == std::array<float, 2>{11, 39}); + REQUIRE(EvaluateVertex(block, {-4, 0.5f}, {-2, 4, 1, -3}) == std::array<float, 2>{9, -1}); +} + +TEST_CASE("Native lowering rejects unsupported forms without a partial executable", "[native-shader]") { + auto code = VertexTransform; + code[3].words[2] |= 1U << 12; // DNZ has no accepted lowering yet. + auto result = Shader::Ampere::LowerToMaxwell(code); + REQUIRE_FALSE(result); + REQUIRE(result.error_offset == 3 * 16); + REQUIRE(result.instructions.empty()); + code = VertexTransform; + code[1].words[2] &= ~(15U << 8); // Partial quad-lane mask. + result = Shader::Ampere::LowerToMaxwell(code); + REQUIRE_FALSE(result); + REQUIRE(result.instructions.empty()); + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell({})); + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(std::span{VertexTransform}.first(6))); +} + +TEST_CASE("Native instruction predicates survive lowering", "[native-shader]") { + auto code = VertexTransform; + code[3].words[0] = (code[3].words[0] & ~0xf000U) | 0x9000U; + const auto result = Shader::Ampere::LowerToMaxwell(code); + REQUIRE(result); + const auto pred = Shader::Maxwell::Instruction{result.instructions[result.source_instruction_offsets[3]]}.Pred(); + REQUIRE(pred.index == 1); + REQUIRE(pred.negated); +} + +// These private input fixtures are read from the investigation directory, not +// embedded or distributed with the emulator's source tree. +TEST_CASE("Captured native shader pair compiles through existing control flow and shader IR", "[.native-shader-captured]") { + const char* directory = std::getenv("EDEN_NATIVE_SHADER_FIXTURES"); + REQUIRE(directory != nullptr); + class FixtureEnvironment final : public EmptyEnvironment { + public: + FixtureEnvironment(const std::vector<u64>& code_, std::span<const u8> header, Shader::Stage stage_) + : code{code_} { + std::memcpy(&sph, header.data(), sizeof(sph)); + stage = stage_; + start_address = 0; + is_proprietary_driver = true; + } + u64 ReadInstruction(u32 address) override { return code.at(address / 8); } + u32 TranslateProgramAddress(u32 address) const override { + const auto translated = Shader::Ampere::NativeProgramAddress(code, address); + if (!translated) throw std::logic_error("invalid private native branch target " + std::to_string(address)); + return *translated; + } + u32 LocalMemorySize() const override { return static_cast<u32>(sph.LocalMemorySize()) + sph.common3.shader_local_memory_crs_size; } + u32 TextureBoundBuffer() const override { return 2; } + u32 ReadCbufValue(u32 bank, u32 offset) override { + if (bank == 1) { + REQUIRE(offset % 4 == 0); + return cbuf1.at(offset / 4); + } + REQUIRE(bank == 2); + return 0; // Synthetic texture handle; only compilation is tested here. + } + Shader::TextureType ReadTextureType(u32) override { return Shader::TextureType::Color2D; } + const std::vector<u64>& code; + std::vector<u32> cbuf1; + }; + for (const int stage : {1, 5}) { + const auto path = std::filesystem::path{directory} / ("native-graphics-shader-" + std::to_string(stage) + ".bin"); + std::ifstream file(path, std::ios::binary | std::ios::ate); + REQUIRE(file.good()); + const auto size = static_cast<size_t>(file.tellg()); + REQUIRE(size >= 0x90); + std::vector<u8> bytes(size); + file.seekg(0); + file.read(reinterpret_cast<char*>(bytes.data()), bytes.size()); + std::vector<Native> input((size - 0x80) / 16); + std::memcpy(input.data(), bytes.data() + 0x80, input.size() * 16); + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + INFO("stage=" << stage << " offset=" << lowered.error_offset << " error=" << lowered.error); + REQUIRE(lowered); + Shader::Ampere::ProgramCaptureState capture; + u32 captured_size{}; + for (u32 i = 0; i < input.size(); ++i) { + const auto status = capture.Observe(input[i], i); + REQUIRE(status != Shader::Ampere::ProgramCaptureState::Result::InvalidTarget); + if (status == Shader::Ampere::ProgramCaptureState::Result::Complete) { + captured_size = (i + 1) * sizeof(Native); + break; + } + } + REQUIRE(captured_size == lowered.source_size); + std::vector<u64> code(0x50 / 8); + for (const u64 word : lowered.instructions) { + if ((code.size() * 8) % 32 == 0) code.push_back(0); + code.push_back(word); + } + code.push_back(0); + Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets); + FixtureEnvironment env{code, std::span{bytes}.first(0x50), stage == 1 ? Shader::Stage::VertexB : Shader::Stage::Fragment}; + auto constants_path = std::filesystem::path{directory} / ("cbuf1-" + std::to_string(stage) + ".bin"); + if (!std::filesystem::exists(constants_path)) constants_path = std::filesystem::path{directory} / "cbuf1.bin"; + std::ifstream constants(constants_path, std::ios::binary | std::ios::ate); + if (constants) { + env.cbuf1.resize(static_cast<size_t>(constants.tellg()) / 4); + constants.seekg(0); + constants.read(reinterpret_cast<char*>(env.cbuf1.data()), env.cbuf1.size() * 4); + } + Shader::ObjectPool<Shader::Maxwell::Flow::Block> flow_pool; + Shader::ObjectPool<IR::Inst> inst_pool; + Shader::ObjectPool<IR::Block> block_pool; + Shader::Maxwell::Flow::CFG cfg{env, flow_pool, 0x50}; + Shader::HostTranslateInfo host; + host.support_int64 = true; + host.support_float64 = true; + host.support_float16 = true; + const auto program = Shader::Maxwell::TranslateProgram(inst_pool, block_pool, env, cfg, host); + REQUIRE(program.stage == env.ShaderStage()); + REQUIRE_FALSE(program.blocks.empty()); + REQUIRE_FALSE(program.post_order_blocks.empty()); + } +} + +TEST_CASE("Native scalar texture sampling preserves coordinates and split outputs", "[native-shader]") { + std::array<Native, 2> input{{ + {{0x04007b60, 0x30800805, 0x041e0f02, 0x100fc200}}, + VertexTransform.back(), + }}; + const bool predicated = GENERATE(false, true); + if (predicated) input[0].words[0] = (input[0].words[0] & ~0xf000U) | 0x3000; + const bool explicit_lod = GENERATE(false, true); + if (explicit_lod) input[0].words[2] |= 1U << 23; + const auto result = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(result); + REQUIRE(result.required_texture_buffer == 2); + REQUIRE(Shader::Maxwell::Instruction{result.instructions.front()}.Pred().index == (predicated ? 3U : 7U)); + REQUIRE_FALSE(Shader::Maxwell::Instruction{result.instructions.front()}.Pred().negated); + REQUIRE(Shader::Maxwell::Decode(result.instructions.front()) == Shader::Maxwell::Opcode::TEXS); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.TEXS(result.instructions.front()); + std::vector<u32> reads; + std::vector<u32> writes; + u32 samples = 0; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::GetRegister) reads.push_back(static_cast<u32>(inst.Arg(0).Reg())); + if (inst.GetOpcode() == IR::Opcode::SetRegister) writes.push_back(static_cast<u32>(inst.Arg(0).Reg())); + if (inst.GetOpcode() == (explicit_lod ? IR::Opcode::BoundImageSampleExplicitLod : IR::Opcode::BoundImageSampleImplicitLod)) { + ++samples; + if (explicit_lod) REQUIRE(inst.Arg(2).F32() == 0.0f); + const auto* coords = inst.Arg(1).Inst(); + REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructF32x2); + for (size_t component = 0; component < 2; ++component) { + const auto* cast = coords->Arg(component).Inst(); + REQUIRE(cast->GetOpcode() == IR::Opcode::BitCastF32U32); + const auto* reg = cast->Arg(0).Inst(); + REQUIRE(reg->GetOpcode() == IR::Opcode::GetRegister); + REQUIRE(static_cast<u32>(reg->Arg(0).Reg()) == 4 + component); + } + REQUIRE(inst.Arg(0).U32() == 0x20); + REQUIRE(inst.Flags<IR::TextureInstInfo>().type == Shader::TextureType::Color2D); + } + } + REQUIRE(samples == 1); + std::ranges::sort(reads); + REQUIRE(reads == std::vector<u32>{4, 5}); + REQUIRE(writes == std::vector<u32>{0, 1, 2, 3}); + // Reject unsupported LOD modes, a different cbuf bank, and sparse results. + for (u32 bit : {89U, 54U, 81U}) { + auto unsupported = input; + unsupported[0].words[bit / 32] ^= 1U << (bit % 32); + const auto rejected = Shader::Ampere::LowerToMaxwell(unsupported); + REQUIRE_FALSE(rejected); + REQUIRE(rejected.instructions.empty()); + } +} + +TEST_CASE("Native texel fetch keeps integer coordinates and RGB outputs", "[native-shader]") { + std::array<Native, 3> input{{ + {{0x00027305, 0x00000004, 0x0021f100, 0x001ff000}}, + {{0x02007b66, 0x30800a05, 0x049e0702, 0x101fc200}}, + VertexTransform.back(), + }}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + REQUIRE(lowered.required_texture_buffer == 2); + REQUIRE(Shader::Maxwell::Decode(lowered.instructions[0]) == Shader::Maxwell::Opcode::F2I_reg); + REQUIRE(Shader::Maxwell::Decode(lowered.instructions[1]) == Shader::Maxwell::Opcode::TLDS); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.F2I_reg(lowered.instructions[0]); + visitor.TLDS(lowered.instructions[1]); + std::vector<u32> writes; + u32 fetches = 0; + bool truncates = false; + bool handles_nan = false; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::SetRegister) writes.push_back(static_cast<u32>(inst.Arg(0).Reg())); + if (inst.GetOpcode() == IR::Opcode::FPTrunc32) { + truncates = true; + REQUIRE(inst.Flags<IR::FpControl>().fmz_mode == IR::FmzMode::FTZ); + } + if (inst.GetOpcode() == IR::Opcode::FPIsNan32) handles_nan = true; + if (inst.GetOpcode() == IR::Opcode::BoundImageFetch) { + ++fetches; + REQUIRE(inst.Arg(0).U32() == 0x28); + REQUIRE(inst.Flags<IR::TextureInstInfo>().type == Shader::TextureType::Color2D); + const auto* coords = inst.Arg(1).Inst(); + REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructU32x2); + REQUIRE(coords->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{2}); + REQUIRE(coords->Arg(1).Inst()->Arg(0).Reg() == IR::Reg{5}); + REQUIRE(inst.Arg(3).U32() == 0); + } + } + REQUIRE(fetches == 1); + REQUIRE(truncates); + REQUIRE(handles_nan); + REQUIRE(writes == std::vector<u32>{2, 0, 1, 2}); + auto unsupported = input; + unsupported[0].words[2] ^= 1U << 13; // NTZ is required by this conversion lowering. + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(unsupported)); + unsupported = input; + unsupported[1].words[2] ^= 1U << 14; // multisample fetch is not implemented here. + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(unsupported)); +} + +TEST_CASE("Native texture component masks preserve sampled components", "[native-shader]") { + for (const u32 mask : {1U, 2U, 4U, 8U, 3U, 9U, 10U, 12U, 7U, 11U, 13U, 14U, 15U}) { + const bool split = std::popcount(mask) > 2; + std::array<Native, 2> input{{ + {{0x04007b60, 0x30800805, 0x041e0f02, 0x100fc200}}, VertexTransform.back(), + }}; + input[0].words[2] = (input[0].words[2] & ~0xfffU) | (split ? 6U : 255U) | (mask << 8); + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.TEXS(lowered.instructions.front()); + std::vector<u32> extracted; + std::vector<u32> writes; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::CompositeExtractF32x4) extracted.push_back(inst.Arg(1).U32()); + if (inst.GetOpcode() == IR::Opcode::SetRegister) writes.push_back(static_cast<u32>(inst.Arg(0).Reg())); + } + std::vector<u32> expected; + for (u32 component = 0; component < 4; ++component) if (mask & (1U << component)) expected.push_back(component); + REQUIRE(extracted == expected); + REQUIRE(writes.size() == expected.size()); + for (u32 i = 0; i < writes.size(); ++i) REQUIRE(writes[i] == (i < 2 ? i : i + 4)); + } +} + +TEST_CASE("Native half packing and arithmetic preserve lane order numerically", "[native-shader]") { + const std::array<Native, 15> input{{ + {{0x0102723e, 0x00000000, 0x000000ff, 0}}, // PACK_AB R2, R1, R0 + {{0x02027232, 0x00000002, 0x00000000, 0}}, // HMUL2 R2, R2, R2 + {{0xff037230, 0x20000002, 0x00004100, 0}}, // extract low half to F32 R3 + {{0xff047230, 0x30000002, 0x00004100, 0}}, // extract high half to F32 R4 + {{0xff05723e, 0x00000000, 0x00004002, 0}}, // MERGE_C R5, R0, R2 + {{0x02067230, 0x20000002, 0x00000800, 0}}, // add two low halves: (2,2) + {{0x02077231, 0x20000002, 0x00040802, 0}}, // fused 1*1+1: (2,2) + {{0x02087240, 0x30000002, 0x03800800, 0}}, // min of low/high: (1,1) + {{0x02097840, 0x3c003c00, 0x07800c00, 0}}, // max of high/one: (4,4) + {{0x000a7308, 0x00000008, 0x00001200, 0}}, // reciprocal of half one + {{0x020b7816, 0x00005410, 0x00000007, 0}}, // concatenate low halves + {{0xff0c7230, 0xa0000008, 0x00004100, 0}}, // extract negated low half + {{0x020d7230, 0x30000002, 0x00000f00, 0}}, // -abs(high) + high: (0,0) + {{0x020e7431, 0x3c003c00, 0x00400802, 0}}, // (1,1) * (1,-4) + (1,1) + VertexTransform.back(), + }}; + auto code = input; + const u32 selector = GENERATE(0x5410U, 0x5432U, 0x7610U, 0x7632U); + code[10].words[1] = selector; + if (selector == 0x7610 || selector == 0x7632) code[10].words[2] = 2; + const auto lowered = Shader::Ampere::LowerToMaxwell(code); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const u64 word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::F2F_reg: visitor.F2F_reg(word); break; + case Opcode::IADD_reg: visitor.IADD_reg(word); break; + case Opcode::ISCADD_reg: visitor.ISCADD_reg(word); break; + case Opcode::HMUL2_reg: visitor.HMUL2_reg(word); break; + case Opcode::HADD2_reg: visitor.HADD2_reg(word); break; + case Opcode::HFMA2_reg: visitor.HFMA2_reg(word); break; + case Opcode::FMNMX_reg: visitor.FMNMX_reg(word); break; + case Opcode::MUFU: visitor.MUFU(word); break; + case Opcode::MOV32I: visitor.MOV32I(word); break; + case Opcode::SHR_imm: visitor.SHR_imm(word); break; + case Opcode::LOP32I: visitor.LOP32I(word); break; + case Opcode::MOV_reg: visitor.MOV_reg(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected canonical half instruction"); + } + } + // Independent values with exact binary16 representations exercise packing, + // multiplication and extraction through the actual emitted IR data flow. + const std::array<std::pair<u16, float>, 5> halves{{{u16{0x4200}, 3.0f}, {u16{0}, 0.0f}, {u16{0x3c00}, 1.0f}, {u16{0x4000}, 2.0f}, {u16{0x4400}, 4.0f}}}; + const auto from_half = [&](u16 bits) { + for (const auto& [encoded, value] : halves) if (encoded == (bits & 0x7fff)) return (bits & 0x8000) ? -value : value; + throw std::logic_error("unexpected half value"); + }; + const auto to_half = [&](float value) { + for (const auto& [encoded, reference] : halves) if (std::abs(value) == reference) return static_cast<u16>(encoded | (std::signbit(value) ? 0x8000 : 0)); + throw std::logic_error("unexpected float value"); + }; + std::array<u32, 256> regs{}; + regs[0] = std::bit_cast<u32>(1.0f); + regs[1] = std::bit_cast<u32>(2.0f); + std::unordered_map<const IR::Inst*, u64> values; + const auto get = [&](const IR::Value& value) -> u64 { + if (!value.IsImmediate()) return values.at(value.Inst()); + if (value.Type() == IR::Type::U1) return value.U1(); + if (value.Type() == IR::Type::F32) return std::bit_cast<u32>(value.F32()); + if (value.Type() == IR::Type::U16 || value.Type() == IR::Type::F16) return value.U16(); + return value.U32(); + }; + for (const auto& inst : block.Instructions()) { + const auto arg = [&](size_t i) { return get(inst.Arg(i)); }; + switch (inst.GetOpcode()) { + case IR::Opcode::GetRegister: values[&inst] = regs[static_cast<u32>(inst.Arg(0).Reg())]; break; + case IR::Opcode::SetRegister: regs[static_cast<u32>(inst.Arg(0).Reg())] = static_cast<u32>(arg(1)); break; + case IR::Opcode::BitCastF32U32: + case IR::Opcode::BitCastU32F32: values[&inst] = arg(0); break; + case IR::Opcode::ConvertF16F32: values[&inst] = to_half(std::bit_cast<float>(static_cast<u32>(arg(0)))); break; + case IR::Opcode::ConvertF32F16: values[&inst] = std::bit_cast<u32>(from_half(static_cast<u16>(arg(0)))); break; + case IR::Opcode::CompositeConstructF16x2: values[&inst] = arg(0) | (arg(1) << 32); break; + case IR::Opcode::CompositeExtractF16x2: values[&inst] = (arg(0) >> (arg(1) * 32)) & 0xffff; break; + case IR::Opcode::PackFloat2x16: values[&inst] = (arg(0) & 0xffff) | ((arg(0) >> 32) << 16); break; + case IR::Opcode::UnpackFloat2x16: values[&inst] = (arg(0) & 0xffff) | ((arg(0) >> 16) << 32); break; + case IR::Opcode::FPMul16: values[&inst] = to_half(from_half(static_cast<u16>(arg(0))) * from_half(static_cast<u16>(arg(1)))); break; + case IR::Opcode::FPAdd16: values[&inst] = to_half(from_half(static_cast<u16>(arg(0))) + from_half(static_cast<u16>(arg(1)))); break; + case IR::Opcode::FPFma16: values[&inst] = to_half(std::fma(from_half(static_cast<u16>(arg(0))), from_half(static_cast<u16>(arg(1))), from_half(static_cast<u16>(arg(2))))); break; + case IR::Opcode::FPNeg16: values[&inst] = arg(0) ^ 0x8000; break; + case IR::Opcode::FPAbs16: values[&inst] = arg(0) & 0x7fff; break; + case IR::Opcode::FPAdd32: values[&inst] = std::bit_cast<u32>(std::bit_cast<float>(static_cast<u32>(arg(0))) + std::bit_cast<float>(static_cast<u32>(arg(1)))); break; + case IR::Opcode::FPMin32: values[&inst] = std::bit_cast<u32>(std::fmin(std::bit_cast<float>(static_cast<u32>(arg(0))), std::bit_cast<float>(static_cast<u32>(arg(1))))); break; + case IR::Opcode::FPMax32: values[&inst] = std::bit_cast<u32>(std::fmax(std::bit_cast<float>(static_cast<u32>(arg(0))), std::bit_cast<float>(static_cast<u32>(arg(1))))); break; + case IR::Opcode::FPRecip32: values[&inst] = std::bit_cast<u32>(1.0f / std::bit_cast<float>(static_cast<u32>(arg(0)))); break; + case IR::Opcode::SelectF32: values[&inst] = arg(0) ? arg(1) : arg(2); break; + case IR::Opcode::BitwiseAnd32: values[&inst] = arg(0) & arg(1); break; + case IR::Opcode::BitwiseXor32: values[&inst] = arg(0) ^ arg(1); break; + + case IR::Opcode::ShiftRightLogical32: values[&inst] = static_cast<u32>(arg(0)) >> arg(1); break; + case IR::Opcode::SLessThan: values[&inst] = static_cast<s32>(arg(0)) < static_cast<s32>(arg(1)); break; + case IR::Opcode::ULessThan: values[&inst] = static_cast<u32>(arg(0)) < static_cast<u32>(arg(1)); break; + case IR::Opcode::SelectU32: values[&inst] = arg(0) ? arg(1) : arg(2); break; + case IR::Opcode::IAdd32: values[&inst] = static_cast<u32>(arg(0) + arg(1)); break; + case IR::Opcode::ShiftLeftLogical32: values[&inst] = static_cast<u32>(arg(0) << arg(1)); break; + default: FAIL("Unexpected IR operation in half arithmetic"); + } + } + REQUIRE(regs[2] == 0x44003c00); + REQUIRE(regs[5] == 0x3c003c00); + REQUIRE(regs[6] == 0x40004000U); + REQUIRE(regs[7] == 0x40004000U); + REQUIRE(regs[8] == 0x3c003c00U); + REQUIRE(regs[9] == 0x44004400U); + REQUIRE(regs[10] == 0x3c00U); + REQUIRE(regs[11] == (selector == 0x7632 ? 0x44004400U : selector == 0x7610 ? 0x44003c00U : selector == 0x5432 ? 0x40004400U : 0x40003c00U)); + REQUIRE(std::bit_cast<float>(regs[12]) == -1.0f); + REQUIRE(regs[13] == 0U); + REQUIRE(regs[14] == 0xc2004000U); + + REQUIRE(std::bit_cast<float>(regs[3]) == 1.0f); + REQUIRE(std::bit_cast<float>(regs[4]) == 4.0f); +} + +TEST_CASE("Live shader overrides require the exact original instructions", "[native-shader]") { + const auto program = Shader::Ampere::LowerToMaxwell(VertexTransform); + REQUIRE(program); + const auto bytes = Shader::Ampere::EncodeProgramOverride(VertexTransform, program); + REQUIRE_FALSE(bytes.empty()); + REQUIRE(Shader::Ampere::ProgramOverrideSourceSize(bytes) == VertexTransform.size() * 16); + REQUIRE(Shader::Ampere::ProgramOverrideSourceSize(std::span{bytes}.first(12)) == 0); + auto invalid_source = bytes; + invalid_source[8] ^= 1; + REQUIRE(Shader::Ampere::ProgramOverrideSourceSize(invalid_source) == 0); + const auto decoded = Shader::Ampere::DecodeProgramOverride(bytes, VertexTransform); + REQUIRE(decoded); + REQUIRE(decoded.instructions == program.instructions); + REQUIRE(decoded.source_instruction_offsets == program.source_instruction_offsets); + auto invalid_map = bytes; + const u32 invalid_target = static_cast<u32>(program.instructions.size()); + std::memcpy(invalid_map.data() + invalid_map.size() - program.source_instruction_offsets.size() * 4, + &invalid_target, sizeof(invalid_target)); + const auto invalid_mapping = Shader::Ampere::DecodeProgramOverride(invalid_map, VertexTransform); + REQUIRE_FALSE(invalid_mapping); + REQUIRE(invalid_mapping.instructions.empty()); + + REQUIRE(decoded.source_size == program.source_size); + REQUIRE(decoded.required_texture_buffer == program.required_texture_buffer); + auto different = VertexTransform; + different[0].words[1] ^= 4; + REQUIRE(Shader::Ampere::NativeProgramKey(different) != Shader::Ampere::NativeProgramKey(VertexTransform)); + const auto mismatch = Shader::Ampere::DecodeProgramOverride(bytes, different); + REQUIRE_FALSE(mismatch); + REQUIRE(mismatch.instructions.empty()); + REQUIRE_FALSE(Shader::Ampere::DecodeProgramOverride(std::span{bytes}.first(12), VertexTransform)); + REQUIRE_FALSE(Shader::Ampere::DecodeProgramOverride(std::span{bytes}.first(bytes.size() - 1), VertexTransform)); + auto wrong_version = bytes; + wrong_version[4] ^= 1; + REQUIRE_FALSE(Shader::Ampere::DecodeProgramOverride(wrong_version, VertexTransform)); + auto wrong_binding = bytes; + const u32 bank = 18; + std::memcpy(wrong_binding.data() + 16, &bank, sizeof(bank)); + REQUIRE_FALSE(Shader::Ampere::DecodeProgramOverride(wrong_binding, VertexTransform)); +} + +TEST_CASE("Native integer shader forms preserve signed arithmetic and immediate values", "[native-shader]") { + // Synthetic vertex program: output (id + arithmetic_shift(id, 1), 4.0). + const std::array<Native, 7> input{{ + {{0xff007321, 0x0002fcff, 0x00000000, 0x000e2200}}, + {{0xff017819, 0x00000001, 0x00011400, 0x000fe400}}, + {{0x00007824, 0x00000001, 0x078e0201, 0x000fe200}}, + {{0x00007245, 0x00000000, 0x00201400, 0x000fc600}}, + {{0xff017424, 0x40800000, 0x078e00ff, 0x000fe200}}, + {{0xff007322, 0x00007000, 0x000004ff, 0x000fe200}}, + VertexTransform.back(), + }}; + auto program = input; + const u32 mode = GENERATE(0U, 1U, 2U, 3U, 4U, 5U, 6U, 7U, 8U); + if (mode == 7) program[2] = Native{{0x00007217, 1, 0x03800200, 0}}; + if (mode == 8) program[2] = Native{{0x00007217, 1, 0x07800200, 0}}; + if (mode == 5) program[2] = Native{{0x00007210, 0x00000001, 0x07ffe1ff, 0}}; + if (mode == 6) program[2] = Native{{0x00007817, 0x0000003f, 0x07800200, 0}}; + if (mode == 4) program[2] = Native{{0x00007817, 0x0000003f, 0x03800200, 0}}; + if (mode == 1) program[2] = Native{{0x00007210, 0x00000001, 0x07ffe0ff, 0x000fe200}}; + if (mode == 3) program[2] = Native{{0x00007210, 0x80000001, 0x07ffe0ff, 0x000fe200}}; + if (mode == 2) program[1] = Native{{0x00017819, 0x00000001, 0x000006ff, 0x000fe200}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(program); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const u64 word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::ALD: visitor.ALD(word); break; + case Opcode::AST: visitor.AST(word); break; + case Opcode::MOV32I: visitor.MOV32I(word); break; + case Opcode::SHR_imm: visitor.SHR_imm(word); break; + case Opcode::SHL_imm: visitor.SHL_imm(word); break; + case Opcode::IMNMX_reg: visitor.IMNMX_reg(word); break; + case Opcode::IMNMX_imm: visitor.IMNMX_imm(word); break; + case Opcode::IADD_reg: visitor.IADD_reg(word); break; + case Opcode::I2F_reg: visitor.I2F_reg(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected lowered integer shader instruction"); + } + } + for (const s32 id : {0, 1, 2, -1, -3, 12345}) { + REQUIRE(EvaluateVertex(block, {}, {}, id) == + std::array<float, 2>{static_cast<float>(mode == 2 ? id * 3 : mode == 3 ? id - (id >> 1) : mode == 4 ? std::min<s32>(id, 63) : mode == 5 ? -id + (id >> 1) : mode == 6 ? std::max<s32>(id, 63) : mode == 7 ? std::min<s32>(id, id >> 1) : mode == 8 ? std::max<s32>(id, id >> 1) : id + (id >> 1)), 4.0f}); + } +} + +TEST_CASE("Native single-channel texel fetch writes only its selected result", "[native-shader]") { + std::array<Native, 2> input{{ + {{0x04017b66, 0x30800805, 0x049e01ff, 0x001fc200}}, + VertexTransform.back(), + }}; + const u32 component = GENERATE(0U, 1U, 2U, 3U); + input[0].words[2] = (input[0].words[2] & ~0xf00U) | (1U << (8 + component)); + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + REQUIRE(lowered.required_texture_buffer == 2); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.TLDS(lowered.instructions.front()); + std::vector<u32> writes; + u32 fetches = 0; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::SetRegister) + writes.push_back(static_cast<u32>(inst.Arg(0).Reg())); + if (inst.GetOpcode() == IR::Opcode::BoundImageFetch) { + ++fetches; + REQUIRE(inst.Arg(0).U32() == 0x20); + const auto* coords = inst.Arg(1).Inst(); + REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructU32x2); + REQUIRE(coords->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{4}); + REQUIRE(coords->Arg(1).Inst()->Arg(0).Reg() == IR::Reg{5}); + } + if (inst.GetOpcode() == IR::Opcode::CompositeExtractF32x4) + REQUIRE(inst.Arg(1).U32() == component); + } + REQUIRE(fetches == 1); + REQUIRE(writes == std::vector<u32>{1}); +} + +TEST_CASE("Native IMAD move loads the actual constant-buffer value", "[native-shader]") { + const std::array<Native, 3> input{{ + {{0xff007624, 0x00c00100, 0x078e00ff, 0x000fe200}}, + {{0xff007322, 0x00007000, 0x000000ff, 0x000fe200}}, + VertexTransform.back(), + }}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + REQUIRE(Shader::Maxwell::Decode(lowered.instructions.front()) == Shader::Maxwell::Opcode::MOV_cbuf); + visitor.MOV_cbuf(lowered.instructions[0]); + visitor.AST(lowered.instructions[1]); + REQUIRE(EvaluateVertex(block, {}, {13, -2.5f, 99, 7}) == std::array<float, 2>{-2.5f, 0.0f}); + auto unsupported = input; + unsupported[0].words[1] |= 1U << 30; + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(unsupported)); +} + +TEST_CASE("Depth-stencil descriptors ignore the unused blue and alpha component types", "[native-gpu]") { + using Tegra::Texture::ComponentType; + using Tegra::Texture::TextureFormat; + using VideoCore::Surface::PixelFormat; + for (const auto unused : {ComponentType::UNORM, ComponentType::UINT}) { + REQUIRE(VideoCommon::PixelFormatFromTextureInfo(TextureFormat::S8Z24, + ComponentType::UNORM, ComponentType::UINT, unused, unused, false) == + PixelFormat::D24_UNORM_S8_UINT); + } +} + +TEST_CASE("Native floating min/max clamps actual shader input", "[native-shader]") { + const std::array<Native, 5> input{{ + {{0xff007321, 0x000080ff, 0x00000000, 0x000e2200}}, + {{0x00007a09, 0x00c00000, 0x07810000, 0x000fe400}}, + {{0x00007a09, 0x00c00100, 0x03810000, 0x000fe400}}, + {{0xff007322, 0x00007000, 0x000000ff, 0x000fe200}}, + VertexTransform.back(), + }}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const auto word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::ALD: visitor.ALD(word); break; + case Opcode::AST: visitor.AST(word); break; + case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break; + case Opcode::FMNMX_reg: visitor.FMNMX_reg(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected native min/max lowering"); + } + } + for (const float value : {-20.0f, -1.0f, 0.25f, 2.0f, 100.0f}) + REQUIRE(EvaluateVertex(block, {value, 0}, {-1, 2, 0, 0}) == + std::array<float, 2>{std::clamp(value, -1.0f, 2.0f), 0}); +} + +TEST_CASE("Native boolean float comparison preserves register moves and absolute inputs", "[native-shader]") { + const std::array<Native, 5> input{{ + {{0xff007321, 0x000080ff, 0x00000000, 0}}, + {{0xff017224, 0x000000ff, 0x078e0000, 0}}, + {{0x01007a0a, 0x00c00000, 0x03811200, 0}}, + {{0xff007322, 0x00007000, 0x000000ff, 0}}, + VertexTransform.back(), + }}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const auto word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::ALD: visitor.ALD(word); break; + case Opcode::AST: visitor.AST(word); break; + case Opcode::MOV_reg: visitor.MOV_reg(word); break; + case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break; + case Opcode::LOP32I: visitor.LOP32I(word); break; + case Opcode::FSET_reg: visitor.FSET_reg(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected native boolean float comparison lowering"); + } + } + for (const float value : {-3.0f, -1.0f, 0.0f, 2.0f, 4.0f}) + REQUIRE(EvaluateVertex(block, {value, 0}, {2, 0, 0, 0}) == + std::array<float, 2>{std::abs(value) < 2.0f ? 1.0f : 0.0f, 0}); +} + +TEST_CASE("Native unsigned conversion saturates, clears NaNs, and extracts the low byte", "[native-shader]") { + std::array<Native, 5> input{{ + {{0xff007321, 0x000080ff, 0x00000000, 0}}, + {{0x00017305, 0x00000000, 0x0021f000, 0}}, + {{0x00007306, 0x00000001, 0x00001000, 0}}, + {{0xff007322, 0x00007000, 0x000000ff, 0}}, + VertexTransform.back(), + }}; + const bool constant_source = GENERATE(false, true); + if (constant_source) input[1] = Native{{0x00017b05, 0x00c00000, 0x0021f000, 0}}; + const u32 conversion_mode = GENERATE(0U, 1U, 2U); + const bool short_round = conversion_mode == 1; + if (conversion_mode == 2) input[1].words[2] = 0x00207000; + if (short_round) input[1].words[2] = 0x00212800; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const auto word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::ALD: visitor.ALD(word); break; + case Opcode::AST: visitor.AST(word); break; + case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break; + case Opcode::MOV32I: visitor.MOV32I(word); break; + case Opcode::FSET_reg: visitor.FSET_reg(word); break; + case Opcode::FMNMX_reg: visitor.FMNMX_reg(word); break; + case Opcode::LOP_reg: visitor.LOP_reg(word); break; + case Opcode::F2I_reg: visitor.F2I_reg(word); break; + case Opcode::I2F_reg: visitor.I2F_reg(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected unsigned conversion lowering"); + } + } + const std::array<std::pair<float, float>, 9> cases{{ + {std::numeric_limits<float>::quiet_NaN(), 0.0f}, + {-std::numeric_limits<float>::infinity(), 0.0f}, {-3.0f, 0.0f}, {1.9f, 1.0f}, + {300.75f, 44.0f}, {65535.0f, 255.0f}, {4294967040.0f, 0.0f}, + {4294967296.0f, 255.0f}, {std::numeric_limits<float>::infinity(), 255.0f}, + }}; + for (const auto& [value, expected] : cases) { + const float rounded = std::isnan(value) ? 0.0f : static_cast<float>(static_cast<u32>( + std::clamp(std::nearbyint(static_cast<double>(value)), 0.0, 65535.0)) & 255); + REQUIRE(EvaluateVertex(block, {value, 0}, {value, 0, 0, 0}) == std::array<float, 2>{short_round ? rounded : expected, 0}); + } +} + +TEST_CASE("Native depth-comparison sample preserves XY coordinates and reference depth", "[native-shader]") { + const std::array<Native, 2> input{{ + {{0x04017b60, 0x30800806, 0x041e41ff, 0}}, + VertexTransform.back(), + }}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.TEXS(lowered.instructions.front()); + u32 samples = 0; + std::vector<u32> writes; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::SetRegister) + writes.push_back(static_cast<u32>(inst.Arg(0).Reg())); + if (inst.GetOpcode() == IR::Opcode::BoundImageSampleDrefImplicitLod) { + ++samples; + REQUIRE(inst.Arg(0).U32() == 0x20); + const auto* coords = inst.Arg(1).Inst(); + REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructF32x2); + for (u32 i = 0; i < 2; ++i) + REQUIRE(coords->Arg(i).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{4 + i}); + REQUIRE(inst.Arg(2).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{6}); + REQUIRE(inst.Flags<IR::TextureInstInfo>().is_depth == 1); + } + } + REQUIRE(samples == 1); + REQUIRE(writes == std::vector<u32>{1}); +} + +TEST_CASE("Native array texel fetch preserves layer and separates XY registers", "[native-shader]") { + std::array<Native, 2> input{{ + {{0x06007b66, 0xb0800809, 0x009e0f02, 0}}, + VertexTransform.back(), + }}; + const bool sparse = GENERATE(false, true); + if (sparse) input[0].words[2] = 0x009e0b23; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.MOV_reg(lowered.instructions[0]); + visitor.MOV_reg(lowered.instructions[1]); + visitor.TLDS(lowered.instructions[2]); + u32 fetches = 0; + std::vector<u32> writes, components; + std::vector<std::pair<u32, u32>> copies; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::SetRegister && static_cast<u32>(inst.Arg(0).Reg()) < 250) + writes.push_back(static_cast<u32>(inst.Arg(0).Reg())); + if (inst.GetOpcode() == IR::Opcode::CompositeExtractF32x4) + components.push_back(inst.Arg(1).U32()); + if (inst.GetOpcode() == IR::Opcode::SetRegister && + static_cast<u32>(inst.Arg(0).Reg()) >= 250) { + REQUIRE(inst.Arg(1).Inst()->GetOpcode() == IR::Opcode::GetRegister); + copies.emplace_back(static_cast<u32>(inst.Arg(0).Reg()), + static_cast<u32>(inst.Arg(1).Inst()->Arg(0).Reg())); + } + if (inst.GetOpcode() == IR::Opcode::BoundImageFetch) { + ++fetches; + REQUIRE(inst.Flags<IR::TextureInstInfo>().type == Shader::TextureType::ColorArray2D); + const auto* coords = inst.Arg(1).Inst(); + REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructU32x3); + REQUIRE(coords->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{250}); + REQUIRE(coords->Arg(1).Inst()->Arg(0).Reg() == IR::Reg{251}); + const auto* layer = coords->Arg(2).Inst(); + REQUIRE(layer->GetOpcode() == IR::Opcode::BitFieldUExtract); + REQUIRE(layer->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{6}); + REQUIRE(layer->Arg(1).U32() == 0); + REQUIRE(layer->Arg(2).U32() == 16); + } + } + REQUIRE(fetches == 1); + REQUIRE(writes == (sparse ? std::vector<u32>{0, 1, 35} : std::vector<u32>{0, 1, 2, 3})); + REQUIRE(components == (sparse ? std::vector<u32>{0, 1, 3} : std::vector<u32>{0, 1, 2, 3})); + REQUIRE(copies == std::vector<std::pair<u32, u32>>{{250, 7}, {251, 9}}); +} + +TEST_CASE("Native bitwise LUT forms preserve source bits without changing predicates", "[native-shader]") { + std::array<Native, 5> input{{ + {{0x00007a02, 0x00c00000, 0x00000f00, 0}}, + {{0x00027a02, 0x00c00200, 0x00000f00, 0}}, + {{0x00007812, 0x80000000, 0x078ef802, 0}}, + {{0xff007322, 0x00007000, 0x000000ff, 0}}, + VertexTransform.back(), + }}; + const u32 mode = GENERATE(0U, 1U, 2U); + if (mode == 2) input[2] = Native{{0x00007812, 0x80000000, 0x078ee202, 0}}; + if (mode == 1) input[2] = Native{{0x00007212, 0x00000002, 0x078efcff, 0}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const auto word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::AST: visitor.AST(word); break; + case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break; + case Opcode::MOV32I: visitor.MOV32I(word); break; + case Opcode::LOP_reg: visitor.LOP_reg(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected native bitwise LUT lowering"); + } + } + for (const auto& inst : block.Instructions()) REQUIRE(inst.GetOpcode() != IR::Opcode::SetPred); + for (const float a : {-1.0f, 1.0f}) for (const float c : {-2.0f, 2.0f}) { + const u32 c_bits = std::bit_cast<u32>(c); + const u32 a_bits = std::bit_cast<u32>(a); + const u32 expected = mode == 2 ? (a_bits & 0x80000000U) | (c_bits & 0x7fffffffU) : + a_bits | (mode == 1 ? c_bits : c_bits & 0x80000000U); + REQUIRE(EvaluateVertex(block, {}, {a, 0, c, 0}) == + std::array<float, 2>{std::bit_cast<float>(expected), 0}); + } +} + +TEST_CASE("Native one-dimensional texel fetch preserves its integer coordinate", "[native-shader]") { + const std::array<Native, 2> input{{ + {{0x04017b66, 0x108008ff, 0x009e01ff, 0}}, + VertexTransform.back(), + }}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.TLDS(lowered.instructions.front()); + u32 fetches = 0; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::BoundImageFetch) { + ++fetches; + REQUIRE(inst.Flags<IR::TextureInstInfo>().type == Shader::TextureType::Color1D); + REQUIRE(inst.Arg(1).Inst()->GetOpcode() == IR::Opcode::GetRegister); + REQUIRE(inst.Arg(1).Inst()->Arg(0).Reg() == IR::Reg{4}); + } + } + REQUIRE(fetches == 1); +} + +TEST_CASE("Native call and return preserve values through the actual shader frontend", "[native-shader]") { + std::array<Native, 9> input{{ + {{0x00007802, 0x40000000, 0x00000f00, 0}}, // argument 2.0 + {{0x00377802, 0x00000030, 0x00000f00, 0}}, // native return PC + {{0x00007944, 0x00000020, 0x03c00000, 0}}, // call offset0x50 + {{0xff007322, 0x00007003, 0x000000ff, 0}}, + VertexTransform.back(), + {{0x00037421, 0x40400000, 0x00010000, 0}}, // callee: add 3.0 + {{0x00087202, 0x00000037, 0x00000f00, 0}}, + {{0x00097802, 0x00000000, 0x00000f00, 0}}, + {{0x08007950, 0xffffff70, 0x03c3ffff, 0}}, + }}; + const bool alternate_link = GENERATE(false, true); + if (alternate_link) { + input[1].words[0] = 0x00397802; + input[6] = Native{{0x00007202, 0x00000039, 0x00000f00, 0}}; + input[7].words[0] = 0x00017802; + input[8].words[0] = 0x00007950; + } + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + REQUIRE(lowered.source_size == input.size() * 16); + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(std::span{input}.first(5))); + auto broken = input; + broken[1].words[1] += 16; + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(broken)); + broken = input; + broken.back().words[1] += 16; + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(broken)); + std::vector<u64> code(0x50 / 8); + for (const auto word : lowered.instructions) { + if ((code.size() * 8) % 32 == 0) code.push_back(0); + code.push_back(word); + } + class CallEnvironment final : public EmptyEnvironment { + public: + explicit CallEnvironment(const std::vector<u64>& code_) : code{code_} { stage = Shader::Stage::VertexB; } + u64 ReadInstruction(u32 address) override { return code.at(address / 8); } + const std::vector<u64>& code; + } env{code}; + Shader::ObjectPool<Shader::Maxwell::Flow::Block> flow_pool; + Shader::ObjectPool<IR::Inst> inst_pool; + Shader::ObjectPool<IR::Block> block_pool; + Shader::Maxwell::Flow::CFG cfg{env, flow_pool, 0x50}; + REQUIRE(cfg.Functions().size() == 2); + Shader::HostTranslateInfo host; + const auto program = Shader::Maxwell::TranslateProgram(inst_pool, block_pool, env, cfg, host); + u32 stores = 0; + for (const auto* block : program.blocks) { + for (const auto& inst : block->Instructions()) { + if (inst.GetOpcode() != IR::Opcode::SetAttribute) continue; + ++stores; + REQUIRE(inst.Arg(1).IsImmediate()); + REQUIRE(inst.Arg(1).F32() == 5.0f); + } + } + REQUIRE(stores == 1); +} + +TEST_CASE("Native scaled multiplication preserves captured power-of-two scales", "[native-shader]") { + std::array<Native, 4> input{{ + VertexTransform.front(), + {{0x00007220, 0x00000001, 0x00310000, 0}}, + {{0xff007322, 0x00007000, 0x000000ff, 0}}, + VertexTransform.back(), + }}; + const u32 scale = GENERATE(3U, 4U, 5U, 6U); + input[1].words[2] = (scale << 20) | 0x10000; + const float factor = scale == 3 ? 0.5f : scale == 5 ? 2.0f : scale == 6 ? 4.0f : 1.0f; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.ALD(lowered.instructions[0]); + visitor.FMUL_reg(lowered.instructions[1]); + visitor.AST(lowered.instructions[2]); + REQUIRE(EvaluateVertex(block, {8, 4}, {}) == std::array<float, 2>{32 * factor, 0}); + REQUIRE(EvaluateVertex(block, {-2, 3}, {}) == std::array<float, 2>{-6 * factor, 0}); +} + +TEST_CASE("Native integer multiply preserves high words and modular multiply-add", "[native-shader]") { + const u32 mode = GENERATE(0U, 1U, 2U, 3U, 4U); + Native instruction{{0x00037827, 0xcccccccd, 0x078e00ff, 0}}; + if (mode == 1) instruction = Native{{0x00037824, 0xfffffff6, 0x078e0202, 0}}; + if (mode == 2) instruction = Native{{0x00037a24, 0x00c00000, 0x078e0202, 0}}; + if (mode == 4) instruction = Native{{0x00037811, 0x00000070, 0x078e20ff, 0}}; + if (mode == 3) instruction = Native{{0x00037224, 0x00000001, 0x078e0202, 0}}; + const std::array<Native, 2> input{{instruction, VertexTransform.back()}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const auto word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::XMAD_imm: visitor.XMAD_imm(word); break; + case Opcode::XMAD_reg: visitor.XMAD_reg(word); break; + case Opcode::MOV32I: visitor.MOV32I(word); break; + case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break; + case Opcode::ISCADD_reg: visitor.ISCADD_reg(word); break; + case Opcode::SHR_imm: visitor.SHR_imm(word); break; + case Opcode::IADD_reg: visitor.IADD_reg(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected canonical integer multiplication instruction"); + } + } + for (u32 a : {0U, 1U, 9U, 10U, 65535U, 65536U, 0x80000000U, 0xfffffffeU, 0xffffffffU, 0x12345678U}) { + std::array<u32, 256> regs{}; + regs[0] = a; + regs[1] = 0x76543210; + regs[2] = 0xfedcba98; + std::unordered_map<const IR::Inst*, u32> values; + const auto get = [&](const IR::Value& value) -> u32 { + if (!value.IsImmediate()) return values.at(value.Inst()); + return value.Type() == IR::Type::U1 ? static_cast<u32>(value.U1()) : value.U32(); + }; + for (const auto& inst : block.Instructions()) { + const auto arg = [&](size_t i) { return get(inst.Arg(i)); }; + switch (inst.GetOpcode()) { + case IR::Opcode::GetRegister: values[&inst] = regs[static_cast<u32>(inst.Arg(0).Reg())]; break; + case IR::Opcode::SetRegister: regs[static_cast<u32>(inst.Arg(0).Reg())] = arg(1); break; + case IR::Opcode::GetCbufU32: + REQUIRE(arg(0) == 3); + REQUIRE(arg(1) == 0); + values[&inst] = 0x76543210; + break; + case IR::Opcode::BitFieldUExtract: values[&inst] = (arg(0) >> arg(1)) & ((1U << arg(2)) - 1); break; + case IR::Opcode::IMul32: values[&inst] = arg(0) * arg(1); break; + case IR::Opcode::IAdd32: values[&inst] = arg(0) + arg(1); break; + case IR::Opcode::ShiftLeftLogical32: values[&inst] = arg(0) << arg(1); break; + case IR::Opcode::ShiftRightLogical32: values[&inst] = arg(0) >> arg(1); break; + case IR::Opcode::ULessThan: values[&inst] = arg(0) < arg(1); break; + case IR::Opcode::SLessThan: values[&inst] = static_cast<s32>(arg(0)) < static_cast<s32>(arg(1)); break; + case IR::Opcode::SelectU32: values[&inst] = arg(0) ? arg(1) : arg(2); break; + default: FAIL("Unexpected multiplication IR operation " << IR::NameOf(inst.GetOpcode())); + } + } + const u32 b = mode == 0 ? 0xcccccccd : mode == 1 ? 0xfffffff6 : 0x76543210; + const u64 product = u64{a} * b; + const u32 expected = mode == 4 ? (a << 4) + 0x70U : mode == 0 ? static_cast<u32>(product >> 32) + : static_cast<u32>(product + 0xfedcba98U); + REQUIRE(regs[3] == expected); + } +} + +TEST_CASE("Native uniform constants keep their indexed load and arithmetic values", "[native-shader]") { + const std::array<Native, 7> input{{ + {{0xff007321, 0x000080ff, 0x00000000, 0}}, + {{0x00047ab9, 0x00c00000, 0x00000800, 0}}, + {{0x04047899, 0x00000002, 0x0800063f, 0}}, + {{0x04057abb, 0x00c00000, 0x08000800, 0}}, + {{0x00007c20, 0x00000005, 0x08410000, 0}}, + {{0x00007e21, 0x80000005, 0x08010000, 0}}, + VertexTransform.back(), + }}; + auto code = input; + // Store the computed scalar instead of ending immediately. + std::vector<Native> program(code.begin(), code.end() - 1); + program.push_back(Native{{0xff007322, 0x00007000, 0x000000ff, 0}}); + program.push_back(VertexTransform.back()); + const auto lowered = Shader::Ampere::LowerToMaxwell(program); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const auto word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::ALD: visitor.ALD(word); break; + case Opcode::AST: visitor.AST(word); break; + case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break; + case Opcode::MOV_reg: visitor.MOV_reg(word); break; + case Opcode::LOP32I: visitor.LOP32I(word); break; + case Opcode::SHL_imm: visitor.SHL_imm(word); break; + case Opcode::LDC: visitor.LDC(word); break; + case Opcode::FMUL_reg: visitor.FMUL_reg(word); break; + case Opcode::FADD_reg: visitor.FADD_reg(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected uniform instruction lowering"); + } + } + const std::array<float, 4> constants{std::bit_cast<float>(1U), 3.0f, 7.0f, 9.0f}; + REQUIRE(EvaluateVertex(block, {2, 0}, constants) == std::array<float, 2>{3, 0}); + REQUIRE(EvaluateVertex(block, {4, 0}, constants) == std::array<float, 2>{9, 0}); + auto conflict = program; + conflict[0].words[0] = 0xfff87321; // Native R248 conflicts with the reserved uniform bank. + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(conflict)); +} + +TEST_CASE("Native predicate-output LUTs preserve boolean bit tests", "[native-shader]") { + const std::array<Native, 3> input{{ + {{0x00ff7812, 0x00000001, 0x0780c0ff, 0}}, + {{0x01ff7812, 0x00000001, 0x07824802, 0}}, + VertexTransform.back(), + }}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const auto word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::MOV32I: visitor.MOV32I(word); break; + case Opcode::LOP_reg: visitor.LOP_reg(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected predicate LUT lowering"); + } + } + for (u32 a = 0; a < 4; ++a) for (u32 b = 0; b < 4; ++b) for (u32 c = 0; c < 4; ++c) { + std::array<u32, 256> regs{}; + regs[0] = a; regs[1] = b; regs[2] = c; + std::array<bool, 7> predicates{false, false, true, false, true, false, true}; + std::unordered_map<const IR::Inst*, u32> values; + const auto get = [&](const IR::Value& value) -> u32 { + if (!value.IsImmediate()) return values.at(value.Inst()); + return value.Type() == IR::Type::U1 ? static_cast<u32>(value.U1()) : value.U32(); + }; + for (const auto& inst : block.Instructions()) { + const auto arg = [&](size_t i) { return get(inst.Arg(i)); }; + switch (inst.GetOpcode()) { + case IR::Opcode::GetRegister: values[&inst] = regs[static_cast<u32>(inst.Arg(0).Reg())]; break; + case IR::Opcode::SetRegister: regs[static_cast<u32>(inst.Arg(0).Reg())] = arg(1); break; + case IR::Opcode::BitwiseAnd32: values[&inst] = arg(0) & arg(1); break; + case IR::Opcode::BitwiseOr32: values[&inst] = arg(0) | arg(1); break; + case IR::Opcode::BitwiseXor32: values[&inst] = arg(0) ^ arg(1); break; + case IR::Opcode::INotEqual: values[&inst] = arg(0) != arg(1); break; + case IR::Opcode::SetPred: predicates[static_cast<u32>(inst.Arg(0).Pred())] = arg(1) != 0; break; + default: FAIL("Unexpected predicate LUT IR " << IR::NameOf(inst.GetOpcode())); + } + } + REQUIRE(predicates[0] == ((a & 1) != 0)); + REQUIRE(predicates[1] == (((b ^ c) & 1) != 0)); + REQUIRE(std::array<bool, 5>{predicates[2], predicates[3], predicates[4], predicates[5], predicates[6]} == + std::array<bool, 5>{true, false, true, false, true}); + } +} + +TEST_CASE("Native cube sampling preserves XYZ, explicit LOD, and split RGB destinations", "[native-shader]") { + std::array<Native, 2> input{{ + {{0x04007b60, 0x70800808, 0x059e0703, 0}}, + VertexTransform.back(), + }}; + const bool volume = GENERATE(false, true); + if (volume) { + input[0].words[1] = (input[0].words[1] & ~(7U << 29)) | (2U << 29); + input[0].words[2] = (input[0].words[2] & ~(7U << 23)) | (1U << 23); + } + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const auto word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::MOV_reg: visitor.MOV_reg(word); break; + case Opcode::TEX: visitor.TEX(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected cube sampling lowering"); + } + } + u32 samples = 0; + std::vector<std::pair<u32, u32>> copies; + std::vector<u32> writes; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::SetRegister) { + const u32 target = static_cast<u32>(inst.Arg(0).Reg()); + if (target < 248) writes.push_back(target); + if (inst.Arg(1).Inst()->GetOpcode() == IR::Opcode::GetRegister) + copies.emplace_back(target, static_cast<u32>(inst.Arg(1).Inst()->Arg(0).Reg())); + } + if (inst.GetOpcode() == IR::Opcode::BoundImageSampleExplicitLod) { + ++samples; + REQUIRE(inst.Flags<IR::TextureInstInfo>().type == (volume ? Shader::TextureType::Color3D : Shader::TextureType::ColorCube)); + const auto* coords = inst.Arg(1).Inst(); + REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructF32x3); + for (u32 i = 0; i < 3; ++i) + REQUIRE(coords->Arg(i).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{250 + i}); + if (volume) REQUIRE(inst.Arg(2).F32() == 0.0f); + else REQUIRE(inst.Arg(2).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{9}); + } + } + REQUIRE(samples == 1); + REQUIRE(writes == std::vector<u32>{0, 1, 3}); + REQUIRE(copies == std::vector<std::pair<u32, u32>>{{250, 4}, {251, 5}, {252, 8}, {0, 250}, {1, 251}, {3, 252}}); +} + +TEST_CASE("Native branch maps survive code caching and reject invalid addresses", "[native-shader]") { + std::vector<u64> code(64); + const std::array<u32, 4> offsets{0, 2, 4, 8}; + Shader::Ampere::AppendNativeAddressMap(code, offsets); + REQUIRE(Shader::Ampere::HasNativeAddressMap(code)); + REQUIRE(Shader::Ampere::NativeProgramAddress(code, 0) == 0x50U); + REQUIRE(Shader::Ampere::NativeProgramAddress(code, 16) == 0x68U); + const auto restored = code; + REQUIRE(Shader::Ampere::NativeProgramAddress(restored, 48) == 0xa8U); + REQUIRE_FALSE(Shader::Ampere::NativeProgramAddress(code, 1)); + REQUIRE_FALSE(Shader::Ampere::NativeProgramAddress(code, 64)); + code[code.size() - 2] = 100000; + REQUIRE_FALSE(Shader::Ampere::NativeProgramAddress(code, 0)); +} + +TEST_CASE("Native computed jumps retain table values while mapping code destinations", "[native-shader]") { + const std::array<Native, 12> input{{ + {{0xff007321, 0x0002fcff, 0x00000000, 0}}, + {{0x00007817, 0x00000001, 0x03800000, 0}}, + {{0x00027819, 0x00000002, 0x000006ff, 0}}, + {{0x02007b82, 0x00400000, 0x00000800, 0}}, + {{0xff037421, 0x3f000000, 0x00010000, 0}}, + {{0x00007949, 0xffffffa0, 0x0383ffff, 0}}, + {{0x00007802, 0x3f800000, 0x00000f00, 0}}, + {{0x00007947, 0x00000020, 0x03800000, 0}}, + {{0x00007802, 0x40000000, 0x00000f00, 0}}, + {{0x00007947, 0x00000000, 0x03800000, 0}}, + {{0xff007322, 0x00007000, 0x000000ff, 0}}, + VertexTransform.back(), + }}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + std::vector<u64> code(0x50 / 8); + for (const auto word : lowered.instructions) { + if ((code.size() * 8) % 32 == 0) code.push_back(0); + code.push_back(word); + } + code.push_back(0); + Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets); + class BranchEnvironment final : public EmptyEnvironment { + public: + explicit BranchEnvironment(const std::vector<u64>& code_) : code{code_} { stage = Shader::Stage::VertexB; } + u64 ReadInstruction(u32 address) override { return code.at(address / 8); } + u32 ReadCbufValue(u32 bank, u32 offset) override { + REQUIRE(bank == 1); + REQUIRE((offset == 0 || offset == 4)); + return offset == 0 ? 0x60 : 0x80; + } + u32 TranslateProgramAddress(u32 address) const override { + const auto target = Shader::Ampere::NativeProgramAddress(code, address); + if (!target) throw std::logic_error("invalid synthetic branch target"); + return *target; + } + const std::vector<u64>& code; + } env{code}; + Shader::ObjectPool<Shader::Maxwell::Flow::Block> flow_pool; + Shader::ObjectPool<IR::Inst> inst_pool; + Shader::ObjectPool<IR::Block> block_pool; + Shader::Maxwell::Flow::CFG cfg{env, flow_pool, 0x50}; + u32 jumps = 0; + for (const auto& block : cfg.Functions().front().blocks) { + if (block.end_class != Shader::Maxwell::Flow::EndClass::IndirectBranch) continue; + ++jumps; + REQUIRE(block.branch_offset == 0); + REQUIRE(block.indirect_branches.size() == 2); + for (const auto& branch : block.indirect_branches) { + REQUIRE((branch.address == 0x60 || branch.address == 0x80)); + REQUIRE(branch.block->begin.Offset() == env.TranslateProgramAddress(branch.address)); + REQUIRE(branch.block->begin.Offset() != branch.address); + } + } + REQUIRE(jumps == 1); + Shader::HostTranslateInfo host; + const auto program = Shader::Maxwell::TranslateProgram(inst_pool, block_pool, env, cfg, host); + REQUIRE_FALSE(program.blocks.empty()); +} + +TEST_CASE("Native forward branches retain paths beyond an early unconditional exit", "[native-shader]") { + const Native exit{{0x0000794d, 0, 0x03800000, 0x000fea00}}; + const std::array<Native, 4> source{{ + {{0x00000947, 0x20, 0x03800000, 0x000fea00}}, + exit, + {{0x00017802, 42, 0x00000f00, 0x000fe400}}, + exit, + }}; + const auto lowered = Shader::Ampere::LowerToMaxwell(source); + REQUIRE(lowered); + REQUIRE(lowered.source_size == sizeof(source)); + REQUIRE(lowered.source_instruction_offsets.size() == source.size()); + REQUIRE(lowered.instructions.size() == 4); + const auto truncated = Shader::Ampere::LowerToMaxwell(std::span{source}.first(2)); + REQUIRE_FALSE(truncated); + REQUIRE(truncated.instructions.empty()); +} + +TEST_CASE("Native register-to-predicate writes only masked predicate bits", "[native-shader]") { + const std::array<Native, 2> source{{{{0x05007804, 7, 0, 0x000fe200}}, VertexTransform.back()}}; + for (u32 byte = 0; byte < 4; ++byte) { + auto selected = source; + selected[0].words[2] = byte << 12; + const auto lowered = Shader::Ampere::LowerToMaxwell(selected); + REQUIRE(lowered); + REQUIRE(Shader::Maxwell::Decode(lowered.instructions[0]) == Shader::Maxwell::Opcode::R2P_imm); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.R2P_imm(lowered.instructions[0]); + for (u32 value = 0; value < 128; ++value) { + std::array<bool, 7> predicates{true, false, true, false, true, false, true}; + std::unordered_map<const IR::Inst*, u32> values; + const auto get = [&](const IR::Value& v) -> u32 { + if (!v.IsImmediate()) return values.at(v.Inst()); + return v.Type() == IR::Type::U1 ? static_cast<u32>(v.U1()) : v.U32(); + }; + for (const auto& inst : block.Instructions()) { + const auto arg = [&](size_t i) { return get(inst.Arg(i)); }; + switch (inst.GetOpcode()) { + case IR::Opcode::GetRegister: + REQUIRE(static_cast<u32>(inst.Arg(0).Reg()) == 5); + values[&inst] = value << (byte * 8); break; + case IR::Opcode::GetPred: values[&inst] = predicates[static_cast<u32>(inst.Arg(0).Pred())]; break; + case IR::Opcode::SetPred: predicates[static_cast<u32>(inst.Arg(0).Pred())] = arg(1) != 0; break; + case IR::Opcode::BitFieldUExtract: values[&inst] = (arg(0) >> arg(1)) & ((1U << arg(2)) - 1); break; + case IR::Opcode::GetZeroFromOp: values[&inst] = arg(0) == 0; break; + case IR::Opcode::IEqual: values[&inst] = arg(0) == arg(1); break; + case IR::Opcode::LogicalNot: values[&inst] = !arg(0); break; + case IR::Opcode::SelectU1: values[&inst] = arg(0) ? arg(1) : arg(2); break; + default: FAIL("Unexpected R2P IR " << IR::NameOf(inst.GetOpcode())); + } + } + for (u32 bit = 0; bit < 7; ++bit) + REQUIRE(predicates[bit] == (bit < 3 ? bool(value & (1U << bit)) : bool(bit % 2 == 0))); + } + } + auto invalid = source; + invalid[0].words[2] = 1; + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(invalid)); +} + +TEST_CASE("Native 2D explicit LOD sampling preserves its paired coordinates", "[native-shader]") { + const std::array<Native, 2> input{{{{0xc007b60, 0x30800812, 0x59e070b, 0x100f4200}}, VertexTransform.back()}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.TEXS(lowered.instructions[0]); + u32 samples = 0; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() != IR::Opcode::BoundImageSampleExplicitLod) continue; + ++samples; + REQUIRE(inst.Flags<IR::TextureInstInfo>().type == Shader::TextureType::Color2D); + const auto* coords = inst.Arg(1).Inst(); + REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructF32x2); + for (u32 i = 0; i < 2; ++i) + REQUIRE(coords->Arg(i).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{12 + i}); + REQUIRE(inst.Arg(2).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{18}); + } + REQUIRE(samples == 1); +} + +namespace { +using Warp = std::array<u32, 32>; +void ExecuteNativeWarp(IR::Block& block, u32 active, + std::array<std::array<u32, 256>, 32>& registers, + std::array<std::array<bool, 7>, 32>& predicates) { + std::unordered_map<const IR::Inst*, Warp> values; + const auto get = [&](const IR::Value& value) -> Warp { + if (!value.IsImmediate()) return values.at(value.Inst()); + Warp result; + result.fill(value.Type() == IR::Type::U1 ? static_cast<u32>(value.U1()) : value.U32()); + return result; + }; + for (const auto& inst : block.Instructions()) { + std::array<Warp, 4> args{}; + for (size_t i = 0; i < inst.NumArgs(); ++i) { + if (inst.Arg(i).Type() != IR::Type::Reg && inst.Arg(i).Type() != IR::Type::Pred) + args[i] = get(inst.Arg(i)); + } + Warp result{}; + const auto op = inst.GetOpcode(); + if (op == IR::Opcode::SubgroupBallot || op == IR::Opcode::VoteAny || op == IR::Opcode::VoteAll) { + u32 mask = 0; + for (u32 lane = 0; lane < 32; ++lane) + if ((active & (1U << lane)) && args[0][lane]) mask |= 1U << lane; + result.fill(op == IR::Opcode::SubgroupBallot ? mask : op == IR::Opcode::VoteAny ? mask != 0 : mask == active); + } else { + for (u32 lane = 0; lane < 32; ++lane) { + const auto a = args[0][lane], b = args[1][lane], c = args[2][lane]; + switch (op) { + case IR::Opcode::GetRegister: result[lane] = static_cast<u32>(inst.Arg(0).Reg()) == 255 ? 0 : registers[lane][static_cast<u32>(inst.Arg(0).Reg())]; break; + case IR::Opcode::SetRegister: + if ((active & (1U << lane)) && static_cast<u32>(inst.Arg(0).Reg()) != 255) registers[lane][static_cast<u32>(inst.Arg(0).Reg())] = b; + break; + case IR::Opcode::GetPred: result[lane] = predicates[lane][static_cast<u32>(inst.Arg(0).Pred())]; break; + case IR::Opcode::SetPred: + if (active & (1U << lane)) predicates[lane][static_cast<u32>(inst.Arg(0).Pred())] = b != 0; + break; + case IR::Opcode::BitCastF32U32: case IR::Opcode::BitCastU32F32: result[lane] = a; break; + case IR::Opcode::SelectU32: case IR::Opcode::SelectU1: result[lane] = a ? b : c; break; + case IR::Opcode::BitwiseAnd32: result[lane] = a & b; break; + case IR::Opcode::BitwiseOr32: result[lane] = a | b; break; + case IR::Opcode::ShiftRightLogical32: result[lane] = b < 32 ? a >> b : 0; break; + case IR::Opcode::ShiftLeftLogical32: result[lane] = b < 32 ? a << b : 0; break; + case IR::Opcode::INeg32: result[lane] = 0U - a; break; + case IR::Opcode::IAdd32: result[lane] = a + b; break; + case IR::Opcode::FindUMsb32: result[lane] = a ? 31U - std::countl_zero(a) : ~0U; break; + case IR::Opcode::BitFieldUExtract: result[lane] = (a >> b) & ((1U << c) - 1); break; + case IR::Opcode::ULessThan: result[lane] = a < b; break; + case IR::Opcode::SLessThan: result[lane] = static_cast<s32>(a) < static_cast<s32>(b); break; + case IR::Opcode::IEqual: result[lane] = a == b; break; + case IR::Opcode::GetZeroFromOp: result[lane] = a == 0; break; + case IR::Opcode::LogicalAnd: result[lane] = a && b; break; + case IR::Opcode::LogicalNot: result[lane] = !a; break; + case IR::Opcode::GetInBoundsFromOp: result[lane] = get(inst.Arg(0).Inst()->Arg(1))[lane] <= 31; break; + case IR::Opcode::FSwizzleAdd: { + const u32 mode = (c >> ((lane & 3) * 2)) & 3; + const float x = std::bit_cast<float>(a), y = std::bit_cast<float>(b); + const float ma = mode == 0 || mode == 2 ? -1.0f : mode == 1 ? 1.0f : 0.0f; + const float mb = mode == 0 || mode == 1 ? -1.0f : mode == 2 ? 1.0f : 0.0f; + result[lane] = std::bit_cast<u32>(x * ma + y * mb); + break; + } + case IR::Opcode::ShuffleButterfly: result[lane] = args[0][lane ^ b]; break; + case IR::Opcode::ShuffleIndex: result[lane] = (active & (1U << (b & 31))) ? args[0][b & 31] : 0xdeadbeefU; break; + default: FAIL("Unexpected warp-uniform IR " << IR::NameOf(op)); + } + } + } + values.emplace(&inst, result); + } +} +} + +TEST_CASE("Native warp uniform transfers use the first selected active lane and preserve predicates", "[native-shader]") { + const u32 operation = GENERATE(0U, 1U, 2U, 3U, 4U, 5U, 6U, 7U); + const Native instruction = operation < 2 + ? Native{{operation == 0 ? 0x020513c2U : 0x020593c2U, 0, 0x000e0000, 0}} + : operation < 4 ? Native{{0x003f7886, 0, operation == 2 ? 0x00800100U : 0x00800000U, 0}} + : Native{{0x0000781c, 0, operation == 4 ? 0x03f2f008U : operation == 5 ? 0x03f0f070U : operation == 6 ? 0x03f0e170U : 0x00f41070U, 0}}; + const std::array<Native, 2> source{{instruction, VertexTransform.back()}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(source); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const auto word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::PSETP: visitor.PSETP(word); break; + case Opcode::SHR_imm: visitor.SHR_imm(word); break; + case Opcode::SHL_imm: visitor.SHL_imm(word); break; + case Opcode::MOV32I: visitor.MOV32I(word); break; + case Opcode::P2R_imm: visitor.P2R_imm(word); break; + case Opcode::R2P_imm: visitor.R2P_imm(word); break; + case Opcode::VOTE: visitor.VOTE(word); break; + case Opcode::IADD_reg: visitor.IADD_reg(word); break; + case Opcode::LOP_reg: visitor.LOP_reg(word); break; + case Opcode::FLO_reg: visitor.FLO_reg(word); break; + case Opcode::SHFL: visitor.SHFL(word); break; + case Opcode::SEL_reg: visitor.SEL_reg(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected warp-uniform lowering"); + } + } + using Warp = std::array<u32, 32>; + for (const u32 active : {1U, 0x80000000U, 0x80000001U, 0x55555555U, 0xffffffffU}) { + for (const u32 selected : {0U, 1U, 0x80000000U, 0xaaaaaaaaU, 0xffffffffU}) { + std::array<std::array<u32, 256>, 32> registers{}; + std::array<std::array<bool, 7>, 32> predicates{}; + for (u32 lane = 0; lane < 32; ++lane) { + registers[lane][2] = 100 + lane * 17; + registers[lane][221] = 0xcafebeef; + registers[lane][239] = (selected & (1U << lane)) ? 64 : 0; + for (u32 p = 0; p < 7; ++p) predicates[lane][p] = ((lane + p) & 1) != 0; + predicates[lane][1] = ((selected & (1U << lane)) != 0) != (operation == 4); + } + const auto original_predicates = predicates; + ExecuteNativeWarp(block, active, registers, predicates); + const u32 ballot = active & (operation == 1 ? ~selected : selected); + const u32 first = ballot ? std::countr_zero(ballot) : 0; + for (u32 lane = 0; lane < 32; ++lane) { + if (!(active & (1U << lane))) continue; + auto expected_predicates = original_predicates[lane]; + if (operation == 7) expected_predicates[2] = original_predicates[lane][1] && original_predicates[lane][0]; + if (operation == 4) expected_predicates[1] = (selected & (1U << lane)) != 0; + if (operation == 5 || operation == 6) expected_predicates[0] = operation == 5; + REQUIRE(predicates[lane] == expected_predicates); + if (operation < 2) REQUIRE(registers[lane][221] == (ballot ? 100 + first * 17 : 0xcafebeef)); + else if (operation < 4) REQUIRE(registers[lane][239] == (operation == 2 ? (ballot ? 64U : 0U) : (ballot == active ? 64U : 0U))); + } + } + } +} + +TEST_CASE("Native uniform 64-bit loads populate consecutive registers", "[native-shader]") { + const std::array<Native, 6> input{{ + {{0x00057ab9, 0x00c00000, 0x00000800, 0}}, + {{0x05067abb, 0x00c00100, 0x08000a00, 0}}, + {{0x00007c02, 6, 0x08000f00, 0}}, + {{0x00017c02, 7, 0x08000f00, 0}}, + {{0xff007322, 0x00007000, 0x000004ff, 0}}, + VertexTransform.back(), + }}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (const auto word : lowered.instructions) { + using Shader::Maxwell::Opcode; + switch (Shader::Maxwell::Decode(word)) { + case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break; + case Opcode::MOV_reg: visitor.MOV_reg(word); break; + case Opcode::LDC: visitor.LDC(word); break; + case Opcode::AST: visitor.AST(word); break; + case Opcode::EXIT: break; + default: FAIL("Unexpected uniform pair lowering"); + } + } + REQUIRE(EvaluateVertex(block, {}, {std::bit_cast<float>(0U), 3, 7, 9}) == std::array<float, 2>{3, 7}); +} + +TEST_CASE("Native texture dimension queries retain mip input and width-height outputs", "[native-shader]") { + const std::array<Native, 2> source{{{{0x03027b6f, 0x00800800, 0x000003ff, 0}}, VertexTransform.back()}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(source); + REQUIRE(lowered); + REQUIRE(lowered.required_texture_buffer == 2); + REQUIRE(Shader::Maxwell::Decode(lowered.instructions[0]) == Shader::Maxwell::Opcode::TXQ); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.TXQ(lowered.instructions[0]); + std::vector<std::pair<u32, u32>> outputs; + u32 queries = 0; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::BoundImageQueryDimensions) { + ++queries; + REQUIRE(inst.Arg(0).U32() == 32); + REQUIRE(inst.Arg(1).Inst()->Arg(0).Reg() == IR::Reg{3}); + REQUIRE(inst.Arg(2).U1()); + } + if (inst.GetOpcode() == IR::Opcode::SetRegister) { + const auto* extract = inst.Arg(1).Inst(); + REQUIRE(extract->GetOpcode() == IR::Opcode::CompositeExtractU32x4); + outputs.emplace_back(static_cast<u32>(inst.Arg(0).Reg()), extract->Arg(1).U32()); + } + } + REQUIRE(queries == 1); + REQUIRE(outputs == std::vector<std::pair<u32, u32>>{{2, 0}, {3, 1}}); + auto invalid = source; + invalid[0].words[1] |= 1U << 30; + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(invalid)); +} + +TEST_CASE("Native scalar global stores retain both uniform address words and source data", "[native-shader]") { + std::array<Native, 2> input{{ + {{0xff007986, 7, 0x0c10a904, 0}}, + VertexTransform.back(), + }}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + REQUIRE(Shader::Maxwell::Decode(lowered.instructions.front()) == Shader::Maxwell::Opcode::STG); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.STG(lowered.instructions.front()); + std::vector<u32> registers; + u32 stores{}; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::GetRegister) + registers.push_back(static_cast<u32>(inst.Arg(0).Reg())); + if (inst.GetOpcode() == IR::Opcode::CompositeConstructU32x2) { + REQUIRE(inst.Arg(0).Inst()->Arg(0).Reg() == IR::Reg::R220); + REQUIRE(inst.Arg(1).Inst()->Arg(0).Reg() == IR::Reg::R221); + } + if (inst.GetOpcode() == IR::Opcode::WriteGlobal32) { + REQUIRE(inst.Arg(1).Inst()->Arg(0).Reg() == IR::Reg::R7); + ++stores; + } + } + std::sort(registers.begin(), registers.end()); + REQUIRE(registers == std::vector<u32>{7, 220, 221}); + REQUIRE(stores == 1); + input[0].words[2] |= 1; // An unaligned uniform address pair is unsupported. + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(input)); + input[0].words[2] = 0x0c10ab04; // A different store width must not become scalar. + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(input)); +} + +TEST_CASE("Captured native compute compiles with shared memory and atomics", "[.native-compute-captured]") { + const char* directory = std::getenv("EDEN_NATIVE_SHADER_FIXTURES"); + REQUIRE(directory != nullptr); + std::ifstream file(std::filesystem::path{directory} / "compute.source.bin", std::ios::binary | std::ios::ate); + REQUIRE(file.good()); + const auto size = static_cast<size_t>(file.tellg()); + REQUIRE(size % 16 == 0); + std::vector<Native> input(size / 16); + file.seekg(0); + file.read(reinterpret_cast<char*>(input.data()), size); + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + INFO("offset=" << lowered.error_offset << " error=" << lowered.error); + REQUIRE(lowered); + std::vector<u64> code(0x50 / 8); + for (u64 word : lowered.instructions) { + if ((code.size() * 8) % 32 == 0) code.push_back(0); + code.push_back(word); + } + code.push_back(0); + Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets); + std::array<u32, 64> qmd{}; + std::ifstream descriptor(std::filesystem::path{directory} / "compute.qmd.bin", std::ios::binary); + REQUIRE(descriptor.good()); + descriptor.read(reinterpret_cast<char*>(qmd.data()), sizeof(qmd)); + const std::array<u32, 3> group{qmd[18] >> 16, qmd[19] & 0xffff, qmd[19] >> 16}; + const u32 shared_size = qmd[17] & 0x3ffff; + class ComputeFixtureEnvironment final : public EmptyEnvironment { + public: + ComputeFixtureEnvironment(const std::vector<u64>& code_, u32 shared_, std::array<u32, 3> group_, Shader::TextureType texture_) + : code{code_}, shared{shared_}, group{group_}, texture{texture_} { + stage = Shader::Stage::Compute; + start_address = 0x50; + is_proprietary_driver = true; + } + u64 ReadInstruction(u32 address) override { return code.at(address / 8); } + u32 TranslateProgramAddress(u32 address) const override { + const auto translated = Shader::Ampere::NativeProgramAddress(code, address); + if (!translated) throw std::logic_error("invalid compute branch target"); + return *translated; + } + u32 ReadCbufValue(u32, u32) override { return 0; } + u32 TextureBoundBuffer() const override { return 0; } + u32 SharedMemorySize() const override { return shared; } + std::array<u32, 3> WorkgroupSize() const override { return group; } + Shader::TextureType ReadTextureType(u32) override { return texture; } + const std::vector<u64>& code; + u32 shared; + std::array<u32, 3> group; + Shader::TextureType texture; + } env{code, shared_size, group, + std::any_of(input.begin(), input.end(), [](const auto& instruction) { + return instruction.Bits(0, 12) == 0xb66 && instruction.Bits(61, 3) == 2; + }) ? Shader::TextureType::Color3D : std::any_of(input.begin(), input.end(), [](const auto& instruction) { + return (instruction.Bits(0, 12) == 0xb66 && instruction.Bits(61, 3) == 1) || + instruction.Bits(0, 12) == 0xb97 || instruction.Bits(0, 12) == 0xb9b; + }) ? Shader::TextureType::Color2D : Shader::TextureType::Color1D}; + Shader::ObjectPool<Shader::Maxwell::Flow::Block> flow_pool; + Shader::ObjectPool<IR::Inst> inst_pool; + Shader::ObjectPool<IR::Block> block_pool; + Shader::Maxwell::Flow::CFG cfg{env, flow_pool, env.StartAddress()}; + Shader::HostTranslateInfo host; + host.support_int64 = true; + host.support_float64 = true; + host.support_float16 = true; + host.support_conditional_barrier = true; + const auto program = Shader::Maxwell::TranslateProgram(inst_pool, block_pool, env, cfg, host); + REQUIRE(program.stage == Shader::Stage::Compute); + REQUIRE_FALSE(program.blocks.empty()); + u32 barriers{}, shared_atomics{}, global_atomics{}, stores{}, image_stores{}, image_loads{}; + for (const auto* block : program.blocks) for (const auto& inst : block->Instructions()) { + image_loads += inst.GetOpcode() == IR::Opcode::ImageRead || inst.GetOpcode() == IR::Opcode::BoundImageRead; + barriers += inst.GetOpcode() == IR::Opcode::Barrier; + shared_atomics += inst.GetOpcode() == IR::Opcode::SharedAtomicIAdd32; + global_atomics += inst.GetOpcode() == IR::Opcode::GlobalAtomicIAdd32 || inst.GetOpcode() == IR::Opcode::StorageAtomicIAdd32; + stores += inst.GetOpcode() == IR::Opcode::WriteGlobal32 || inst.GetOpcode() == IR::Opcode::WriteStorage32; + image_stores += inst.GetOpcode() == IR::Opcode::ImageWrite || inst.GetOpcode() == IR::Opcode::BoundImageWrite; + } + const bool surface = std::any_of(input.begin(), input.end(), [](const auto& instruction) { return instruction.Bits(0, 12) == 0xb9b; }); + if (surface) { + const auto expected_barriers = std::count_if(input.begin(), input.end(), [](const auto& instruction) { + return instruction.Bits(0, 12) == 0xb1d; + }); + REQUIRE(barriers >= expected_barriers); + REQUIRE(image_stores >= 1); + const auto expected_loads = std::count_if(input.begin(), input.end(), [](const auto& instruction) { + return instruction.Bits(0, 12) == 0xb97; + }); + REQUIRE(image_loads >= expected_loads); + } else if (std::any_of(input.begin(), input.end(), [](const auto& instruction) { return instruction.Bits(0, 12) == 0xf8c; })) { + REQUIRE(barriers >= 2); + REQUIRE(shared_atomics >= 4); + REQUIRE(global_atomics >= 2); + } else if (shared_size) { + REQUIRE(barriers >= 5); + REQUIRE(shared_atomics >= 2); + REQUIRE(global_atomics >= 1); + REQUIRE(stores >= 8); + } else { + REQUIRE(barriers == 0); + REQUIRE(stores >= 4); + } +} + +namespace { +double DecodeTestHalf(u16 bits) { + const u32 exponent = (bits >> 10) & 31, mantissa = bits & 1023; + const double value = exponent == 31 ? (mantissa ? std::numeric_limits<double>::quiet_NaN() + : std::numeric_limits<double>::infinity()) + : exponent == 0 ? std::ldexp(static_cast<double>(mantissa), -24) + : std::ldexp(static_cast<double>(1024 + mantissa), static_cast<int>(exponent) - 25); + return (bits & 0x8000) ? -value : value; +} +u16 EncodeTestHalf(double value) { + const u16 sign = std::signbit(value) ? 0x8000 : 0; + value = std::abs(value); + if (std::isnan(value)) return 0xfe01; // Deliberately noncanonical: ReLU must canonicalize it. + if (value >= 65520) return sign | 0x7c00; + if (value < std::ldexp(1.0, -14)) + return sign | static_cast<u16>(std::nearbyint(std::ldexp(value, 24))); + int exponent{}; + std::frexp(value, &exponent); + --exponent; + u32 mantissa = static_cast<u32>(std::nearbyint(std::ldexp(value, 10 - exponent))); + if (mantissa == 2048) { ++exponent; mantissa = 1024; } + return sign | static_cast<u16>(((exponent + 15) << 10) | (mantissa - 1024)); +} +struct ScalarNativeMachine { + std::array<u32, 256> regs{}; + std::array<bool, 7> predicates{}; + std::unordered_map<u64, u32> constants, global, shared; + u32 workgroup_barriers{}, device_barriers{}; + void Execute(IR::Block& block) { + std::unordered_map<const IR::Inst*, u64> values; + bool carry{}, zero{}, sign{}, overflow{}; + const auto get = [&](const IR::Value& value) -> u64 { + if (!value.IsImmediate()) return values.at(value.Inst()); + if (value.Type() == IR::Type::U1) return value.U1(); + if (value.Type() == IR::Type::F16 || value.Type() == IR::Type::U16) return value.U16(); + if (value.Type() == IR::Type::U64) return value.U64(); + if (value.Type() == IR::Type::F32) return std::bit_cast<u32>(value.F32()); + return value.U32(); + }; + for (const auto& inst : block.Instructions()) { + const auto arg = [&](size_t i) { return get(inst.Arg(i)); }; + u64 result{}; + switch (inst.GetOpcode()) { + case IR::Opcode::BitCastF32U32: case IR::Opcode::BitCastU32F32: result = arg(0); break; + case IR::Opcode::UnpackFloat2x16: result = (arg(0) & 0xffff) | ((arg(0) >> 16) << 32); break; + case IR::Opcode::PackFloat2x16: result = (arg(0) & 0xffff) | ((arg(0) >> 32) << 16); break; + case IR::Opcode::CompositeConstructF16x2: result = arg(0) | (arg(1) << 32); break; + case IR::Opcode::CompositeExtractF16x2: result = (arg(0) >> (arg(1) * 32)) & 0xffff; break; + case IR::Opcode::FPNeg16: result = arg(0) ^ 0x8000; break; + case IR::Opcode::FPAbs16: result = arg(0) & 0x7fff; break; + case IR::Opcode::FPFma16: result = EncodeTestHalf(std::fma(DecodeTestHalf(static_cast<u16>(arg(0))), + DecodeTestHalf(static_cast<u16>(arg(1))), DecodeTestHalf(static_cast<u16>(arg(2))))); break; + case IR::Opcode::FPFloor32: result = std::bit_cast<u32>(std::floor(std::bit_cast<float>(static_cast<u32>(arg(0))))); break; + case IR::Opcode::ConvertF32F16: result = std::bit_cast<u32>(static_cast<float>(DecodeTestHalf(static_cast<u16>(arg(0))))); break; + case IR::Opcode::FPOrdEqual32: result = std::bit_cast<float>(static_cast<u32>(arg(0))) == std::bit_cast<float>(static_cast<u32>(arg(1))); break; + case IR::Opcode::FPOrdGreaterThanEqual32: result = std::bit_cast<float>(static_cast<u32>(arg(0))) >= std::bit_cast<float>(static_cast<u32>(arg(1))); break; + case IR::Opcode::FPMax32: result = std::bit_cast<u32>(std::fmax(std::bit_cast<float>(static_cast<u32>(arg(0))), std::bit_cast<float>(static_cast<u32>(arg(1))))); break; + case IR::Opcode::FPMin32: result = std::bit_cast<u32>(std::fmin(std::bit_cast<float>(static_cast<u32>(arg(0))), std::bit_cast<float>(static_cast<u32>(arg(1))))); break; + case IR::Opcode::ConvertU32F32: result = static_cast<u32>(std::bit_cast<float>(static_cast<u32>(arg(0)))); break; + case IR::Opcode::FPTrunc32: result = std::bit_cast<u32>(std::trunc(std::bit_cast<float>(static_cast<u32>(arg(0))))); break; + case IR::Opcode::FPNeg32: result = arg(0) ^ 0x80000000U; break; + case IR::Opcode::FPAbs32: result = arg(0) & 0x7fffffffU; break; + case IR::Opcode::ConvertF16F32: result = EncodeTestHalf(std::bit_cast<float>(static_cast<u32>(arg(0)))); break; + case IR::Opcode::FPRoundEven32: result = std::bit_cast<u32>(std::nearbyint(std::bit_cast<float>(static_cast<u32>(arg(0))))); break; + case IR::Opcode::FPIsNan32: result = std::isnan(std::bit_cast<float>(static_cast<u32>(arg(0)))); break; + case IR::Opcode::FPClamp32: result = std::bit_cast<u32>(std::clamp(std::bit_cast<float>(static_cast<u32>(arg(0))), std::bit_cast<float>(static_cast<u32>(arg(1))), std::bit_cast<float>(static_cast<u32>(arg(2))))); break; + case IR::Opcode::ConvertS32F32: { + const float value = std::bit_cast<float>(static_cast<u32>(arg(0))); + result = std::isnan(value) ? 0x87654321U : static_cast<u32>(static_cast<s32>(value)); + break; + } + case IR::Opcode::GetRegister: result = inst.Arg(0).Reg() == IR::Reg::RZ ? 0 : regs[static_cast<u32>(inst.Arg(0).Reg())]; break; + case IR::Opcode::SetRegister: if (inst.Arg(0).Reg() != IR::Reg::RZ) regs[static_cast<u32>(inst.Arg(0).Reg())] = static_cast<u32>(arg(1)); break; + case IR::Opcode::GetPred: result = predicates[static_cast<u32>(inst.Arg(0).Pred())]; break; + case IR::Opcode::SetPred: predicates[static_cast<u32>(inst.Arg(0).Pred())] = arg(1) != 0; break; + case IR::Opcode::IAdd32: result = static_cast<u32>(arg(0) + arg(1)); break; + case IR::Opcode::ISub32: result = static_cast<u32>(arg(0) - arg(1)); break; + case IR::Opcode::IMul32: result = static_cast<u32>(arg(0) * arg(1)); break; + case IR::Opcode::IAdd64: result = arg(0) + arg(1); break; + case IR::Opcode::UMax32: result = std::max(arg(0), arg(1)); break; + case IR::Opcode::UMin32: result = std::min(arg(0), arg(1)); break; + case IR::Opcode::FindUMsb32: result = arg(0) ? 31U - std::countl_zero(static_cast<u32>(arg(0))) : ~0U; break; + case IR::Opcode::BitReverse32: { + const u32 source = static_cast<u32>(arg(0)); + for (u32 bit = 0; bit < 32; ++bit) result |= u64{(source >> bit) & 1U} << (31 - bit); + break; + } + case IR::Opcode::ShiftLeftLogical64: result = arg(1) < 64 ? arg(0) << arg(1) : 0; break; + case IR::Opcode::ShiftRightLogical64: result = arg(1) < 64 ? arg(0) >> arg(1) : 0; break; + case IR::Opcode::INeg32: result = 0U - static_cast<u32>(arg(0)); break; + case IR::Opcode::ShiftLeftLogical32: result = arg(1) < 32 ? static_cast<u32>(arg(0) << arg(1)) : 0; break; + case IR::Opcode::ShiftRightArithmetic32: result = static_cast<u32>(static_cast<s32>(arg(0)) >> std::min<u64>(arg(1), 31)); break; + case IR::Opcode::ShiftRightLogical32: result = arg(1) < 32 ? static_cast<u32>(arg(0)) >> arg(1) : 0; break; + case IR::Opcode::BitwiseAnd32: result = arg(0) & arg(1); break; + case IR::Opcode::BitwiseOr32: result = arg(0) | arg(1); break; + case IR::Opcode::BitwiseXor32: result = arg(0) ^ arg(1); break; + case IR::Opcode::BitwiseNot32: result = arg(0) ^ 0xffffffffULL; break; + case IR::Opcode::BitFieldUExtract: result = (arg(0) >> arg(1)) & ((u64{1} << arg(2)) - 1); break; + case IR::Opcode::BitFieldInsert: { const u64 mask = ((u64{1} << arg(3)) - 1) << arg(2); result = (arg(0) & ~mask) | ((arg(1) << arg(2)) & mask); break; } + case IR::Opcode::SelectF32: case IR::Opcode::SelectU32: case IR::Opcode::SelectU1: result = arg(0) ? arg(1) : arg(2); break; + case IR::Opcode::LogicalAnd: result = arg(0) && arg(1); break; + case IR::Opcode::LogicalOr: result = arg(0) || arg(1); break; + case IR::Opcode::LogicalNot: result = !arg(0); break; + case IR::Opcode::SMin32: result = static_cast<u32>(std::min(static_cast<s32>(arg(0)), static_cast<s32>(arg(1)))); break; + case IR::Opcode::SMax32: result = static_cast<u32>(std::max(static_cast<s32>(arg(0)), static_cast<s32>(arg(1)))); break; + case IR::Opcode::SLessThan: result = static_cast<s32>(arg(0)) < static_cast<s32>(arg(1)); break; + case IR::Opcode::ULessThan: result = arg(0) < arg(1); break; + case IR::Opcode::IEqual: result = arg(0) == arg(1); break; + case IR::Opcode::GetCFlag: result = carry; break; + case IR::Opcode::SetCFlag: carry = arg(0) != 0; break; + case IR::Opcode::GetZFlag: result = zero; break; + case IR::Opcode::SetZFlag: zero = arg(0) != 0; break; + case IR::Opcode::GetSFlag: result = sign; break; + case IR::Opcode::SetSFlag: sign = arg(0) != 0; break; + case IR::Opcode::GetOFlag: result = overflow; break; + case IR::Opcode::SetOFlag: overflow = arg(0) != 0; break; + case IR::Opcode::GetCarryFromOp: { const auto* sum = inst.Arg(0).Inst(); result = get(sum->Arg(0)) + get(sum->Arg(1)) > 0xffffffffULL; break; } + case IR::Opcode::GetZeroFromOp: result = arg(0) == 0; break; + case IR::Opcode::GetSignFromOp: result = (arg(0) & 0x80000000U) != 0; break; + case IR::Opcode::GetOverflowFromOp: { const auto* sum = inst.Arg(0).Inst(); result = ((get(sum->Arg(0)) ^ arg(0)) & (get(sum->Arg(1)) ^ arg(0)) & 0x80000000U) != 0; break; } + case IR::Opcode::GetCbufU32: result = constants.at((arg(0) << 32) | arg(1)); break; + case IR::Opcode::CompositeConstructU32x2: result = arg(0) | (arg(1) << 32); break; + case IR::Opcode::PackUint2x32: case IR::Opcode::UnpackUint2x32: result = arg(0); break; + case IR::Opcode::CompositeExtractU32x2: result = static_cast<u32>(arg(0) >> (arg(1) * 32)); break; + case IR::Opcode::WriteSharedU64: shared[arg(0)] = static_cast<u32>(arg(1)); shared[arg(0) + 4] = static_cast<u32>(arg(1) >> 32); break; + case IR::Opcode::LoadSharedU64: result = shared.at(arg(0)) | (u64{shared.at(arg(0) + 4)} << 32); break; + case IR::Opcode::WriteGlobal32: global[arg(0)] = static_cast<u32>(arg(1)); break; + case IR::Opcode::WriteSharedU32: shared[arg(0)] = static_cast<u32>(arg(1)); break; + case IR::Opcode::LoadSharedU32: result = shared.at(arg(0)); break; + case IR::Opcode::GlobalAtomicIAdd32: result = global.at(arg(0)); global[arg(0)] += static_cast<u32>(arg(1)); break; + case IR::Opcode::SharedAtomicIAdd32: result = shared.at(arg(0)); shared[arg(0)] += static_cast<u32>(arg(1)); break; + case IR::Opcode::Barrier: ++workgroup_barriers; break; + case IR::Opcode::DeviceMemoryBarrier: ++device_barriers; break; + default: FAIL("Unexpected scalar compute IR " << IR::NameOf(inst.GetOpcode())); + } + values[&inst] = result; + } + } +}; +void TranslateScalarNative(std::span<const Native> input, IR::Block& block) { + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + INFO("offset=" << lowered.error_offset << " error=" << lowered.error); + REQUIRE(lowered); + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (u64 word : lowered.instructions) { + const auto opcode = Shader::Maxwell::Decode(word); + if (opcode == Shader::Maxwell::Opcode::EXIT) continue; + switch (opcode) { +#define INST(name, cute, encode) case Shader::Maxwell::Opcode::name: visitor.name(word); break; +#include "shader_recompiler/frontend/maxwell/maxwell.inc" +#undef INST + } + } +} +} + +TEST_CASE("Native uniform comparisons and selects preserve independent predicate banks", "[native-shader]") { + const std::array<Native, 7> input{{ + {{0x09057891, 0x40, 0x0f8e303f, 0}}, + {{0x0500728c, 7, 0x0bf21070, 0}}, + {{0x05057287, 7, 0x08800000, 0}}, + {{0x05057290, 7, 0x0fffe13f, 0}}, + {{0x0400728c, 6, 0x0bf01070, 0}}, + {{0x04047287, 6, 0x08000000, 0}}, + VertexTransform.back(), + }}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (u32 width : {100U, 256U}) { + ScalarNativeMachine machine; + machine.regs[225] = 2; + machine.regs[223] = width; + machine.regs[220] = 40; + machine.regs[222] = 32; + for (u32 p = 0; p < 7; ++p) machine.predicates[p] = (p & 1) != 0; + const auto original = machine.predicates; + machine.Execute(block); + REQUIRE(machine.regs[221] == width - std::min(192U, width)); + REQUIRE(machine.regs[220] == 32); + REQUIRE(((machine.regs[239] & 0x20) != 0) == (192U < width)); + REQUIRE((machine.regs[239] & 0x40) == 0); + REQUIRE(machine.predicates == original); + } +} + +TEST_CASE("Native global stores carry a register offset into the high address word", "[native-shader]") { + const std::array<Native, 2> input{{{{0x02007986, 9, 0x0810a906, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + ScalarNativeMachine machine; + machine.regs[222] = 0xfffffff0; + machine.regs[223] = 0x21; + machine.regs[2] = 0x30; + machine.regs[9] = 0xdeadbeef; + machine.Execute(block); + REQUIRE(machine.global == std::unordered_map<u64, u32>{{0x2200000020ULL, 0xdeadbeef}}); +} + +TEST_CASE("Native LEA predicate carry reaches the high pointer word", "[native-shader]") { + const std::array<Native, 4> input{{ + {{0x00007a11, 0x800, 0x078010ff, 0}}, + {{0xff017a10, 0x900, 0x007fe4ff, 0}}, + {{0x00007386, 7, 0x0010a900, 0}}, + VertexTransform.back(), + }}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (u32 base : {0x100U, 0xfffffff0U}) { + ScalarNativeMachine machine; + machine.regs[0] = 8; + machine.regs[7] = 0x12345678; + machine.constants[0x20] = base; + machine.constants[0x24] = 0x21; + machine.Execute(block); + REQUIRE(machine.global == std::unordered_map<u64, u32>{{(u64{0x21} << 32) + base + 32, 0x12345678}}); + } +} + +TEST_CASE("Native shared atomics and barriers retain real shared-memory effects", "[native-shader]") { + const std::array<Native, 6> input{{ + {{0x03007388, 0x407, 0x4800, 0}}, + {{0x03097984, 0x400, 0x4800, 0}}, + {{0xff0a738c, 0x7fc09, 0, 0}}, + {{0x00007992, 0, 0x5000, 0}}, + {{0x00007b1d, 0, 0, 0}}, + VertexTransform.back(), + }}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + ScalarNativeMachine machine; + machine.regs[3] = 3; + machine.regs[7] = 5; + machine.shared[0x7fc] = 100; + machine.Execute(block); + REQUIRE(machine.shared.at(16) == 5); + REQUIRE(machine.shared.at(0x7fc) == 105); + REQUIRE(machine.regs[10] == 100); + REQUIRE(machine.workgroup_barriers == 1); + REQUIRE(machine.device_barriers == 1); +} + +TEST_CASE("Native uniform high shifts and multiply operands retain numeric values", "[native-shader]") { + std::array<Native, 7> input{{ + {{0x00057882, 123, 0, 0}}, + {{0x3f047899, 5, 0x08010605, 0}}, + {{0x05057899, 5, 0x0800063f, 0}}, + {{0x01027c24, 5, 0x0f8e0202, 0}}, + {{0x00007c17, 4, 0x0b800200, 0}}, + {{0x000a7424, 4, 0x078e0205, 0}}, + VertexTransform.back(), + }}; + for (u32 value : {0U, 1U, 123U, 0x07ffffffU, 0xffffffffU}) { + input[0].words[1] = value; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + ScalarNativeMachine machine; + machine.regs[0] = 5000; + machine.regs[1] = 11; + machine.regs[2] = 7; + machine.regs[5] = 3; + machine.Execute(block); + const u32 shifted = value << 5; + const u32 minimum = static_cast<u32>(std::min(5000, static_cast<s32>(shifted))); + REQUIRE(machine.regs[220] == shifted); + REQUIRE(machine.regs[221] == shifted); + REQUIRE(machine.regs[2] == 11U * shifted + 7); + REQUIRE(machine.regs[0] == minimum); + REQUIRE(machine.regs[10] == minimum * 3U + 4); + } +} + +TEST_CASE("Native bit reversal and register shifts preserve FFT indexing", "[native-shader]") { + const std::array<Native, 6> input{{ + {{0x00037301, 19, 0, 0}}, + {{0x03017211, 255, 0x078f50ff, 0}}, + {{0xff087219, 4, 0x00011613, 0}}, + {{0x0f0f7219, 12, 0x00000eff, 0}}, + {{0xff047224, 255, 0x078e0a0c, 0}}, + VertexTransform.back(), + }}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (u32 shift : {0U, 1U, 10U, 31U, 32U, 33U, 63U}) { + ScalarNativeMachine machine; + machine.regs[19] = 0x12345678; + machine.regs[4] = shift; + machine.regs[12] = shift; + machine.regs[15] = 0x9abcdef0; + machine.Execute(block); + REQUIRE(machine.regs[3] == 0x1e6a2c48); + REQUIRE(machine.regs[1] == (0x1e6a2c48U >> 22)); + REQUIRE(machine.regs[8] == (shift < 32 ? 0x12345678U >> shift : 0)); + REQUIRE(machine.regs[15] == (0x9abcdef0U << (shift & 31))); + REQUIRE(machine.regs[4] == 0U - shift); + } +} + +TEST_CASE("Native paired shared values keep both words under scaled addressing", "[native-shader]") { + const std::array<Native, 3> input{{ + {{0x13007388, 4, 0x8a00, 0}}, + {{0x13027984, 0, 0x8a00, 0}}, + VertexTransform.back(), + }}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + ScalarNativeMachine machine; + machine.regs[19] = 17; + machine.regs[4] = 0x12345678; + machine.regs[5] = 0x9abcdef0; + machine.Execute(block); + REQUIRE(machine.shared.at(17 * 8) == 0x12345678); + REQUIRE(machine.shared.at(17 * 8 + 4) == 0x9abcdef0); + REQUIRE(machine.regs[2] == 0x12345678); + REQUIRE(machine.regs[3] == 0x9abcdef0); +} + +TEST_CASE("Native formatted surface stores retain coordinates channels and descriptor", "[native-shader]") { + const std::array<Native, 2> input{{{{0x00007b9b, 0x60000a04, 0x00100f00, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + u32 writes{}; + for (const auto& inst : block.Instructions()) if (inst.GetOpcode() == IR::Opcode::BoundImageWrite) { + REQUIRE(inst.Arg(0).U32() == 40); + const auto info = inst.Flags<IR::TextureInstInfo>(); + REQUIRE(info.type.Value() == Shader::TextureType::Color2D); + const auto* coordinates = inst.Arg(1).Inst(); + REQUIRE(coordinates->Arg(0).Inst()->Arg(0).Reg() == IR::Reg::R0); + REQUIRE(coordinates->Arg(1).Inst()->Arg(0).Reg() == IR::Reg::R1); + const auto* color = inst.Arg(2).Inst(); + for (u32 component = 0; component < 4; ++component) + REQUIRE(static_cast<u32>(color->Arg(component).Inst()->Arg(0).Reg()) == 4 + component); + ++writes; + } + REQUIRE(writes == 1); +} + +TEST_CASE("Native predicated predicate copies retain their guard and source", "[native-shader]") { + const std::array<Native, 2> input{{{{0x0000281c, 0, 0x01f0f070, 0}}, VertexTransform.back()}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + const auto predicate = Shader::Maxwell::Instruction{lowered.instructions.front()}.Pred(); + REQUIRE(predicate.index == 2); + REQUIRE_FALSE(predicate.negated); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (bool source : {false, true}) { + ScalarNativeMachine machine; + machine.predicates[3] = source; + machine.Execute(block); + REQUIRE(machine.predicates[0] == source); + } +} + +TEST_CASE("Native NTZ nearest conversion rounds ties to even and clears NaNs", "[native-shader]") { + const std::array<Native, 2> input{{{{0x00117305, 2, 0x00203100, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + u32 rounds{}; + for (const auto& inst : block.Instructions()) if (inst.GetOpcode() == IR::Opcode::FPRoundEven32) { + REQUIRE(inst.Flags<IR::FpControl>().fmz_mode == IR::FmzMode::None); + ++rounds; + } + REQUIRE(rounds == 1); + for (const auto [value, expected] : std::array<std::pair<float, s32>, 9>{{ + {0.5f, 0}, {1.5f, 2}, {2.5f, 2}, {-0.5f, 0}, {-1.5f, -2}, {-2.5f, -2}, + {0.9f, 1}, {-0.9f, -1}, {std::numeric_limits<float>::quiet_NaN(), 0}, + }}) { + ScalarNativeMachine machine; + machine.regs[2] = std::bit_cast<u32>(value); + machine.Execute(block); + REQUIRE(machine.regs[17] == static_cast<u32>(expected)); + } + auto unsupported = input; + unsupported[0].words[2] ^= 1U << 13; + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(unsupported)); +} + +TEST_CASE("Native predicate packing merges only selected predicate bits", "[native-shader]") { + for (u32 mask : {0U, 8U, 16U, 0x55U, 0x7fU}) { + const std::array<Native, 2> input{{{{0x02137803, mask, 0, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (u32 bits = 0; bits < 128; ++bits) { + ScalarNativeMachine machine; + machine.regs[2] = 0xa5a5a5a5U; + for (u32 i = 0; i < 7; ++i) machine.predicates[i] = (bits >> i) & 1; + const auto saved = machine.predicates; + machine.Execute(block); + REQUIRE(machine.regs[19] == ((0xa5a5a5a5U & ~mask) | (bits & mask))); + REQUIRE(machine.predicates == saved); + } + } +} + +TEST_CASE("Native signed NTZ floor conversion rounds negative fractions down", "[native-shader]") { + const std::array<Native, 2> input{{{{0x00237305, 0x24, 0x00207100, 0x000e2400}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (const auto [value, expected] : std::array<std::pair<float, s32>, 8>{{ + {1.9f, 1}, {-1.1f, -2}, {-0.1f, -1}, {0.9f, 0}, + {0.0f, 0}, {-2.0f, -2}, {12345.5f, 12345}, + {std::numeric_limits<float>::quiet_NaN(), 0}, + }}) { + ScalarNativeMachine machine; + machine.regs[36] = std::bit_cast<u32>(value); + machine.Execute(block); + REQUIRE(machine.regs[35] == static_cast<u32>(expected)); + } +} + +TEST_CASE("Native dependency waits retain the full six-bit pending-operation count", "[native-shader]") { + for (u32 count : {0U, 1U, 15U, 22U, 63U}) { + const std::array<Native, 2> input{{{{0x0000791a, 0xd000U | (count << 6), 0, 0}}, VertexTransform.back()}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + REQUIRE(lowered.instructions.size() == 1); + } + const std::array<Native, 2> unsupported{{{{0x0000791a, 0xc000, 0, 0}}, VertexTransform.back()}}; + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(unsupported)); +} + +TEST_CASE("Native histogram warp match compares both address words and preserves predicates", "[native-shader]") { + const std::array<Native, 2> input{{{{0x00ff73a1, 0, 0x00000200, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (u32 active : {1U, 0x80000000U, 0x80000001U, 0x55555555U, 0xffffffffU}) { + for (u32 mismatch : {0U, 1U, 2U}) { + std::array<std::array<u32, 256>, 32> registers{}; + std::array<std::array<bool, 7>, 32> predicates{}; + const u32 first = std::countr_zero(active); + for (u32 lane = 0; lane < 32; ++lane) { + registers[lane][0] = 0xfffffff0U + (mismatch == 1 && lane != first ? 4 : 0); + registers[lane][1] = 0x21U + (mismatch == 2 && lane != first ? 1 : 0); + for (u32 p = 0; p < 7; ++p) predicates[lane][p] = ((lane + p) & 1) != 0; + } + const auto original = predicates; + ExecuteNativeWarp(block, active, registers, predicates); + for (u32 lane = 0; lane < 32; ++lane) { + auto expected = original[lane]; + if (active & (1U << lane)) expected[0] = mismatch == 0 || std::popcount(active) == 1; + REQUIRE(predicates[lane] == expected); + } + } + } +} + +TEST_CASE("Native histogram reduction sums only active lanes with unsigned wrap", "[native-shader]") { + const std::array<Native, 2> input{{{{0x030573c4, 0, 0x0000c000, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (u32 active : {1U, 0x80000000U, 0x80000001U, 0x55555555U, 0xffffffffU}) { + std::array<std::array<u32, 256>, 32> registers{}; + std::array<std::array<bool, 7>, 32> predicates{}; + u32 sum = 0; + for (u32 lane = 0; lane < 32; ++lane) { + registers[lane][3] = 0xfffffff0U + lane * 17; + registers[lane][221] = 0xcafebeef; + if (active & (1U << lane)) sum += registers[lane][3]; + for (u32 p = 0; p < 7; ++p) predicates[lane][p] = ((lane + p) & 1) != 0; + } + const auto original = predicates; + ExecuteNativeWarp(block, active, registers, predicates); + REQUIRE(predicates == original); + for (u32 lane = 0; lane < 32; ++lane) + REQUIRE(registers[lane][221] == ((active & (1U << lane)) ? sum : 0xcafebeef)); + } +} + +TEST_CASE("Native histogram pointer carry and global reduction retain the address and addend", "[native-shader]") { + const std::array<Native, 4> input{{ + {{0x00007a10, 0x00001000, 0x07f1e0ff, 0}}, + {{0xff017624, 0x00001100, 0x000e06ff, 0}}, + {{0x0000798e, 3, 0x0010e100, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (u32 low : {0U, 0xfffffff0U, 0xffffffffU}) { + ScalarNativeMachine machine; + machine.regs[0] = 32; + machine.regs[3] = 17; + machine.constants[0x40] = low; + machine.constants[0x44] = 0x21; + const u64 address = 0x2100000000ULL + low + 32; + machine.global[address] = 0xfffffff8U; + machine.Execute(block); + REQUIRE(machine.regs[0] == static_cast<u32>(address)); + REQUIRE(machine.regs[1] == address >> 32); + REQUIRE(machine.predicates[0] == (low > 0xffffffdfU)); + REQUIRE(machine.global == std::unordered_map<u64, u32>{{address, 9}}); + } +} + +TEST_CASE("Native histogram shared increment scales its address and increments once", "[native-shader]") { + const std::array<Native, 2> input{{{{0x03ff7f8c, 0, 0x0d00403f, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + ScalarNativeMachine machine; + machine.regs[3] = 63; + machine.shared[252] = 0xffffffffU; + machine.Execute(block); + REQUIRE(machine.shared == std::unordered_map<u64, u32>{{252, 0}}); +} + +TEST_CASE("Native histogram uniform leading bit and comparison select the highest active lane", "[native-shader]") { + const std::array<Native, 3> input{{{{0x000472bd, 4, 0x080e0000, 0}}, + {{0x02007c0c, 4, 0x0bf02070, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (u32 mask : {1U, 0x80000000U, 0x55555555U, 0xffffffffU}) { + for (u32 lane = 0; lane < 32; ++lane) { + ScalarNativeMachine machine; + machine.regs[220] = mask; + machine.regs[2] = lane; + machine.Execute(block); + REQUIRE(machine.regs[220] == 31U - std::countl_zero(mask)); + REQUIRE(machine.predicates[0] == (lane == 31U - std::countl_zero(mask))); + } + } +} + +TEST_CASE("Native formatted surface loads retain red data coordinates descriptor and guard", "[native-shader]") { + const Native instruction = GENERATE(Native{{0x400a0b97, 0x60001200, 0x001e0100, 0}}, + Native{{0x12250b97, 0x60001200, 0x001e0100, 0}}); + const std::array<Native, 2> input{{instruction, VertexTransform.back()}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + REQUIRE(lowered.required_texture_buffer == 0); + const auto guard = Shader::Maxwell::Instruction{lowered.instructions.front()}.Pred(); + REQUIRE(guard.index == 0); + REQUIRE_FALSE(guard.negated); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + u32 loads{}, writes{}; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::BoundImageRead) { + REQUIRE(inst.Arg(0).U32() == 0x12 * 4); + REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::Color2D); + const auto* coords = inst.Arg(1).Inst(); + REQUIRE(static_cast<u32>(coords->Arg(0).Inst()->Arg(0).Reg()) == instruction.Bits(24, 8)); + REQUIRE(static_cast<u32>(coords->Arg(1).Inst()->Arg(0).Reg()) == instruction.Bits(24, 8) + 1); + ++loads; + } + if (inst.GetOpcode() == IR::Opcode::SetRegister) { + REQUIRE(static_cast<u32>(inst.Arg(0).Reg()) == instruction.Bits(16, 8)); + const auto* red = inst.Arg(1).Inst(); + REQUIRE(red->GetOpcode() == IR::Opcode::CompositeExtractU32x4); + REQUIRE(red->Arg(1).U32() == 0); + REQUIRE(red->Arg(0).Inst()->GetOpcode() == IR::Opcode::BoundImageRead); + ++writes; + } + } + REQUIRE(loads == 1); + REQUIRE(writes == 1); + auto invalid = input; + invalid[0].words[2] ^= 0x200; // Reject other channel masks until implemented. + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(invalid)); +} + +TEST_CASE("Native volume texel fetch preserves XYZ LOD descriptor and split RGBA outputs", "[native-shader]") { + const std::array<Native, 2> input{{{{0x00087b66, 0x50000a02, 0x049e0f0a, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + u32 fetches{}; + std::vector<u32> writes; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::BoundImageFetch) { + REQUIRE(inst.Arg(0).U32() == 40); + REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::Color3D); + const auto* coords = inst.Arg(1).Inst(); + for (u32 i = 0; i < 3; ++i) + REQUIRE(static_cast<u32>(coords->Arg(i).Inst()->Arg(0).Reg()) == i); + REQUIRE(inst.Arg(3).U32() == 0); + ++fetches; + } + if (inst.GetOpcode() == IR::Opcode::SetRegister) { + writes.push_back(static_cast<u32>(inst.Arg(0).Reg())); + const auto* extract = inst.Arg(1).Inst()->Arg(0).Inst(); + REQUIRE(extract->GetOpcode() == IR::Opcode::CompositeExtractF32x4); + REQUIRE(extract->Arg(1).U32() == writes.size() - 1); + } + } + REQUIRE(fetches == 1); + REQUIRE(writes == std::vector<u32>{8, 9, 10, 11}); + auto invalid = input; + invalid[0].words[0] |= 1U << 24; + REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(invalid)); +} + +TEST_CASE("Native volume surface store retains XYZ and all four data channels", "[native-shader]") { + const std::array<Native, 2> input{{{{0x00007b9b, 0xa0000c04, 0x00100f00, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + u32 stores{}; + for (const auto& inst : block.Instructions()) if (inst.GetOpcode() == IR::Opcode::BoundImageWrite) { + REQUIRE(inst.Arg(0).U32() == 48); + REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::Color3D); + for (u32 i = 0; i < 3; ++i) + REQUIRE(static_cast<u32>(inst.Arg(1).Inst()->Arg(i).Inst()->Arg(0).Reg()) == i); + for (u32 i = 0; i < 4; ++i) + REQUIRE(static_cast<u32>(inst.Arg(2).Inst()->Arg(i).Inst()->Arg(0).Reg()) == 4 + i); + ++stores; + } + REQUIRE(stores == 1); +} + +TEST_CASE("Native half FMA ReLU preserves fusion values above one and canonical NaNs", "[native-shader]") { + const std::array<Native, 2> input{{{{0x0d127231, 0x3000000d, 0x03848d10, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + const std::array<std::array<u16, 3>, 9> cases{{ + {{0x3c00, 0x4400, 0x4200}}, // -1*1+4 = 3, not saturated to one + {{0x4000, 0x3c00, 0x0000}}, // -2*2+1 < 0 + {{0x3c00, 0x3c00, 0x0000}}, + {{0x0000, 0x0001, 0x0001}}, // smallest positive subnormal survives + {{0x7c00, 0x0000, 0x0000}}, + {{0x0000, 0x7c00, 0x7c00}}, + {{0x7e01, 0x3c00, 0x7fff}}, + {{0xfe01, 0x3c00, 0x7fff}}, + {{0x8000, 0x8000, 0x0000}}, + }}; + for (const auto& values : cases) { + ScalarNativeMachine machine; + machine.regs[13] = (u32{values[0]} << 16) | 0x4400; + machine.regs[16] = 0xc0000000 | values[1]; + machine.predicates = {true, false, true, false, true, false, true}; + const auto predicates = machine.predicates; + machine.Execute(block); + REQUIRE(machine.regs[18] == (u32{values[2]} | (u32{values[2]} << 16))); + REQUIRE(machine.predicates == predicates); + } + // A separate register operand permits a fused product residual that would + // vanish if the multiplication rounded before adding its third operand. + auto fused = input; + fused[0].words[1] = 0x3000000e; + IR::Block fused_block{pool}; + TranslateScalarNative(fused, fused_block); + ScalarNativeMachine machine; + machine.regs[13] = 0x3c013c01; + machine.regs[14] = 0x3bfe3bfe; + machine.regs[16] = 0x3c003c00; + machine.Execute(fused_block); + REQUIRE(machine.regs[18] == 0x00100010); // 2^-20 in each half +} + +TEST_CASE("Native half FMA ReLU accepts packed immediates without losing input aliases", "[native-shader]") { + const std::array<Native, 2> input{{{{0x02027831, 0x3c003c00, 0x03968805, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (const auto [source, addend, expected] : std::array<std::array<u16, 3>, 4>{{ + {{0x4000, 0x3c00, 0x3c00}}, {{0x3c00, 0x4400, 0x0000}}, + {{0x4400, 0x3c00, 0x4200}}, {{0x7e01, 0x3c00, 0x7fff}}, + }}) { + ScalarNativeMachine machine; + machine.regs[2] = source | 0xc4000000U; + machine.regs[5] = (u32{addend} << 16) | 0x4800; + machine.Execute(block); + REQUIRE(machine.regs[2] == (u32{expected} | (u32{expected} << 16))); + } +} + +TEST_CASE("Native quad butterfly and swizzled add retain horizontal and vertical differences", "[native-shader]") { + const bool vertical = GENERATE(false, true); + const std::array<Native, 3> input{{ + {{0x05107f89, vertical ? 0x0c5c0300U : 0x0c3c0300U, 0x000e0000, 0}}, + {{0x100c7822, vertical ? 0x5aU : 0x66U, 5, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + std::array<std::array<u32, 256>, 32> registers{}; + std::array<std::array<bool, 7>, 32> predicates{}; + for (u32 lane = 0; lane < 32; ++lane) + registers[lane][5] = std::bit_cast<u32>(static_cast<float>(lane / 4 * 100 + (lane & 1) * 3 + ((lane >> 1) & 1) * 11)); + ExecuteNativeWarp(block, 0xffffffffU, registers, predicates); + for (u32 lane = 0; lane < 32; ++lane) { + REQUIRE(registers[lane][16] == registers[lane ^ (vertical ? 2 : 1)][5]); + REQUIRE(std::bit_cast<float>(registers[lane][12]) == (vertical ? 11.0f : 3.0f)); + } +} + +TEST_CASE("Native array gradient fetch retains array index XY derivatives descriptor and RGBA", "[native-shader]") { + const std::array<Native, 2> input{{{{0x04047b6c, 0xa080080c, 0x001e0f06, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + u32 samples{}; + std::vector<u32> writes; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::BoundImageGradient) { + REQUIRE(inst.Arg(0).U32() == 32); + REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::ColorArray2D); + REQUIRE(inst.Flags<IR::TextureInstInfo>().num_derivatives.Value() == 2); + const auto* coords = inst.Arg(1).Inst(); + REQUIRE(coords->Arg(0).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{4}); + REQUIRE(coords->Arg(1).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{5}); + const auto* layer = coords->Arg(2).Inst()->Arg(0).Inst(); + REQUIRE(layer->GetOpcode() == IR::Opcode::BitFieldUExtract); + REQUIRE(layer->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{6}); + REQUIRE(layer->Arg(2).U32() == 16); + for (u32 component = 0; component < 4; ++component) + REQUIRE(inst.Arg(2).Inst()->Arg(component).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{12 + component}); + ++samples; + } + if (inst.GetOpcode() == IR::Opcode::SetRegister) writes.push_back(static_cast<u32>(inst.Arg(0).Reg())); + } + REQUIRE(samples == 1); + REQUIRE(writes == std::vector<u32>{4, 5, 6, 7}); +} + +TEST_CASE("Native capture distinguishes forward branches from call targets beyond main EXIT", "[native-shader]") { + using Capture = Shader::Ampere::ProgramCaptureState; + Capture calls; + REQUIRE(calls.Observe(Native{{0x7944, 48, 0x03c00000, 0}}, 1) == Capture::Result::Continue); + REQUIRE(calls.Observe(VertexTransform.back(), 2) == Capture::Result::Continue); + REQUIRE(calls.Observe(Native{{0x08007950, 0, 0, 0}}, 6) == Capture::Result::Complete); + Capture branches; + REQUIRE(branches.Observe(Native{{0x7947, 48, 0x03800000, 0}}, 0) == Capture::Result::Continue); + REQUIRE(branches.Observe(VertexTransform.back(), 1) == Capture::Result::Continue); + REQUIRE(branches.Observe(VertexTransform.back(), 4) == Capture::Result::Complete); + Capture multiple; + REQUIRE(multiple.Observe(Native{{0x7944, 96, 0x03c00000, 0}}, 1) == Capture::Result::Continue); + REQUIRE(multiple.Observe(Native{{0x7944, 128, 0x03c00000, 0}}, 3) == Capture::Result::Continue); + REQUIRE(multiple.Observe(VertexTransform.back(), 4) == Capture::Result::Continue); + REQUIRE(multiple.Observe(Native{{0x08007950, 0, 0, 0}}, 9) == Capture::Result::Continue); + REQUIRE(multiple.Observe(Native{{0x08007950, 0, 0, 0}}, 13) == Capture::Result::Complete); + Capture loop_exit; + REQUIRE(loop_exit.Observe(Native{{0x0944, 16, 0x03c00000, 0}}, 0) == Capture::Result::Continue); + REQUIRE(loop_exit.Observe(Native{{0x7947, 0xfffffff0, 0x0383ffff, 0}}, 1) == Capture::Result::Continue); + REQUIRE(loop_exit.Observe(Native{{0x7941, 0, 0x03800000, 0}}, 2) == Capture::Result::Continue); + REQUIRE(loop_exit.Observe(VertexTransform.back(), 3) == Capture::Result::Complete); + Capture invalid; + REQUIRE(invalid.Observe(Native{{0x7947, 1, 0x03800000, 0}}, 0) == Capture::Result::InvalidTarget); +} + +TEST_CASE("Native implicit volume sample preserves XYZ and all four split channels", "[native-shader]") { + const std::array<Native, 2> input{{{{0x0a0a7b60, 0x5080400e, 0x041e0f0c, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + u32 samples{}; + std::vector<std::pair<u32, u32>> copies; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::BoundImageSampleImplicitLod) { + REQUIRE(inst.Arg(0).U32() == 256); + REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::Color3D); + for (u32 i = 0; i < 3; ++i) + REQUIRE(inst.Arg(1).Inst()->Arg(i).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{250 + i}); + ++samples; + } + if (inst.GetOpcode() == IR::Opcode::SetRegister && inst.Arg(1).Inst()->GetOpcode() == IR::Opcode::GetRegister) + copies.emplace_back(static_cast<u32>(inst.Arg(0).Reg()), static_cast<u32>(inst.Arg(1).Inst()->Arg(0).Reg())); + } + REQUIRE(samples == 1); + REQUIRE(copies == std::vector<std::pair<u32, u32>>{{250, 10}, {251, 11}, {252, 14}, {10, 250}, {11, 251}, {12, 252}, {13, 253}}); +} + +TEST_CASE("Native array shadow sampling retains layer XY depth reference and scalar destination", "[native-shader]") { + const std::array<Native, 2> input{{{{0x082e7b60, 0xb080140a, 0x001e41ff, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + u32 samples{}; + std::vector<std::pair<u32, u32>> copies; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::BoundImageSampleDrefImplicitLod) { + REQUIRE(inst.Arg(0).U32() == 80); + REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::ColorArray2D); + REQUIRE(inst.Flags<IR::TextureInstInfo>().is_depth.Value() == 1); + REQUIRE(inst.Arg(1).Inst()->Arg(0).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{251}); + REQUIRE(inst.Arg(1).Inst()->Arg(1).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{252}); + REQUIRE(inst.Arg(2).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{11}); + ++samples; + } + if (inst.GetOpcode() == IR::Opcode::SetRegister && inst.Arg(1).Inst()->GetOpcode() == IR::Opcode::GetRegister) + copies.emplace_back(static_cast<u32>(inst.Arg(0).Reg()), static_cast<u32>(inst.Arg(1).Inst()->Arg(0).Reg())); + } + REQUIRE(samples == 1); + REQUIRE(copies == std::vector<std::pair<u32, u32>>{{250, 8}, {251, 9}, {252, 10}}); + REQUIRE(block.Instructions().back().Arg(0).Reg() == IR::Reg{46}); +} + +TEST_CASE("Native immediate uniform comparison and select preserve predicates at unsigned boundaries", "[native-shader]") { + const std::array<Native, 3> input{{{{0x0400788c, 6, 0x0bf01070, 0}}, {{0x04057887, 6, 0x08000000, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (u32 value : {0U, 5U, 6U, 7U, 0x80000000U, 0xffffffffU}) { + ScalarNativeMachine machine; + machine.regs[220] = value; + for (u32 i = 0; i < 7; ++i) machine.predicates[i] = (i & 1) != 0; + const auto saved = machine.predicates; + machine.Execute(block); + REQUIRE(((machine.regs[239] >> 6) & 1) == (value < 6)); + REQUIRE(machine.regs[221] == std::min(value, 6U)); + REQUIRE(machine.predicates == saved); + } +} + +TEST_CASE("Native uniform signed right shift sign extends the high word", "[native-shader]") { + const std::array<Native, 2> input{{{{0x3f057899, 31, 0x08011404, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (u32 value : {0U, 1U, 0x7fffffffU, 0x80000000U, 0xffffffffU}) { + ScalarNativeMachine machine; + machine.regs[220] = value; + machine.Execute(block); + REQUIRE(machine.regs[221] == ((value & 0x80000000U) ? 0xffffffffU : 0U)); + } +} + +TEST_CASE("Native indexed constant load adds a signed byte displacement", "[native-shader]") { + const std::array<Native, 2> input{{{{0x30077b82, 0x00fe0500, 0x00000800, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + ScalarNativeMachine machine; + machine.regs[48] = 0x800; + machine.constants[(u64{3} << 32) | 0x14] = 0x12345678; + machine.Execute(block); + REQUIRE(machine.regs[7] == 0x12345678); +} + +TEST_CASE("Native local spill retains byte address data and execution predicate", "[native-shader]") { + const std::array<Native, 3> input{{{{0xff001387, 85, 0x00100800, 0}}, + {{0xff2a7983, 0, 0x00300800, 0}}, VertexTransform.back()}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + REQUIRE(Shader::Maxwell::Decode(lowered.instructions[0]) == Shader::Maxwell::Opcode::STL); + REQUIRE(Shader::Maxwell::Instruction{lowered.instructions[0]}.Pred().index == 1); + REQUIRE((lowered.instructions[0] & 255) == 85); + REQUIRE((lowered.instructions[1] & 255) == 42); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + class LocalEnvironment : public EmptyEnvironment { + public: + u32 LocalMemorySize() const override { return 4; } + } env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.STL(lowered.instructions[0]); + visitor.LDL(lowered.instructions[1]); + u32 loads{}, stores{}; + for (const auto& inst : block.Instructions()) { + if (inst.GetOpcode() == IR::Opcode::LoadLocal) { + REQUIRE(inst.Arg(0).U32() == 0); + ++loads; + } + if (inst.GetOpcode() == IR::Opcode::WriteLocal) { + REQUIRE(inst.Arg(0).U32() == 0); + REQUIRE(inst.Arg(1).Inst()->Arg(0).Reg() == IR::Reg{85}); + ++stores; + } + } + REQUIRE(loads == 1); + REQUIRE(stores == 1); +} + +TEST_CASE("Native scalar half conversion applies absolute value before rounding", "[native-shader]") { + const bool negate = GENERATE(false, true); + const std::array<Native, 2> input{{{{0x00047304, negate ? 0x80000000U : 0x40000000U, 0x00200800, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (const auto [value, expected] : std::array<std::pair<float, u32>, 7>{{ + {-1.0f, 0x3c00}, {-0.0f, 0}, {-2.5f, 0x4100}, + {1.00048828125f, 0x3c00}, {1.00146484375f, 0x3c02}, + {-0x1p-24f, 1}, {-std::numeric_limits<float>::infinity(), 0x7c00}, + }}) { + ScalarNativeMachine machine; + machine.regs[0] = std::bit_cast<u32>(value); + machine.Execute(block); + REQUIRE(machine.regs[4] == (expected | (negate && !std::signbit(value) ? 0x8000U : 0U))); + } +} + +TEST_CASE("Native predicated half ReLU guards its complete expansion including predicate scratch", "[native-shader]") { + for (u32 pred : {1U, 6U, 9U, 14U}) { + const std::array<Native, 2> input{{{{0x12140231U | (pred << 12), 0x20000012, 0x03848914, 0}}, VertexTransform.back()}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + REQUIRE(Shader::Maxwell::Decode(lowered.instructions.front()) == Shader::Maxwell::Opcode::BRA); + REQUIRE(((lowered.instructions.front() >> 16) & 15) == (pred ^ 8)); + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + EmptyEnvironment env; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + for (size_t i = 1; i + 1 < lowered.instructions.size(); ++i) { + const u64 word = lowered.instructions[i]; + REQUIRE(((word >> 16) & 15) == 7); + switch (Shader::Maxwell::Decode(word)) { +#define INST(name, cute, encode) case Shader::Maxwell::Opcode::name: visitor.name(word); break; +#include "shader_recompiler/frontend/maxwell/maxwell.inc" +#undef INST + } + } + for (bool value : {false, true}) { + ScalarNativeMachine machine; + machine.regs[18] = 0x3c003c00; + machine.regs[20] = 0x44004400; + machine.predicates.fill(true); + machine.predicates[pred & 7] = value; + const auto saved = machine.predicates; + if (value != ((pred & 8) != 0)) machine.Execute(block); + REQUIRE(machine.regs[20] == (value != ((pred & 8) != 0) ? 0x42004200U : 0x44004400U)); + REQUIRE(machine.predicates == saved); + } + } +} + +TEST_CASE("Native unsigned half conversion truncates and saturates NTZ edge cases", "[native-shader]") { + const std::array<Native, 2> input{{{{0x000e7305, 13, 0x0010f000, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (const auto [value, expected] : std::array<std::pair<u32, u32>, 8>{{ + {0x3e00, 1}, {0x4100, 2}, {0xbe00, 0}, {0x0001, 0}, + {0x7bff, 65504}, {0x7c00, 0xffffffffU}, {0xfc00, 0}, {0x7e00, 0}, + }}) { + ScalarNativeMachine machine; + machine.regs[13] = 0xdead0000 | value; + machine.Execute(block); + REQUIRE(machine.regs[14] == expected); + } +} + +TEST_CASE("Native predicate LUT OR combines three inputs while preserving other predicates", "[native-shader]") { + const std::array<Native, 3> input{{{{0x0000781c, 0, 0x01f25f16, 0}}, {{0x0000781c, 0, 0x00703570, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + for (u32 bits = 0; bits < 128; ++bits) { + ScalarNativeMachine machine; + for (u32 i = 0; i < 7; ++i) machine.predicates[i] = (bits >> i) & 1; + auto expected = machine.predicates; + expected[1] = expected[1] || expected[2] || expected[3]; + expected[0] = expected[0] || expected[1]; + machine.Execute(block); + REQUIRE(machine.predicates == expected); + } +} + +TEST_CASE("Native half FMA absolute source clears each half sign before broadcast", "[native-shader]") { + for (bool broadcast : {false, true}) { + const std::array<Native, 2> input{{{{0x1a087231, 0x20000008, + broadcast ? 0x00040a0dU : 0x0004020dU, 0}}, VertexTransform.back()}}; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + TranslateScalarNative(input, block); + ScalarNativeMachine machine; + machine.regs[26] = 0xc400c000; + machine.regs[8] = 0x40003c00; + machine.regs[13] = 0x44004200; + machine.Execute(block); + REQUIRE(machine.regs[8] == (broadcast ? 0x45004500U : 0x47004500U)); + } +} + +TEST_CASE("Native constant interpolation retains attribute and flat input behavior", "[native-shader]") { + const std::array<Native, 2> input{{{{0x00047326, 255, 0x000e403c, 0}}, VertexTransform.back()}}; + const auto lowered = Shader::Ampere::LowerToMaxwell(input); + REQUIRE(lowered); + REQUIRE(((lowered.instructions[0] >> 54) & 3) == 2); + class FlatEnvironment : public EmptyEnvironment { + public: + FlatEnvironment() { sph.ps.imap_generic_vector[7].raw = 0x55; } + } env; + Shader::ObjectPool<IR::Inst> pool; + IR::Block block{pool}; + Shader::Maxwell::TranslatorVisitor visitor{env, block}; + visitor.IPA(lowered.instructions[0]); + u32 reads{}; + for (const auto& inst : block.Instructions()) { + REQUIRE(inst.GetOpcode() != IR::Opcode::FPMul32); + if (inst.GetOpcode() == IR::Opcode::GetAttribute) { + REQUIRE(inst.Arg(0).Attribute() == IR::Attribute{60}); + ++reads; + } + } + REQUIRE(reads == 1); +} diff --git a/src/tests/video_core/native_swizzle.cpp b/src/tests/video_core/native_swizzle.cpp new file mode 100644 index 0000000000..93c4b85dac --- /dev/null +++ b/src/tests/video_core/native_swizzle.cpp @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <algorithm> +#include <array> +#include <bit> +#include <vector> +#include <catch2/catch_test_macros.hpp> +#include "video_core/textures/decoders.h" + +namespace { +u64 Digest(const std::vector<u8>& bytes) { + u64 digest = 14695981039346656037ULL; + for (const u8 byte : bytes) digest = (digest ^ byte) * 1099511628211ULL; + return digest; +} +} + +TEST_CASE("Native Orin texture mips match independent upstream swizzle vectors", "[native-swizzle]") { + struct Fixture { u32 width, height, levels, bytes; std::array<u64, 4> hashes; }; + const std::array fixtures{ + Fixture{32, 64, 2, 8, {0x9fcd630e81008de4ULL, 0x0b316aa26800970aULL}}, + Fixture{128, 256, 4, 16, {0x1d16743f55444bd6ULL, 0x628ec170d3dc24b6ULL, 0x9d5ce8671743eadcULL, 0x68c86243a7167475ULL}}, + Fixture{128, 128, 3, 8, {0xa3694bb8d6333e7eULL, 0xe748d5f21bd06d5dULL, 0xabfa06266de4d322ULL}}, + Fixture{256, 64, 2, 16, {0x9820bfdf5000d898ULL, 0x2177c49e446a9450ULL}}, + Fixture{64, 256, 4, 16, {0x30e15cf8e750e118ULL, 0x286e806b76ac89e4ULL, 0xa0dc28aebfbe0862ULL, 0xdca4a8b9b78cafc2ULL}}, + }; + for (const auto& fixture : fixtures) { + u32 state = 0x12345678; + for (u32 level = 0; level < fixture.levels; ++level) { + const u32 width = (std::max(1U, fixture.width >> level) + 3) / 4; + const u32 height = (std::max(1U, fixture.height >> level) + 3) / 4; + u32 block_height = 16; + while (block_height > 1 && height <= block_height * 4) block_height >>= 1; + const u32 block_log2 = static_cast<u32>(std::countr_zero(block_height)); + const size_t size = Tegra::Texture::CalculateSize(true, fixture.bytes, width, height, 1, block_log2, 0); + std::vector<u8> tiled(size); + for (auto& byte : tiled) { state = state * 1664525U + 1013904223U; byte = static_cast<u8>(state >> 24); } + std::vector<u8> linear(width * height * fixture.bytes); + Tegra::Texture::UnswizzleTexture(linear, tiled, fixture.bytes, width, height, 1, block_log2, 0, 1, true); + INFO("width=" << width << " height=" << height << " mip=" << level); + REQUIRE(Digest(linear) == fixture.hashes[level]); + auto restored = tiled; + Tegra::Texture::SwizzleTexture(restored, linear, fixture.bytes, width, height, 1, block_log2, 0, 1, true); + REQUIRE(restored == tiled); + } + } +} + +TEST_CASE("Native and legacy GOB byte lanes remain distinct", "[native-swizzle]") { + std::vector<u8> tiled(512); + for (u32 i = 0; i < tiled.size(); ++i) tiled[i] = static_cast<u8>(i / 16); + std::vector<u8> native(512), legacy(512); + Tegra::Texture::UnswizzleTexture(native, tiled, 1, 64, 8, 1, 0, 0, 1, true); + Tegra::Texture::UnswizzleTexture(legacy, tiled, 1, 64, 8, 1, 0, 0); + REQUIRE(native[16] == 4); + REQUIRE(legacy[16] == 2); + REQUIRE(native[2 * 64] == 2); + REQUIRE(legacy[2 * 64] == 4); + REQUIRE(native != legacy); + std::vector<u8> crop(16 * 3); + Tegra::Texture::UnswizzleSubrect(crop, tiled, 1, 64, 8, 1, 16, 2, 16, 3, 0, 0, 16, true); + for (u32 y = 0; y < 3; ++y) { + REQUIRE(std::equal(crop.begin() + y * 16, crop.begin() + (y + 1) * 16, + native.begin() + (y + 2) * 64 + 16)); + } +} diff --git a/src/tests/video_core/pending_download.cpp b/src/tests/video_core/pending_download.cpp new file mode 100644 index 0000000000..4b2a218933 --- /dev/null +++ b/src/tests/video_core/pending_download.cpp @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <deque> +#include <memory> +#include <vector> +#include <catch2/catch_test_macros.hpp> +#include "common/slot_vector.h" +#include "video_core/texture_cache/pending_download.h" + +namespace { +struct DownloadImage { + DownloadImage(u32 value, std::shared_ptr<std::vector<u32>> destroyed_) + : handle{std::make_unique<u32>(value)}, destroyed{std::move(destroyed_)} {} + DownloadImage(DownloadImage&&) noexcept = default; + DownloadImage& operator=(DownloadImage&&) noexcept = default; + ~DownloadImage() { if (handle) destroyed->push_back(*handle); } + std::unique_ptr<u32> handle; + std::shared_ptr<std::vector<u32>> destroyed; +}; +using Download = VideoCommon::PendingImageDownload<DownloadImage>; +} + +TEST_CASE("Queued texture downloads survive deletion and reuse of their cache slot", "[texture-download]") { + auto destroyed = std::make_shared<std::vector<u32>>(); + Common::SlotVector<DownloadImage> images; + const auto old_id = images.insert(42U, destroyed); + std::vector<Download> pending{{true, 0, old_id, {}}, {false, 0, old_id, {}}}; + std::deque<std::vector<Download>> submitted{{{true, 0, old_id, {}}}}; + REQUIRE(VideoCommon::RetainPendingImageDownloads(old_id, images[old_id], pending, submitted)); + REQUIRE_FALSE(images[old_id].handle); + REQUIRE(pending[0].retired_image == submitted[0][0].retired_image); + REQUIRE_FALSE(pending[1].retired_image); // Buffer-download slot IDs are a separate namespace. + images.erase(old_id); + const auto replacement_id = images.insert(99U, destroyed); + REQUIRE(replacement_id == old_id); + REQUIRE(*pending[0].GetImage(images).handle == 42); + REQUIRE(*submitted[0][0].GetImage(images).handle == 42); + REQUIRE(*images[replacement_id].handle == 99); + // Retiring the replacement must not redirect an earlier generation's downloads. + pending.push_back({true, 1, replacement_id, {}}); + REQUIRE(VideoCommon::RetainPendingImageDownloads(replacement_id, images[replacement_id], pending, submitted)); + images.erase(replacement_id); + REQUIRE(*pending[0].GetImage(images).handle == 42); + REQUIRE(*pending[2].GetImage(images).handle == 99); + REQUIRE(destroyed->empty()); + pending.clear(); + REQUIRE(*destroyed == std::vector<u32>{99}); + REQUIRE(*submitted[0][0].GetImage(images).handle == 42); + submitted.pop_front(); + REQUIRE(*destroyed == std::vector<u32>{99, 42}); +} + +TEST_CASE("Textures without queued downloads retain ordinary cache retirement", "[texture-download]") { + auto destroyed = std::make_shared<std::vector<u32>>(); + Common::SlotVector<DownloadImage> images; + const auto id = images.insert(7U, destroyed); + std::vector<Download> pending{{false, 0, id, {}}}; + std::deque<std::vector<Download>> submitted; + REQUIRE_FALSE(VideoCommon::RetainPendingImageDownloads(id, images[id], pending, submitted)); + REQUIRE(*images[id].handle == 7); + images.erase(id); + REQUIRE(*destroyed == std::vector<u32>{7}); +} + +TEST_CASE("Repeated whole-image downloads keep first occurrence and buffer requests", "[texture-download]") { + auto destroyed = std::make_shared<std::vector<u32>>(); + Common::SlotVector<DownloadImage> images; + const auto a = images.insert(42U, destroyed); + const auto b = images.insert(99U, destroyed); + std::vector<Download> pending{{true, 0, a, {}}, {true, 0, b, {}}, + {false, 3, Common::SlotId{10000}, {}}, {true, 0, a, {}}}; + const auto writeback = [&](const auto& downloads) { + u32 value = 0; + for (auto it = downloads.rbegin(); it != downloads.rend(); ++it) + if (it->is_swizzle) value = *it->GetImage(images).handle; + return value; + }; + const u32 expected = writeback(pending); + REQUIRE(VideoCommon::DeduplicatePendingImageDownloads(pending, images) == 1); + REQUIRE(pending.size() == 3); + REQUIRE(pending[0].object_id == a); + REQUIRE(pending[1].object_id == b); + REQUIRE_FALSE(pending[2].is_swizzle); + REQUIRE(pending[2].async_buffer_id == 3); + REQUIRE(writeback(pending) == expected); + REQUIRE(expected == 42); +} + +TEST_CASE("Readback deduplication distinguishes recycled image generations", "[texture-download]") { + auto destroyed = std::make_shared<std::vector<u32>>(); + Common::SlotVector<DownloadImage> images; + const auto id = images.insert(42U, destroyed); + std::vector<Download> pending{{true, 0, id, {}}}; + std::deque<std::vector<Download>> submitted; + REQUIRE(VideoCommon::RetainPendingImageDownloads(id, images[id], pending, submitted)); + auto old = pending.front(); + images.erase(id); + const auto replacement = images.insert(99U, destroyed); + REQUIRE(replacement == id); + pending.push_back({true, 0, replacement, {}}); + for (size_t i = 0; i < 1000; ++i) pending.push_back(old); + REQUIRE(VideoCommon::DeduplicatePendingImageDownloads(pending, images) == 1000); + REQUIRE(pending.size() == 2); + REQUIRE(*pending[0].GetImage(images).handle == 42); + REQUIRE(*pending[1].GetImage(images).handle == 99); + REQUIRE(destroyed->empty()); + pending.clear(); + REQUIRE(destroyed->empty()); + old.retired_image.reset(); + REQUIRE(*destroyed == std::vector<u32>{42}); +} diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 64d14ab2a6..79028f0baf 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -93,6 +93,8 @@ add_library(video_core STATIC fence_manager.h gpu.cpp gpu.h + native_query_trace.h + native_semaphore.h gpu_thread.cpp gpu_thread.h guest_memory.h @@ -242,6 +244,7 @@ add_library(video_core STATIC texture_cache/image_base.h texture_cache/image_info.cpp texture_cache/image_info.h + texture_cache/pending_download.h texture_cache/image_view_base.cpp texture_cache/image_view_base.h texture_cache/image_view_info.cpp diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index f30c85911f..126e73c9b6 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -9,6 +9,7 @@ #include <algorithm> #include <memory> #include <numeric> +#include <stdexcept> #include "common/range_sets.inc" #include "video_core/buffer_cache/buffer_cache_base.h" @@ -1372,7 +1373,12 @@ void BufferCache<P>::UpdateVertexBuffer(u32 index) { const auto& array = maxwell3d->regs.vertex_streams[index]; const auto& limit = maxwell3d->regs.vertex_stream_limits[index]; const GPUVAddr gpu_addr_begin = array.Address(); - const GPUVAddr gpu_addr_end = limit.Address() + 1; + const bool native = maxwell3d->GetNativeMacroEngine().native_turing; + const u64 native_size = native ? maxwell3d->regs.NativeVertexStreamSize(index) : 0; + if (native && array.enable != 0 && native_size > (std::numeric_limits<u32>::max)()) { + throw std::runtime_error("Native vertex stream exceeds supported 32-bit buffer size"); + } + const GPUVAddr gpu_addr_end = native ? gpu_addr_begin + native_size : limit.Address() + 1; const std::optional<DAddr> device_addr = gpu_memory->GpuToCpuAddress(gpu_addr_begin); const u32 address_size = static_cast<u32>(gpu_addr_end - gpu_addr_begin); u32 size = address_size; // TODO: Analyze stride and number of vertices diff --git a/src/video_core/control/channel_state.cpp b/src/video_core/control/channel_state.cpp index 12a452dfc0..57113ce68e 100644 --- a/src/video_core/control/channel_state.cpp +++ b/src/video_core/control/channel_state.cpp @@ -55,6 +55,7 @@ void ChannelState::Init(Core::System& system, u64 program_id_) { ASSERT(memory_manager); program_id = program_id_; payload.emplace(system, *memory_manager, *this); + payload->maxwell_3d.SetNativeMacroContext(native_macro_context); BindNvkDefaultSubchannels(*payload); initialized = true; } diff --git a/src/video_core/control/channel_state.h b/src/video_core/control/channel_state.h index 80811c1458..9f64f4bfe3 100644 --- a/src/video_core/control/channel_state.h +++ b/src/video_core/control/channel_state.h @@ -56,12 +56,15 @@ struct ChannelState { Engines::Nv01Timer nv01_timer; DmaPusher dma_pusher; }; + std::shared_ptr<NativeMacroContext> native_macro_context; + std::shared_ptr<MemoryManager> memory_manager_owner; std::optional<Payload> payload; MemoryManager* memory_manager = nullptr; s32 bind_id = -1; u64 program_id = 0; bool initialized = false; + bool native_abi = false; }; } // namespace Control diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp index 8af13e415c..5f8a830c83 100644 --- a/src/video_core/control/scheduler.cpp +++ b/src/video_core/control/scheduler.cpp @@ -4,7 +4,9 @@ // SPDX-FileCopyrightText: 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include <algorithm> #include <memory> +#include <vector> #include "common/assert.h" #include "video_core/control/channel_state.h" @@ -26,6 +28,26 @@ void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) { // Multiple channels can prepare their commands in parallel. channel_state->payload->dma_pusher.Push(std::move(entries)); channel_state->payload->dma_pusher.DispatchCalls(); + if (channel_state->payload->dma_pusher.HasPendingNativeAcquire() && + std::find(pending_native_channels.begin(), pending_native_channels.end(), channel_state) == pending_native_channels.end()) { + pending_native_channels.push_back(channel_state); + } + if (!pending_native_channels.empty()) ResumePending(gpu); +} + +void Scheduler::ResumePending(GPU& gpu) { + ResumeReadyNativeChannels(pending_native_channels, + [](const auto& state) { return state->payload->dma_pusher.HasPendingNativeAcquire() && + state->payload->dma_pusher.CanResumeNativeAcquire(); }, + [&](const auto& state) { + { + std::scoped_lock lock{scheduling_guard}; + gpu.BindChannel(state->bind_id); + } + state->payload->dma_pusher.DispatchCalls(); + }); + std::erase_if(pending_native_channels, + [](const auto& state) { return !state->payload->dma_pusher.HasPendingNativeAcquire(); }); } void Scheduler::DeclareChannel(std::shared_ptr<ChannelState> new_channel) { diff --git a/src/video_core/control/scheduler.h b/src/video_core/control/scheduler.h index 342bea5025..b6e9544ca0 100644 --- a/src/video_core/control/scheduler.h +++ b/src/video_core/control/scheduler.h @@ -8,6 +8,7 @@ #include <memory> #include <mutex> +#include <vector> #include "common/container/unordered_map.h" #include "video_core/dma_pusher.h" @@ -24,11 +25,15 @@ class Scheduler { public: void Push(GPU& gpu, s32 channel, CommandList&& entries); + void ResumePending(GPU& gpu); + [[nodiscard]] bool HasPendingNativeChannels() const { return !pending_native_channels.empty(); } + void DeclareChannel(std::shared_ptr<ChannelState> new_channel); private: ::Common::unordered_map<s32, std::shared_ptr<ChannelState>> channels; std::mutex scheduling_guard; + std::vector<std::shared_ptr<ChannelState>> pending_native_channels; }; } // namespace Control diff --git a/src/video_core/dirty_flags.cpp b/src/video_core/dirty_flags.cpp index c2ecc12f56..e6856aaca3 100644 --- a/src/video_core/dirty_flags.cpp +++ b/src/video_core/dirty_flags.cpp @@ -22,6 +22,7 @@ void SetupDirtyVertexBuffers(Maxwell3D::DirtyState::Tables& tables) { OFF(vertex_stream_limits) + i * NUM(vertex_stream_limits[0]); FillBlock(tables, array_offset, num_array, VertexBuffer0 + i, VertexBuffers); + FillBlock(tables, 0x600 / sizeof(u32) + i * 2, 2, VertexBuffer0 + i, VertexBuffers); FillBlock(tables, limit_offset, NUM(vertex_stream_limits), VertexBuffer0 + i, VertexBuffers); } diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 2c0ba007ef..3e9359ea3f 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -4,9 +4,11 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> #include "common/settings.h" #include "core/core.h" #include "video_core/dma_pusher.h" +#include "video_core/control/channel_state.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/gpu.h" #include "video_core/guest_memory.h" @@ -29,22 +31,38 @@ DmaPusher::DmaPusher(Core::System& system_, MemoryManager& memory_manager_, Cont DmaPusher::~DmaPusher() = default; void DmaPusher::DispatchCalls() { - dma_pushbuffer_subindex = 0; dma_state.is_last_call = true; while (system.IsPoweredOn()) { if (!Step()) { break; } } + if (channel_state.native_abi && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { + static u32 dispatch_traces{}; + if (dispatch_traces++ < 512) { + LOG_INFO(HW_GPU, "Native dispatch end: channel={}, queued={}, failed={}, acquire={}, method={:#x}, remaining={}, resume={}", + channel_state.bind_id, dma_pushbuffer.size(), HasFailedNativeWork(), + HasPendingNativeAcquire(), dma_state.method, dma_state.method_count, command_resume_offset); + } + } system.GPU().FlushCommands(); system.GPU().OnCommandListEnd(); } +bool DmaPusher::HasFailedNativeWork() const { + const auto& context = channel_state.native_macro_context; + return channel_state.native_abi && context && + (context->engine.native_execution_failed || context->compute_engine.native_execution_failed); +} + bool DmaPusher::Step() { + if (HasFailedNativeWork()) return false; if (!ib_enable || dma_pushbuffer.empty()) { return false; } + if (!puller.ResumeNativeAcquire(*this)) return false; + CommandList& command_list = dma_pushbuffer.front(); const size_t prefetch_size = command_list.prefetch_command_list.size(); @@ -57,7 +75,10 @@ bool DmaPusher::Step() { } if (prefetch_size > 0) { - ProcessCommands(command_list.prefetch_command_list); + const std::span<const CommandHeader> commands{command_list.prefetch_command_list}; + command_resume_offset += ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); + if (puller.HasPendingNativeAcquire() || HasFailedNativeWork()) return false; + command_resume_offset = 0; dma_pushbuffer.pop(); return true; } @@ -86,13 +107,17 @@ bool DmaPusher::Step() { const bool use_safe = Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe(); if (use_safe) { Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::SafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers); - ProcessCommands(headers); + const std::span<const CommandHeader> commands{headers.data(), headers.size()}; + command_resume_offset += ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); } else { Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::UnsafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers); - ProcessCommands(headers); + const std::span<const CommandHeader> commands{headers.data(), headers.size()}; + command_resume_offset += ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); } } + if (puller.HasPendingNativeAcquire() || HasFailedNativeWork()) return false; + command_resume_offset = 0; if (++dma_pushbuffer_subindex >= command_list_size) { dma_pushbuffer.pop(); dma_pushbuffer_subindex = 0; @@ -111,20 +136,21 @@ bool DmaPusher::Step() { return true; } -void DmaPusher::ProcessCommands(std::span<const CommandHeader> commands) { +size_t DmaPusher::ProcessCommands(std::span<const CommandHeader> commands, size_t word_offset) { for (size_t index = 0; index < commands.size();) { // Data word of methods command if (dma_state.method_count && dma_state.non_incrementing) { auto const& command_header = commands[index]; //must ref (MUltiMethod re) - dma_state.dma_word_offset = u32(index * sizeof(u32)); - const u32 max_write = u32(std::min<std::size_t>(index + dma_state.method_count, commands.size()) - index); + dma_state.dma_word_offset = u64((word_offset + index) * sizeof(u32)); + u32 max_write = u32(std::min<std::size_t>(index + dma_state.method_count, commands.size()) - index); + if (channel_state.native_abi && dma_state.method < non_puller_methods) max_write = 1; CallMultiMethod(&command_header.argument, max_write); dma_state.method_count -= max_write; dma_state.is_last_call = true; index += max_write; } else if (dma_state.method_count) { auto const command_header = commands[index]; //can copy - dma_state.dma_word_offset = u32(index * sizeof(u32)); + dma_state.dma_word_offset = u64((word_offset + index) * sizeof(u32)); dma_state.is_last_call = dma_state.method_count <= 1; CallMethod(command_header.argument); dma_state.method += !dma_state.non_incrementing ? 1 : 0; @@ -163,7 +189,9 @@ void DmaPusher::ProcessCommands(std::span<const CommandHeader> commands) { } index++; } + if (puller.HasPendingNativeAcquire() || HasFailedNativeWork()) return index; } + return commands.size(); } void DmaPusher::SetState(const CommandHeader& command_header) { diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 62af628511..51c735baea 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -131,6 +131,8 @@ public: } void DispatchCalls(); + [[nodiscard]] bool HasPendingNativeAcquire() const { return puller.HasPendingNativeAcquire(); } + [[nodiscard]] bool CanResumeNativeAcquire() { return !HasFailedNativeWork() && puller.CanResumeNativeAcquire(*this); } void BindSubchannel(Engines::EngineInterface* engine, u32 subchannel_id, Engines::EngineTypes engine_type) { subchannels[subchannel_id] = engine; @@ -142,8 +144,10 @@ public: private: static constexpr u32 non_puller_methods = 0x40; static constexpr u32 max_subchannels = 8; + bool HasFailedNativeWork() const; bool Step(); - void ProcessCommands(std::span<const CommandHeader> commands); + size_t ProcessCommands(std::span<const CommandHeader> commands, size_t word_offset = 0); + size_t command_resume_offset{}; void SetState(const CommandHeader& command_header); diff --git a/src/video_core/engines/draw_manager.cpp b/src/video_core/engines/draw_manager.cpp index 5891c25622..a610fb1661 100644 --- a/src/video_core/engines/draw_manager.cpp +++ b/src/video_core/engines/draw_manager.cpp @@ -5,11 +5,15 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <cstring> +#include <fstream> +#include <cstdlib> +#include <set> #include "common/settings.h" #include "video_core/dirty_flags.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/rasterizer_interface.h" +#include "video_core/memory_manager.h" namespace Tegra::Engines { @@ -277,6 +281,45 @@ void Maxwell3D::DrawManager::ProcessDraw(Maxwell3D& maxwell3d, bool draw_indexed LOG_TRACE(HW_GPU, "called, topology={}, count={}", draw_state.topology, draw_indexed ? draw_state.index_buffer.count : draw_state.vertex_buffer.count); UpdateTopology(maxwell3d); if (maxwell3d.ShouldExecute()) { + auto& native = maxwell3d.GetNativeMacroEngine(); + if (native.native_turing) { + const auto count = draw_indexed ? draw_state.index_buffer.count : draw_state.vertex_buffer.count; + if (!count || !instance_count) return; + if (native.native_execution_failed) return; + if (Settings::values.renderer_backend.GetValue() != Settings::RendererBackend::Vulkan) { + native.native_execution_failed = true; + LOG_ERROR(HW_GPU, "Native graphics shaders require the experimental Vulkan path"); + return; + } + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { + static thread_local std::set<std::array<GPUVAddr, Maxwell3D::Regs::MaxShaderProgram>> captured; + std::array<GPUVAddr, Maxwell3D::Regs::MaxShaderProgram> programs{}; + for (size_t stage = 0; stage < programs.size(); ++stage) { + if (!maxwell3d.regs.IsShaderConfigEnabled(stage)) continue; + const auto& pipeline = maxwell3d.regs.pipelines[stage]; + programs[stage] = (GPUVAddr{pipeline.reserved[0] & 0xff} << 32) | pipeline.reserved[1]; + } + if (captured.size() < 4096 && captured.insert(programs).second) { + std::ofstream registers("native-graphics-registers.bin", std::ios::binary); + registers.write(reinterpret_cast<const char*>(&maxwell3d.regs), sizeof(maxwell3d.regs)); + for (size_t stage = 0; stage < Maxwell3D::Regs::MaxShaderProgram; ++stage) { + if (!maxwell3d.regs.IsShaderConfigEnabled(stage)) continue; + const auto& pipeline = maxwell3d.regs.pipelines[stage]; + // C797 uses per-stage program addresses at 0x2014/0x2018, + // unlike Maxwell's program-region plus 0x2004 offset. + const GPUVAddr address = (GPUVAddr{pipeline.reserved[0] & 0xff} << 32) | pipeline.reserved[1]; + LOG_INFO(HW_GPU, "Native graphics shader stage={}, address={:#x}", stage, address); + if (!maxwell3d.memory_manager.IsFullyMappedRange(address, 0x1000)) continue; + std::array<u8, 0x1000> bytes{}; + maxwell3d.memory_manager.ReadBlockUnsafe(address, bytes.data(), bytes.size()); + std::ofstream shader("native-graphics-shader-" + std::to_string(stage) + ".bin", std::ios::binary); + shader.write(reinterpret_cast<const char*>(bytes.data()), bytes.size()); + } + LOG_DEBUG(HW_GPU, "Translating native graphics draw: indexed={}, vertices={}, instances={}", + draw_indexed, count, instance_count); + } + } + } maxwell3d.rasterizer->Draw(draw_indexed, instance_count); } } @@ -285,6 +328,12 @@ void Maxwell3D::DrawManager::ProcessDrawIndirect(Maxwell3D& maxwell3d) { LOG_TRACE(HW_GPU, "called, topology={}, is_indexed={}, includes_count={}, buffer_size={}, max_draw_count={}", draw_state.topology, indirect_state.is_indexed, indirect_state.include_count, indirect_state.buffer_size, indirect_state.max_draw_counts); UpdateTopology(maxwell3d); if (maxwell3d.ShouldExecute()) { + auto& native = maxwell3d.GetNativeMacroEngine(); + if (native.native_turing) { + native.native_execution_failed = true; + LOG_ERROR(HW_GPU, "Native graphics shader translation required: indirect draw"); + return; + } maxwell3d.rasterizer->DrawIndirect(); } } diff --git a/src/video_core/engines/engine_upload.cpp b/src/video_core/engines/engine_upload.cpp index ba21eec4c4..0ffdae8036 100644 --- a/src/video_core/engines/engine_upload.cpp +++ b/src/video_core/engines/engine_upload.cpp @@ -27,6 +27,7 @@ void State::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { void State::ProcessExec(const bool is_linear_) { write_offset = 0; + flushed_offset = 0; copy_size = regs.line_length_in * regs.line_count; inner_buffer.resize_destructive(copy_size); is_linear = is_linear_; @@ -39,10 +40,29 @@ void State::ProcessData(const u32 data, const bool is_last_call) { if (!is_last_call) { return; } + if (native_partial && is_linear && regs.line_count == 1) { + const u32 count = write_offset - flushed_offset; + if (count) { + rasterizer->AccelerateInlineToMemory(regs.dest.Address() + flushed_offset, count, + std::span<const u8>{inner_buffer.data() + flushed_offset, count}); + flushed_offset = write_offset; + } + return; + } ProcessData(inner_buffer); } void State::ProcessData(const u32* data, size_t num_data) { + if (native_partial && is_linear && regs.line_count == 1) { + const auto count = std::min<size_t>(num_data * sizeof(u32), copy_size - write_offset); + if (count) { + rasterizer->AccelerateInlineToMemory(regs.dest.Address() + write_offset, count, + std::span<const u8>{reinterpret_cast<const u8*>(data), count}); + write_offset += static_cast<u32>(count); + flushed_offset = write_offset; + } + return; + } std::span<const u8> read_buffer(reinterpret_cast<const u8*>(data), num_data * sizeof(u32)); ProcessData(read_buffer); } diff --git a/src/video_core/engines/engine_upload.h b/src/video_core/engines/engine_upload.h index 21bf8aeb46..55a82f938b 100644 --- a/src/video_core/engines/engine_upload.h +++ b/src/video_core/engines/engine_upload.h @@ -62,6 +62,7 @@ public: explicit State(MemoryManager& memory_manager_, Registers& regs_); ~State(); + void SetNativePartialTransfers(bool enabled) { native_partial = enabled; } void ProcessExec(bool is_linear_); void ProcessData(u32 data, bool is_last_call); void ProcessData(const u32* data, size_t num_data); @@ -80,6 +81,8 @@ public: private: void ProcessData(std::span<const u8> read_buffer); + bool native_partial{}; + u32 flushed_offset{}; u32 write_offset = 0; u32 copy_size = 0; Common::ScratchBuffer<u8> inner_buffer; diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 96a12539a2..1de51fd7aa 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -5,12 +5,15 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <bitset> +#include <fstream> #include "common/assert.h" #include "common/logging.h" +#include "shader_recompiler/exception.h" #include "core/core.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/memory_manager.h" +#include "video_core/native_query_trace.h" #include "video_core/rasterizer_interface.h" #include "video_core/textures/decoders.h" @@ -40,8 +43,173 @@ void KeplerCompute::ConsumeSinkImpl(Core::System& system) { method_sink.clear(); } +bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call) { + if (native_context) { + if (method == 0x45 || method == 0x47) native_compute_uploaded = true; + native_macro = native_compute_uploaded ? &native_context->compute_engine : &native_context->engine; + native_macro_positions = native_compute_uploaded ? std::span<u32>{native_context->compute_positions} : std::span<u32>{native_context->positions}; + } + if (method >= native_registers.size()) { + LOG_ERROR(HW_GPU, "Native compute method out of range: {:#x}", method); + return true; + } + if (method >= 0xe00) { + if (!native_executing_macro) { + if (method & 1) { LOG_ERROR(HW_GPU, "Native compute macro starts on argument method"); return true; } + native_executing_macro = method; + } + native_parameters.push_back(argument); + if (!is_last_call) return true; + const u32 entry = ((*native_executing_macro - 0xe00) / 2) % native_macro_positions.size(); + native_executing_macro.reset(); + const auto result = native_macro->native_interpreter.Execute(native_macro->native_code, + native_macro_positions[entry], native_parameters, + [&](u32 address) { return address < native_registers.size() ? native_registers[address] : 0U; }, + [&](u32 address, u32 value) { + static u32 diagnostic_writes{}; + if (diagnostic_writes++ < 4096) LOG_DEBUG(HW_GPU, "Native compute MME emit method={:#x}, value={:#x}", address, value); + if (address >= 0xe00) return false; + CallMethod(system, address, value, true); + return true; + }); + if (!result.success) { + native_macro->native_execution_failed = true; + std::ofstream dump("native-compute-mme.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_macro->native_code.data()), sizeof(native_macro->native_code)); + LOG_ERROR(HW_GPU, "Native MME failed: compute start={:#x}, pc={:#x}, consumed={}/{}, error={}, parameters={:08x}", + native_macro_positions[entry], result.pc, result.consumed, native_parameters.size(), result.error, + fmt::join(native_parameters, ",")); + } + native_parameters.clear(); + return true; + } + native_registers[method] = argument; + if (method == 0x5a || (method >= 0xc6 && method <= 0x108)) { + VideoCommon::TraceNativeQuery(12, memory_manager.GetID(), method, 0, 0, argument, 0); + } + if (method == 0x107) { + const GPUVAddr address = ((GPUVAddr{native_registers[0xc6]} << 32) | + native_registers[0xc7]) << 8; + const GPUVAddr program = (GPUVAddr{native_registers[0xc8 + 49] & 0x1ffff} << 32) | + native_registers[0xc8 + 48]; + const u32 control = native_registers[0xa7]; + LOG_WARNING(HW_GPU, "Native inline QMD: address={:#x}, program={:#x}, control={:#x}, grid={}/{}/{}", + address, program, control, native_registers[0xc8 + 12], + native_registers[0xc8 + 13] & 0xffff, native_registers[0xc8 + 14] & 0xffff); + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { + std::ofstream dump("native-inline-qmd.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_registers.data() + 0xc8), 256); + std::ofstream registers("native-inline-compute-registers.bin", std::ios::binary); + registers.write(reinterpret_cast<const char*>(native_registers.data()), sizeof(native_registers)); + if (memory_manager.IsFullyMappedRange(program, 0x10000)) { + std::vector<u8> code(0x10000); + memory_manager.ReadBlockUnsafe(program, code.data(), code.size()); + std::ofstream shader("native-inline-compute-shader.bin", std::ios::binary); + shader.write(reinterpret_cast<const char*>(code.data()), code.size()); + } + } + if (!memory_manager.IsFullyMappedRange(address, 256)) { + native_macro->native_execution_failed = true; + LOG_ERROR(HW_GPU, "Unmapped native inline QMD destination"); + return true; + } + rasterizer->AccelerateInlineToMemory(address, 256, + std::span<const u8>{reinterpret_cast<const u8*>(native_registers.data() + 0xc8), 256}); + if ((control & (1U << 24)) || !(native_registers[0xcc] & (1U << 12))) { + const u32 previous = native_registers[0xad]; + native_registers[0xad] = static_cast<u32>(address >> 8); + ProcessNativeMethod(system, 0xb0, 3, true); + native_registers[0xad] = previous; + } + return true; + } + switch (method) { + case 0x45: native_macro->ClearCode(argument); return true; + case 0x46: native_macro->AddCode(0, argument); return true; + case 0x47: native_bind_pointer = argument; return true; + case 0x48: + if (native_bind_pointer < native_macro_positions.size()) native_macro_positions[native_bind_pointer++] = argument; + else LOG_ERROR(HW_GPU, "Native compute macro bind out of bounds"); + return true; + case 0xae: case 0xaf: case 0xb0: { + const GPUVAddr qmd_address = GPUVAddr{native_registers[0xad]} << 8; + VideoCommon::TraceNativeQuery(9, memory_manager.GetID(), qmd_address, + memory_manager.GpuToCpuAddress(qmd_address).value_or(0), method, argument, 0); + if (method == 0xae || (method == 0xaf && !(argument & 2)) || + (method == 0xb0 && ((argument & 15) == 0 || (argument & 15) == 1))) return true; + if ((argument & 15) == 12 && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { + static u32 captured{}; + if (captured++ == 0 && memory_manager.IsFullyMappedRange(qmd_address, 256)) { + std::array<u32, 64> descriptor{}; + memory_manager.ReadBlockUnsafe(qmd_address, descriptor.data(), sizeof(descriptor)); + std::ofstream dump("native-increment-qmd.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(descriptor.data()), sizeof(descriptor)); + LOG_WARNING(HW_GPU, "Native increment dependency descriptor: address={:#x}, words={:08x}", + qmd_address, fmt::join(descriptor, ",")); + } + } + const auto error = native_qmd.Signal(qmd_address, argument, + [&](u64 address, NativeQmdScheduler::Descriptor& descriptor) { + if (!memory_manager.IsFullyMappedRange(address, sizeof(descriptor))) return false; + memory_manager.ReadBlockUnsafe(address, descriptor.data(), sizeof(descriptor)); + for (u32 slot = 0; slot < 2; ++slot) { + const u32 flags = descriptor[25 + slot * 4]; + if (!(flags & (1U << 23))) continue; + const GPUVAddr destination = (GPUVAddr{flags & 0xff} << 32) | descriptor[24 + slot * 4]; + VideoCommon::TraceNativeQuery(7 + slot, memory_manager.GetID(), destination, + memory_manager.GpuToCpuAddress(destination).value_or(0), + descriptor[4], descriptor[26 + slot * 4], flags); + } + return true; + }, + [&](u64 address, u32 payload, u32 flags) { + if (!memory_manager.IsFullyMappedRange(address, sizeof(payload))) return false; + if (flags & (1U << 26)) { + rasterizer->SignalFence([this, address, payload] { + const u32 value = memory_manager.Read<u32>(address) + payload; + memory_manager.Write<u32>(address, value); + VideoCommon::TraceNativeQuery(1, memory_manager.GetID(), address, + memory_manager.GpuToCpuAddress(address).value_or(0), 0, value, 1U << 26); + }); + } else { + rasterizer->Query(address, VideoCommon::QueryType::Payload, + VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0); + } + return true; + }, [&](const auto& descriptor) { return DispatchNativeCompute(descriptor); }); + if (error.empty()) { + LOG_DEBUG(HW_GPU, "Native QMD control action={:#x}, QMD={:#x}", argument, qmd_address); + return true; + } + native_macro->native_execution_failed = true; + LOG_ERROR(HW_GPU, "Native QMD failed: {}", error); + std::array<u32, 128> qmd{}; + if (memory_manager.IsFullyMappedRange(qmd_address, sizeof(qmd))) { + memory_manager.ReadBlockUnsafe(qmd_address, qmd.data(), sizeof(qmd)); + std::ofstream dump("native-qmd.bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(qmd.data()), sizeof(qmd)); + std::ofstream macros("native-dispatch-mme.bin", std::ios::binary); + macros.write(reinterpret_cast<const char*>(native_macro->native_code.data()), sizeof(native_macro->native_code)); + std::ofstream registers("native-dispatch-registers.bin", std::ios::binary); + registers.write(reinterpret_cast<const char*>(native_registers.data()), sizeof(native_registers)); + } + LOG_ERROR(HW_GPU, "Native compute dispatch unsupported: method={:#x}, value={:#x}, QMD={:#x}, words={:08x}", method, argument, + qmd_address, fmt::join(qmd, ",")); + return true; + } + default: + // The inline upload register block is shared with the older class. + if (method < Regs::NUM_REGS) regs.reg_array[method] = argument; + return method != KEPLER_COMPUTE_REG_INDEX(exec_upload) && method != KEPLER_COMPUTE_REG_INDEX(data_upload); + } +} + void KeplerCompute::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) { - ASSERT_MSG(method < Regs::NUM_REGS, "Invalid KeplerCompute register, increase the size of the Regs structure"); + if (native_mode && ProcessNativeMethod(system, method, method_argument, is_last_call)) return; + if (method >= Regs::NUM_REGS) { + LOG_ERROR(HW_GPU, "Compute register out of range: {:#x}", method); + return; + } regs.reg_array[method] = method_argument; @@ -85,6 +253,10 @@ void KeplerCompute::CallMethod(Core::System& system, u32 method, u32 method_argu } void KeplerCompute::CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) { + if (native_mode) { + for (u32 i = 0; i < amount; ++i) CallMethod(system, method, base_start[i], methods_pending - i <= 1); + return; + } switch (method) { case KEPLER_COMPUTE_REG_INDEX(data_upload): upload_address = current_dma_segment; @@ -100,6 +272,44 @@ void KeplerCompute::CallMultiMethod(Core::System& system, u32 method, const u32* } } +bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& qmd) { + launch_description = {}; + launch_description.grid_dim_x.Assign(qmd[12]); + launch_description.grid_dim_y.Assign(qmd[13] & 0xffff); + launch_description.grid_dim_z.Assign(qmd[14] & 0xffff); + launch_description.shared_alloc.Assign(qmd[17] & 0x3ffff); + launch_description.block_dim_x.Assign(qmd[18] >> 16); + launch_description.block_dim_y.Assign(qmd[19] & 0xffff); + launch_description.block_dim_z.Assign(qmd[19] >> 16); + launch_description.const_buffer_enable_mask.Assign(qmd[20] & 0xff); + launch_description.local_pos_alloc.Assign(qmd[23] & 0xffffff); + launch_description.local_crs_alloc.Assign(qmd[50] & 0xffffff); + regs.code_loc.address_low = qmd[48]; + regs.code_loc.address_high = qmd[49] & 0x1ffff; + for (u32 slot = 0; slot < NumConstBuffers; ++slot) { + const u32 high = qmd[33 + slot * 2]; + if (high & 0x1ff00) { + LOG_ERROR(HW_GPU, "Native compute constant address exceeds supported GPU width"); + return false; + } + auto& buffer = launch_description.const_buffer_config[slot]; + buffer.address_low = qmd[32 + slot * 2]; + buffer.address_high.Assign(high & 0xff); + buffer.size.Assign(((high >> 19) & 0x1fff) << 4); + } + if (!launch_description.block_dim_x || !launch_description.block_dim_y || !launch_description.block_dim_z) + return false; + try { + LOG_INFO(HW_GPU, "Native compute begin: program={:#x}", regs.code_loc.Address()); + rasterizer->DispatchCompute(); + LOG_INFO(HW_GPU, "Native compute complete"); + } catch (const Shader::Exception& error) { + LOG_ERROR(HW_GPU, "Native compute shader rejected: {}", error.what()); + return false; + } + return true; +} + void KeplerCompute::ProcessLaunch() { const GPUVAddr launch_desc_loc = regs.launch_desc_loc.Address(); memory_manager.ReadBlockUnsafe(launch_desc_loc, &launch_description, diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index 078d3d45be..99cf74d1e8 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -15,6 +15,8 @@ #include "common/common_types.h" #include "video_core/engines/engine_interface.h" #include "video_core/engines/engine_upload.h" +#include "video_core/macro.h" +#include "video_core/native_qmd.h" #include "video_core/textures/texture.h" namespace Core { @@ -51,6 +53,22 @@ public: /// Binds a rasterizer to this engine. void BindRasterizer(VideoCore::RasterizerInterface* rasterizer); + void SetNativeMacroMode(MacroEngine* shared_macro, std::span<u32> positions, NativeMacroContext* context = nullptr) { + if (native_context != context) native_compute_uploaded = false; + native_context = context; + native_mode = shared_macro != nullptr; + upload_state.SetNativePartialTransfers(native_mode); + native_macro = shared_macro; + native_macro_positions = positions; + if (native_mode) execution_mask.set(); + else { + execution_mask.reset(); + execution_mask[KEPLER_COMPUTE_REG_INDEX(exec_upload)] = true; + execution_mask[KEPLER_COMPUTE_REG_INDEX(data_upload)] = true; + execution_mask[KEPLER_COMPUTE_REG_INDEX(launch)] = true; + } + } + static constexpr std::size_t NumConstBuffers = 8; struct Regs { @@ -211,7 +229,22 @@ public: return indirect_compute; } + bool IsNativeMode() const { return native_mode; } + private: + bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor); + bool ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call); + NativeQmdScheduler native_qmd; + bool native_mode{}; + NativeMacroContext* native_context{}; + bool native_compute_uploaded{}; + MacroEngine* native_macro{}; + std::array<u32, 0x1000> native_registers{}; + std::span<u32> native_macro_positions; + std::vector<u32> native_parameters; + std::optional<u32> native_executing_macro; + u32 native_bind_pointer{}; + void ProcessLaunch(); void ConsumeSinkImpl(Core::System& system) override; diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index e7da9d9138..f28a042d19 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <cstring> +#include <limits> #include <optional> #include "common/assert.h" #include "common/bit_util.h" @@ -222,6 +223,13 @@ void Maxwell3D::ProcessMacro(Core::System& system, u32 method, const u32* base_s } } +bool Maxwell3D::ReadMacroMemory(GPUVAddr address, std::span<u32> destination) { + if (!memory_manager.IsFullyMappedRange(address, destination.size_bytes())) return false; + // ReadBlock flushes renderer caches before reading data written by GPU work. + memory_manager.ReadBlock(address, destination.data(), destination.size_bytes()); + return true; +} + void Maxwell3D::RefreshParametersImpl() { if (!Settings::IsGPULevelHigh()) { return; @@ -253,7 +261,10 @@ u32 Maxwell3D::GetMaxCurrentVertices() { const auto& limit = regs.vertex_stream_limits[index]; const GPUVAddr gpu_addr_begin = array.Address(); const GPUVAddr gpu_addr_end = limit.Address() + 1; - const u32 address_size = static_cast<u32>(gpu_addr_end - gpu_addr_begin); + const u64 stream_size = GetNativeMacroEngine().native_turing + ? regs.NativeVertexStreamSize(index) + : gpu_addr_end - gpu_addr_begin; + const u32 address_size = static_cast<u32>(std::min<u64>(stream_size, std::numeric_limits<u32>::max())); num_vertices = (std::max)( num_vertices, address_size / (std::max)(attribute.SizeInBytes(), array.stride.Value())); break; @@ -305,6 +316,9 @@ void Maxwell3D::ProcessDirtyRegisters(u32 method, u32 argument) { } void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argument, bool is_last_call) { + if (GetNativeMacroEngine().native_turing && method == MAXWELL3D_REG_INDEX(clear_surface)) { + LOG_INFO(HW_GPU, "Native GPU surface clear: value={:#x}", argument); + } switch (method) { case MAXWELL3D_REG_INDEX(wait_for_idle): return rasterizer->WaitForIdle(); @@ -312,9 +326,9 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume shadow_state.shadow_ram_control = static_cast<Regs::ShadowRamControl>(nonshadow_argument); return; case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): - return macro_engine.ClearCode(regs.load_mme.instruction_ptr); + return GetNativeMacroEngine().ClearCode(regs.load_mme.instruction_ptr); case MAXWELL3D_REG_INDEX(load_mme.instruction): - return macro_engine.AddCode(regs.load_mme.instruction_ptr, argument); + return GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr, argument); case MAXWELL3D_REG_INDEX(load_mme.start_address): return ProcessMacroBind(argument); case MAXWELL3D_REG_INDEX(falcon[4]): @@ -378,10 +392,10 @@ void Maxwell3D::CallMacroMethod(Core::System& system, u32 method, const std::vec // Lookup the macro offset const u32 entry = - ((method - MacroRegistersStart) >> 1) % static_cast<u32>(macro_positions.size()); + ((method - MacroRegistersStart) >> 1) % static_cast<u32>(GetNativeMacroPositions().size()); // Execute the current macro. - macro_engine.Execute(system, *this, macro_positions[entry], parameters); + GetNativeMacroEngine().Execute(system, *this, GetNativeMacroPositions()[entry], parameters); draw_manager.DrawDeferred(*this); } @@ -453,11 +467,11 @@ void Maxwell3D::CallMultiMethod(Core::System& system, u32 method, const u32* bas } void Maxwell3D::ProcessMacroUpload(u32 data) { - macro_engine.AddCode(regs.load_mme.instruction_ptr++, data); + GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr++, data); } void Maxwell3D::ProcessMacroBind(u32 data) { - macro_positions[regs.load_mme.start_address_ptr++] = data; + GetNativeMacroPositions()[regs.load_mme.start_address_ptr++] = data; } void Maxwell3D::ProcessFirmwareCall4() { @@ -586,6 +600,9 @@ void Maxwell3D::ProcessCounterReset() { } void Maxwell3D::ProcessSyncPoint() { + if (GetNativeMacroEngine().native_turing) { + LOG_DEBUG(HW_GPU, "Native 3D syncpoint increment: id={}, raw={:#x}", regs.sync_info.sync_point.Value(), regs.reg_array[MAXWELL3D_REG_INDEX(sync_info)]); + } const u32 sync_point = regs.sync_info.sync_point.Value(); [[maybe_unused]] const u32 cache_flush = regs.sync_info.clean_l2.Value(); rasterizer->SignalSyncPoint(sync_point); diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 0cab2ae986..9533626ecb 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -2605,6 +2605,12 @@ public: u32 stop_shader_mask; }; + // Native C597/C797 use byte counts at 0x600 rather than legacy end addresses. + u64 NativeVertexStreamSize(std::size_t index) const { + const std::size_t method = 0x600 / sizeof(u32) + index * 2; + return (u64{reg_array[method] & 0xffU} << 32) | reg_array[method + 1]; + } + // clang-format off union { struct { @@ -3129,6 +3135,17 @@ public: /// Reads a register value located at the input method address u32 GetRegisterValue(u32 method) const; + void SetNativeTuringMacroMode(bool enabled) { + GetNativeMacroEngine().native_turing = enabled; + upload_state.SetNativePartialTransfers(enabled); + } + void SetNativeMacroContext(std::shared_ptr<NativeMacroContext> context) { + native_macro_context = std::move(context); + } + MacroEngine& GetNativeMacroEngine() { return native_macro_context ? native_macro_context->engine : macro_engine; } + std::span<u32> GetNativeMacroPositions() { return native_macro_context ? std::span<u32>{native_macro_context->positions} : std::span<u32>{macro_positions}; } + + /// Write the value to the register identified by method. void CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) override; @@ -3180,6 +3197,8 @@ public: return current_macro_dirty; } + bool ReadMacroMemory(GPUVAddr address, std::span<u32> destination); + u32 GetMaxCurrentVertices(); size_t EstimateIndexBufferSize(); @@ -3264,6 +3283,7 @@ private: std::vector<u32> macro_params; /// Interpreter for the macro codes uploaded to the GPU. + std::shared_ptr<NativeMacroContext> native_macro_context; MacroEngine macro_engine; Upload::State upload_state; diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp index ed4675ed12..7c8872e209 100644 --- a/src/video_core/engines/maxwell_dma.cpp +++ b/src/video_core/engines/maxwell_dma.cpp @@ -14,6 +14,7 @@ #include "video_core/engines/maxwell_dma.h" #include "video_core/guest_memory.h" #include "video_core/memory_manager.h" +#include "video_core/native_query_trace.h" #include "video_core/renderer_base.h" #include "video_core/textures/decoders.h" @@ -58,6 +59,12 @@ void MaxwellDMA::CallMultiMethod(Core::System& system, u32 method, const u32* ba } void MaxwellDMA::Launch() { + if (memory_manager.UsesNativeTiling()) { + const GPUVAddr destination = regs.offset_out; + VideoCommon::TraceNativeQuery(10, memory_manager.GetID(), destination, + memory_manager.GpuToCpuAddress(destination).value_or(0), + regs.line_count, static_cast<GPUVAddr>(regs.offset_in), regs.line_length_in); + } LOG_TRACE(Render_OpenGL, "DMA copy {:#x} -> {:#x}", static_cast<GPUVAddr>(regs.offset_in), GPUVAddr(regs.offset_out)); // TODO(Subv): Perform more research and implement all features of this engine. @@ -219,7 +226,7 @@ void MaxwellDMA::CopyBlockLinearToPitch() { UnswizzleSubrect(tmp_write_buffer, tmp_read_buffer, bytes_per_pixel, width, height, depth, x_offset, src_params.origin.y, x_elements, regs.line_count, block_height, - block_depth, dst_operand.pitch); + block_depth, dst_operand.pitch, memory_manager.UsesNativeTiling()); } void MaxwellDMA::CopyPitchToBlockLinear() { @@ -284,7 +291,7 @@ void MaxwellDMA::CopyPitchToBlockLinear() { // If the input is linear and the output is tiled, swizzle the input and copy it over. SwizzleSubrect(tmp_write_buffer, tmp_read_buffer, bytes_per_pixel, width, height, depth, x_offset, dst_params.origin.y, x_elements, regs.line_count, block_height, - block_depth, regs.pitch_in); + block_depth, regs.pitch_in, memory_manager.UsesNativeTiling()); } void MaxwellDMA::CopyBlockLinearToBlockLinear() { @@ -337,11 +344,11 @@ void MaxwellDMA::CopyBlockLinearToBlockLinear() { UnswizzleSubrect(intermediate_buffer, tmp_read_buffer, bytes_per_pixel, src_width, src.height, src.depth, src_x_offset, src.origin.y, x_elements, regs.line_count, - src.block_size.height, src.block_size.depth, pitch); + src.block_size.height, src.block_size.depth, pitch, memory_manager.UsesNativeTiling()); SwizzleSubrect(tmp_write_buffer, intermediate_buffer, bytes_per_pixel, dst_width, dst.height, dst.depth, dst_x_offset, dst.origin.y, x_elements, regs.line_count, - dst.block_size.height, dst.block_size.depth, pitch); + dst.block_size.height, dst.block_size.depth, pitch, memory_manager.UsesNativeTiling()); } void MaxwellDMA::ReleaseSemaphore() { diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 0b05d31718..9031ea75df 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include <cstdlib> + #include "common/assert.h" #include "common/logging.h" #include "common/settings.h" @@ -18,6 +20,7 @@ #include "video_core/engines/puller.h" #include "video_core/gpu.h" #include "video_core/memory_manager.h" +#include "video_core/native_query_trace.h" #include "video_core/rasterizer_interface.h" namespace Tegra::Engines { @@ -27,6 +30,19 @@ namespace { constexpr u32 Gf100BindClassMask = 0xffff; constexpr u32 Gf100BindValidMask = 0x1f0000 | Gf100BindClassMask; +u32 ReadNativeSemaphoreWord(DmaPusher& dma_pusher, GPUVAddr address) { + const auto cpu_address = dma_pusher.memory_manager.GpuToCpuAddress(address); + if (cpu_address && dma_pusher.rasterizer->MustFlushRegion(*cpu_address, sizeof(u32), VideoCommon::CacheType::BufferCache)) { + dma_pusher.rasterizer->FlushRegion(*cpu_address, sizeof(u32), VideoCommon::CacheType::BufferCache); + const u32 value = dma_pusher.memory_manager.Read<u32>(address); + LOG_WARNING(HW_GPU, "Native semaphore buffer readback: address={:#x}, current={}", address, value); + VideoCommon::TraceNativeQuery(6, dma_pusher.memory_manager.GetID(), address, *cpu_address, 0, value, 0); + return value; + } + return dma_pusher.memory_manager.Read<u32>(address); +} + + } // Anonymous namespace void Puller::ProcessBindMethod(DmaPusher& dma_pusher, const MethodCall& method_call) { @@ -36,18 +52,39 @@ void Puller::ProcessBindMethod(DmaPusher& dma_pusher, const MethodCall& method_c engine &= Gf100BindClassMask; } + if (dma_pusher.channel_state.native_abi) dma_pusher.memory_manager.EnableNativeTiling(); const auto engine_id = static_cast<EngineID>(engine); bound_engines[method_call.subchannel] = engine_id; switch (engine_id) { case EngineID::FERMI_TWOD_A: dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->fermi_2d, method_call.subchannel, EngineTypes::Fermi2D); break; + case EngineID::AMPERE_B: + if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; } + dma_pusher.channel_state.payload->maxwell_3d.SetNativeTuringMacroMode(true); + dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->maxwell_3d, method_call.subchannel, EngineTypes::Maxwell3D); + break; case EngineID::MAXWELL_B: + dma_pusher.channel_state.payload->maxwell_3d.SetNativeTuringMacroMode(false); dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->maxwell_3d, method_call.subchannel, EngineTypes::Maxwell3D); break; + case EngineID::AMPERE_COMPUTE_B: + if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; } + [[fallthrough]]; case EngineID::KEPLER_COMPUTE_B: + if (engine_id == EngineID::AMPERE_COMPUTE_B && dma_pusher.channel_state.native_macro_context) { + auto& context = *dma_pusher.channel_state.native_macro_context; + dma_pusher.channel_state.payload->kepler_compute.SetNativeMacroMode(&context.engine, context.positions, &context); + } else { + dma_pusher.channel_state.payload->kepler_compute.SetNativeMacroMode( + engine_id == EngineID::AMPERE_COMPUTE_B ? &dma_pusher.channel_state.payload->maxwell_3d.GetNativeMacroEngine() : nullptr, + dma_pusher.channel_state.payload->maxwell_3d.GetNativeMacroPositions()); + } dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->kepler_compute, method_call.subchannel, EngineTypes::KeplerCompute); break; + case EngineID::AMPERE_DMA_COPY_B: + if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; } + [[fallthrough]]; case EngineID::MAXWELL_DMA_COPY_A: dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->maxwell_dma, method_call.subchannel, EngineTypes::MaxwellDMA); break; @@ -64,6 +101,14 @@ void Puller::ProcessBindMethod(DmaPusher& dma_pusher, const MethodCall& method_c } void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { + if (dma_pusher.channel_state.native_abi && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { + static u32 fence_traces{}; + if (fence_traces++ < 512) { + LOG_INFO(HW_GPU, "Native fence action: channel={}, id={}, op={}, value={}", + dma_pusher.channel_state.bind_id, regs.fence_action.syncpoint_id.Value(), + static_cast<u32>(regs.fence_action.op.Value()), regs.fence_value); + } + } switch (regs.fence_action.op) { case Puller::FenceOperation::Acquire: // UNIMPLEMENTED_MSG("Channel Scheduling pending."); @@ -71,6 +116,13 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { dma_pusher.rasterizer->ReleaseFences(); break; case Puller::FenceOperation::Increment: + if (dma_pusher.channel_state.native_abi) { + const auto& context = dma_pusher.channel_state.native_macro_context; + if (context && (context->engine.native_execution_failed || context->compute_engine.native_execution_failed)) { + LOG_ERROR(HW_GPU, "Native submission completion blocked by failed GPU work: syncpoint={}", regs.fence_action.syncpoint_id.Value()); + break; + } + } dma_pusher.rasterizer->SignalSyncPoint(regs.fence_action.syncpoint_id); break; default: @@ -79,16 +131,68 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { } } +bool Puller::CanResumeNativeAcquire(DmaPusher& dma_pusher) const { + return native_acquire && !dma_pusher.memory_manager.IsShuttingDown() && + native_acquire->Matches(ReadNativeSemaphoreWord(dma_pusher, native_acquire->address)); +} + +bool Puller::ResumeNativeAcquire(DmaPusher& dma_pusher) { + if (!native_acquire) return true; + if (!CanResumeNativeAcquire(dma_pusher)) return false; + LOG_WARNING(HW_GPU, "Native channel resumed after semaphore: channel={}, address={:#x}, current={}, required={}", + dma_pusher.channel_state.bind_id, native_acquire->address, + dma_pusher.memory_manager.Read<u32>(native_acquire->address), native_acquire->value); + VideoCommon::TraceNativeQuery(5, dma_pusher.memory_manager.GetID(), native_acquire->address, + dma_pusher.memory_manager.GpuToCpuAddress(native_acquire->address).value_or(0), + 0, native_acquire->value, native_acquire->operation); + native_acquire.reset(); + return true; +} + void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { const auto semaphoreOperationMask = 0xF; const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask); if (op == GpuSemaphoreOperation::WriteLong) { const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()}; const u32 payload = regs.semaphore_sequence; - dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::HasTimeout, payload, 0); + const auto properties = dma_pusher.channel_state.native_abi && (regs.semaphore_trigger & (1U << 24)) + ? VideoCommon::QueryPropertiesFlags::IsAFence : VideoCommon::QueryPropertiesFlags::HasTimeout; + dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0); } else { do { - const u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress()); + u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress()); + if (dma_pusher.channel_state.native_abi) { + word = ReadNativeSemaphoreWord(dma_pusher, regs.semaphore_address.SemaphoreAddress()); + const auto ready = [&] { + if (op == GpuSemaphoreOperation::AcquireEqual) return word == regs.semaphore_sequence; + if (op == GpuSemaphoreOperation::AcquireGequal) return word >= regs.semaphore_sequence; + if (op == GpuSemaphoreOperation::AcquireMask) return (word & regs.semaphore_sequence) != 0; + return false; + }; + if (!ready()) { + dma_pusher.rasterizer->ReleaseFences(true); + word = ReadNativeSemaphoreWord(dma_pusher, regs.semaphore_address.SemaphoreAddress()); + static u32 native_wait_traces{}; + if (ready() && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && native_wait_traces++ < 8) { + LOG_WARNING(HW_GPU, "Native semaphore completed after pending GPU fences: address={:#x}, current={}, required={}", + regs.semaphore_address.SemaphoreAddress(), word, regs.semaphore_sequence); + } + } + if (!ready()) { + if (!NativeSemaphoreWait::Supported(static_cast<u32>(op))) { + if (auto context = dma_pusher.channel_state.native_macro_context) context->engine.native_execution_failed = true; + LOG_ERROR(HW_GPU, "Unsupported native semaphore operation {}", static_cast<u32>(op)); + } else { + native_acquire = NativeSemaphoreWait{regs.semaphore_address.SemaphoreAddress(), regs.semaphore_sequence, static_cast<u32>(op)}; + VideoCommon::TraceNativeQuery(4, dma_pusher.memory_manager.GetID(), native_acquire->address, + dma_pusher.memory_manager.GpuToCpuAddress(native_acquire->address).value_or(0), + 0, native_acquire->value, native_acquire->operation); + LOG_WARNING(HW_GPU, "Native channel suspended for semaphore: channel={}, address={:#x}, current={}, required={}", + dma_pusher.channel_state.bind_id, native_acquire->address, word, native_acquire->value); + } + } + break; + } regs.acquire_source = true; regs.acquire_value = regs.semaphore_sequence; if (op == GpuSemaphoreOperation::AcquireEqual) { @@ -211,12 +315,19 @@ void Puller::CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_ca case EngineID::FERMI_TWOD_A: dma_pusher.channel_state.payload->fermi_2d.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); break; + case EngineID::AMPERE_B: case EngineID::MAXWELL_B: dma_pusher.channel_state.payload->maxwell_3d.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); break; + case EngineID::AMPERE_COMPUTE_B: + if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; } + [[fallthrough]]; case EngineID::KEPLER_COMPUTE_B: dma_pusher.channel_state.payload->kepler_compute.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); break; + case EngineID::AMPERE_DMA_COPY_B: + if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; } + [[fallthrough]]; case EngineID::MAXWELL_DMA_COPY_A: dma_pusher.channel_state.payload->maxwell_dma.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall()); break; @@ -239,12 +350,19 @@ void Puller::CallEngineMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subcha case EngineID::FERMI_TWOD_A: dma_pusher.channel_state.payload->fermi_2d.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); break; + case EngineID::AMPERE_B: case EngineID::MAXWELL_B: dma_pusher.channel_state.payload->maxwell_3d.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); break; + case EngineID::AMPERE_COMPUTE_B: + if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; } + [[fallthrough]]; case EngineID::KEPLER_COMPUTE_B: dma_pusher.channel_state.payload->kepler_compute.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); break; + case EngineID::AMPERE_DMA_COPY_B: + if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; } + [[fallthrough]]; case EngineID::MAXWELL_DMA_COPY_A: dma_pusher.channel_state.payload->maxwell_dma.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending); break; diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index e8994f5640..8dde8440bd 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -8,11 +8,13 @@ #include <array> #include <cstddef> +#include <optional> #include <vector> #include "common/bit_field.h" #include "common/common_funcs.h" #include "common/common_types.h" #include "video_core/engines/engine_interface.h" +#include "video_core/native_semaphore.h" namespace Core { class System; @@ -23,6 +25,9 @@ class MemoryManager; class DmaPusher; enum class EngineID { + AMPERE_B = 0xc797, + AMPERE_COMPUTE_B = 0xc7c0, + AMPERE_DMA_COPY_B = 0xc7b5, NV01_TIMER = 0x0004, FERMI_TWOD_A = 0x902D, // 2D Engine MAXWELL_B = 0xB197, // 3D Engine @@ -70,6 +75,10 @@ public: BitField<8, 24, u32> syncpoint_id; }; + [[nodiscard]] bool HasPendingNativeAcquire() const { return native_acquire.has_value(); } + [[nodiscard]] bool CanResumeNativeAcquire(DmaPusher& dma_pusher) const; + bool ResumeNativeAcquire(DmaPusher& dma_pusher); + void CallMethod(DmaPusher& dma_pusher, const MethodCall& method_call); void CallMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending); void BindRasterizer(DmaPusher& dma_pusher, VideoCore::RasterizerInterface* rasterizer); @@ -127,6 +136,8 @@ private: void ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher); [[nodiscard]] bool ExecuteMethodOnEngine(DmaPusher& dma_pusher, u32 method); + std::optional<NativeSemaphoreWait> native_acquire; + /// Mapping of command subchannels to their bound engine ids std::array<EngineID, 8> bound_engines{}; diff --git a/src/video_core/framebuffer_config.h b/src/video_core/framebuffer_config.h index 1fc70583f9..bb041cf12f 100644 --- a/src/video_core/framebuffer_config.h +++ b/src/video_core/framebuffer_config.h @@ -37,6 +37,7 @@ struct FramebufferConfig { Service::android::BufferTransformFlags transform_flags{}; Common::Rectangle<int> crop_rect{}; BlendMode blending{}; + bool native_tiling{}; u32 layer_stack_mask{Service::Nvnflinger::DefaultLayerStackMask}; }; diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 1621b24125..e8a0685e72 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -252,6 +252,8 @@ struct GPU::Impl { } void RequestComposite(std::vector<Tegra::FramebufferConfig>&& layers, std::vector<Service::Nvidia::NvFence>&& fences) { + static u32 trace_composites{}; + if (trace_composites++ < 16) LOG_INFO(HW_GPU, "Composite requested: layers={}, fences={}", layers.size(), fences.size()); const size_t num_fences{fences.size()}; size_t current_request_counter{}; if (num_fences != 0) { @@ -281,6 +283,7 @@ struct GPU::Impl { } free_swap_counters.push_back(current_request_counter); } + LOG_DEBUG(HW_GPU, "Composite fences complete"); renderer->Composite(composite_layers); }; for (size_t i = 0; i < num_fences; i++) { diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index a01375e7be..a8bf6cc848 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -4,6 +4,9 @@ // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <chrono> +#include <thread> + #include "common/assert.h" #include "common/scope_exit.h" #include "common/settings.h" @@ -36,7 +39,15 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten auto current_context = context.Acquire(); CommandDataContainer next; while (!stop_token.stop_requested()) { - state.queue.PopWait(next, stop_token); + if (scheduler.HasPendingNativeChannels()) { + scheduler.ResumePending(system.GPU()); + if (!state.queue.TryPop(next)) { + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + continue; + } + } else { + state.queue.PopWait(next, stop_token); + } if (stop_token.stop_requested()) { break; } diff --git a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp index 2a3446ca66..bbe73acd1a 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp @@ -39,6 +39,7 @@ UNIFORM(4) uint block_size; UNIFORM(5) uint x_shift; UNIFORM(6) uint block_height; UNIFORM(7) uint block_height_mask; +UNIFORM(8) uint native_tiling; END_PUSH_CONSTANTS #if HAS_EXTENDED_TYPES @@ -77,7 +78,12 @@ uint SwizzleTable(uint pos) { uint SwizzleOffset(uvec2 pos) { pos = pos & SWIZZLE_MASK; - return SwizzleTable(pos.y * 64 + pos.x); + uint offset = SwizzleTable(pos.y * 64 + pos.x); + if (native_tiling != 0) { + uint difference = ((offset >> 5) ^ (offset >> 6)) & 1; + offset ^= (difference << 5) | (difference << 6); + } + return offset; } uvec4 ReadTexel(uint offset) { diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp index 34e4f93307..8dfaf0cd1a 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp @@ -41,6 +41,7 @@ UNIFORM(6) uint block_height; UNIFORM(7) uint block_height_mask; UNIFORM(8) uint block_depth; UNIFORM(9) uint block_depth_mask; +UNIFORM(10) uint native_tiling; END_PUSH_CONSTANTS #if HAS_EXTENDED_TYPES @@ -79,7 +80,12 @@ uint SwizzleTable(uint pos) { uint SwizzleOffset(uvec2 pos) { pos = pos & SWIZZLE_MASK; - return SwizzleTable(pos.y * 64 + pos.x); + uint offset = SwizzleTable(pos.y * 64 + pos.x); + if (native_tiling != 0) { + uint difference = ((offset >> 5) ^ (offset >> 6)) & 1; + offset ^= (difference << 5) | (difference << 6); + } + return offset; } uvec4 ReadTexel(uint offset) { diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp index e084837f25..28b1dcf599 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp @@ -42,7 +42,7 @@ layout(push_constant) uniform PushConstants { uint block_depth; // Offset 48 uint block_depth_mask; // Offset 52 - int _pad; // Offset 56 + uint native_tiling; // Offset 56 ivec3 destination; // Offset 60 } pc; @@ -59,6 +59,7 @@ BEGIN_PUSH_CONSTANTS UNIFORM(8) uint block_depth; UNIFORM(9) uint block_depth_mask; UNIFORM(10) uvec3 blocks_dim; + UNIFORM(11) uint native_tiling; END_PUSH_CONSTANTS #define pc // Map pc prefix to nothing for OpenGL compatibility #endif @@ -103,7 +104,12 @@ uint SwizzleTable(uint pos) { // --- Helpers --- uint SwizzleOffset(uvec2 pos) { pos &= SWIZZLE_MASK; - return SwizzleTable(pos.y * 64u + pos.x); + uint offset = SwizzleTable(pos.y * 64 + pos.x); + if (pc.native_tiling != 0) { + uint difference = ((offset >> 5) ^ (offset >> 6)) & 1; + offset ^= (difference << 5) | (difference << 6); + } + return offset; } uvec4 ReadTexel(uint offset) { diff --git a/src/video_core/macro.cpp b/src/video_core/macro.cpp index 86f528af15..4364204740 100644 --- a/src/video_core/macro.cpp +++ b/src/video_core/macro.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> +#include <unordered_set> #include <cstring> #include <fstream> #include <optional> @@ -1339,6 +1341,42 @@ static void Dump(u64 hash, std::span<const u32> code, bool decompiled = false) { void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u32 method, std::span<const u32> parameters) { + if (native_turing) { + maxwell3d.RefreshParameters(); + const auto context = reinterpret_cast<std::uintptr_t>(this); + static std::unordered_set<std::uintptr_t> captured_contexts; + if (std::getenv("EDEN_NATIVE_FRAME_CAPTURE") && captured_contexts.insert(context).second) { + std::ofstream dump("native-graphics-mme-" + std::to_string(context) + ".bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code)); + } + LOG_DEBUG(HW_GPU, "Native MME context={:#x}", context); + LOG_DEBUG(HW_GPU, "Native MME start={:#x}, parameters={}, data={:08x}", method, parameters.size(), fmt::join(parameters, ",")); + u32 dma_count{}; + const auto result = native_interpreter.Execute(native_code, method, parameters, + [&](u32 address) { return address < Maxwell3D::Regs::NUM_REGS ? maxwell3d.GetRegisterValue(address) : 0U; }, + [&](u32 address, u32 value) { + if (address >= 0xe00) return false; + if (address == 0x158) dma_count = value; // MME_DMA_READ_FIFOED + maxwell3d.CallMethod(system, address, value, true); + return true; + }, [&]() -> std::optional<std::vector<u32>> { + if (dma_count > 0x10000) return std::nullopt; + std::vector<u32> data(dma_count); + const GPUVAddr address = (u64{maxwell3d.GetRegisterValue(0x154) & 0xff} << 32) | + maxwell3d.GetRegisterValue(0x155); + if (dma_count && !maxwell3d.ReadMacroMemory(address, data)) return std::nullopt; + dma_count = 0; + return data; + }); + if (!result.success) { + native_execution_failed = true; + std::ofstream dump("native-mme-" + std::to_string(method) + ".bin", std::ios::binary); + dump.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code)); + LOG_ERROR(HW_GPU, "Native MME failed: start={:#x}, pc={:#x}, consumed={}/{}, error={}", + method, result.pc, result.consumed, parameters.size(), result.error); + } + return; + } const auto execute_variant = [&system, &maxwell3d, ¶meters, method](AnyCachedMacro& cached) { if (std::holds_alternative<MacroInterpreterImpl>(cached) || diff --git a/src/video_core/macro.h b/src/video_core/macro.h index 90519a5429..52dfe7c82c 100644 --- a/src/video_core/macro.h +++ b/src/video_core/macro.h @@ -7,6 +7,7 @@ #pragma once #include <memory> +#include "video_core/macro_turing.h" #include <span> #include <variant> #include <vector> @@ -223,10 +224,15 @@ struct MacroEngine { MacroEngine(bool is_interpreted_) noexcept : is_interpreted{is_interpreted_} {} // Store the uploaded macro code to compile them when they're called. inline void AddCode(u32 method, u32 data) noexcept { + if (native_turing) { + if (native_write_offset < native_code.size()) native_code[native_write_offset++] = data; + return; + } uploaded_macro_code[method].push_back(data); } // Clear the code associated with a method. inline void ClearCode(u32 method) noexcept { + if (native_turing) { native_write_offset = method < 0x1000 ? method * 3 : static_cast<u32>(native_code.size()); return; } macro_cache.erase(method); uploaded_macro_code.erase(method); } @@ -240,6 +246,19 @@ struct MacroEngine { ::Common::unordered_map<u32, CacheInfo> macro_cache; ::Common::unordered_map<u32, std::vector<u32>> uploaded_macro_code; bool is_interpreted; + bool native_turing{}; + bool native_execution_failed{}; + u32 native_write_offset{}; + std::array<u32, 0x3000> native_code{}; + TuringMacroInterpreter native_interpreter; + +}; + +struct NativeMacroContext { + MacroEngine engine{true}; + std::array<u32, 0x80> positions{}; + MacroEngine compute_engine{true}; + std::array<u32, 0x80> compute_positions{}; }; } // namespace Tegra diff --git a/src/video_core/macro_turing.h b/src/video_core/macro_turing.h new file mode 100644 index 0000000000..18966fc083 --- /dev/null +++ b/src/video_core/macro_turing.h @@ -0,0 +1,237 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +// Instruction semantics adapted from Mesa mme_tu104_sim.c and mme_tu104.c: +// Copyright © 2022 Collabora Ltd. SPDX-License-Identifier: MIT +#pragma once + +#include <algorithm> +#include <array> +#include <bit> +#include <cstdint> +#include <span> +#include <optional> +#include <vector> +#include <string_view> + +namespace Tegra { + +// TU104/GA10x MME has two ALUs and two outputs per 96-bit instruction. +class TuringMacroInterpreter { +public: + using Word = std::uint32_t; + struct Result { + bool success; + Word pc; + std::size_t consumed; + std::string_view error; + }; + + template <typename Read, typename Emit> + Result Execute(std::span<const Word> code, Word start, + std::span<const Word> parameters, Read read, Emit emit) { + return Execute(code, start, parameters, read, emit, + []() -> std::optional<std::vector<Word>> { return std::nullopt; }); + } + + template <typename Read, typename Emit, typename Barrier> + Result Execute(std::span<const Word> code, Word start, + std::span<const Word> parameters, Read read, Emit emit, Barrier barrier) { + std::vector<Word> dma_parameters; + std::span<const Word> fifo = parameters; + std::size_t fifo_index{}; + bool dma_fifo{}; + std::array<Word, 24> regs{}; + Word initialized{}, carry{}, method{}, increment{}; + Word pc = start, loop_count{}, loop_start{}, loop_end{}; + std::size_t consumed{}; + bool end_next{}, stopped{}; + std::string_view error; + const auto fail = [&](std::string_view message) { if (error.empty()) error = message; }; + for (Word steps = 0; steps < 1000000; ++steps) { + if (static_cast<std::size_t>(pc) * 3 + 2 >= code.size()) { + fail("instruction outside uploaded RAM"); + break; + } + const std::array<Word, 3> bits{code[pc * 3 + 2], code[pc * 3 + 1], code[pc * 3]}; + const auto field = [&](Word low, Word count) { + std::uint64_t value = bits[low / 32]; + if (low / 32 < 2) value |= std::uint64_t{bits[low / 32 + 1]} << 32; + return Word((value >> (low % 32)) & ((std::uint64_t{1} << count) - 1)); + }; + const std::array<Word, 2> imm{field(30, 16), field(66, 16)}; + const std::array<Word, 2> op{field(10, 5), field(46, 5)}; + const std::array<Word, 2> dst{field(15, 5), field(51, 5)}; + const std::array<Word, 2> src_a{field(20, 5), field(56, 5)}; + const std::array<Word, 2> src_b{field(25, 5), field(61, 5)}; + const std::array<Word, 2> out_method{field(82, 3), field(89, 3)}; + const std::array<Word, 2> out_emit{field(85, 4), field(92, 4)}; + const Word pred_reg = field(5, 5), pred_mode = field(1, 4); + std::array<Word, 2> load{}, alu{}; + for (Word i = 0; i < 2; ++i) { + const Word reg = 28 + i, out = 3 + i; + const bool needed = pred_reg == reg || src_a[0] == reg || src_a[1] == reg || + src_b[0] == reg || src_b[1] == reg || out_method[0] == out || + out_method[1] == out || out_emit[0] == out || out_emit[1] == out; + if (needed) { + if (fifo_index >= fifo.size()) { fail("parameter FIFO underflow"); break; } + load[i] = fifo[fifo_index++]; + if (!dma_fifo) ++consumed; + } + } + // Both ALUs read the register file before either destination is + // committed. Carry chaining remains explicit between the lanes. + const auto input_regs = regs; + const Word input_initialized = initialized; + const auto get = [&](Word reg, Word lane) -> Word { + if (reg < 24) { + if (!(input_initialized & (1U << reg))) fail("uninitialized register read"); + return input_regs[reg]; + } + switch (reg) { + case 24: return 0; + case 25: return Word(std::int32_t(std::int16_t(imm[lane]))); + case 26: return Word(std::int32_t(std::int16_t(imm[1 - lane]))); + case 27: return (imm[0] << 16) | imm[1]; + case 28: return load[0]; + case 29: return load[1]; + default: fail("invalid source register"); return 0; + } + }; + constexpr std::array<std::string_view, 16> predicates{ + "UUUU", "TTTT", "FFFF", "TTUU", "FFUU", "TFUU", "TUUU", "FUUU", + "UUTT", "UUTF", "UUTU", "UUFT", "UUFF", "UUFU", "UUUT", "UUUF"}; + Word mask = 15; + if (pred_mode) { + const char skipped = get(pred_reg, 0) ? 'T' : 'F'; + mask = 0; + for (Word i = 0; i < 4; ++i) + if (predicates[pred_mode][i] != skipped) mask |= 1U << i; + } + Word next_pc = pc + 1; + bool branch_cancels_end{}; + for (Word lane = 0; lane < 2 && error.empty(); ++lane) { + if (!(mask & (1U << lane))) continue; + const Word x = get(src_a[lane], lane), y = get(src_b[lane], lane); + Word value{}; + const auto condition = [&](Word comparison) { + switch (comparison) { + case 17: case 25: return std::int32_t(x) < std::int32_t(y); + case 18: case 26: return x < y; + case 19: case 27: return std::int32_t(x) <= std::int32_t(y); + case 20: case 28: return x <= y; + default: return x == y; + } + }; + switch (op[lane]) { + case 0: value = x + y; carry = value < x; break; + case 1: value = x + y + carry; break; + case 2: value = x - y; carry = value > x; break; + case 3: value = x - y - carry; break; + case 4: case 6: { + const std::uint64_t a = op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(x))) : x; + const std::uint64_t b = op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(y))) : y; + const auto product = a * b; + value = Word(product); carry = Word(product >> 32); break; + } + case 5: value = carry; break; + case 7: { + if (x != 0x1000 || y != 1) { + fail("unsupported extended operation"); + break; + } + auto data = barrier(); + if (!data) { fail("macro DMA barrier failed"); break; } + if (!data->empty()) { + dma_parameters = std::move(*data); + fifo = dma_parameters; + fifo_index = 0; + dma_fifo = true; + } + break; + } + case 8: value = Word(std::countl_zero(x)); break; + case 9: value = x << (y & 31); break; + case 10: value = x >> (y & 31); break; + case 11: value = Word(std::int32_t(x) >> (y & 31)); break; + case 12: value = x & y; break; + case 13: value = ~(x & y); break; + case 14: value = x | y; break; + case 15: value = x ^ y; break; + case 16: { + const Word dest = (imm[lane] >> 10) & 63, count = (imm[lane] >> 5) & 31; + const Word source = imm[lane] & 31; + if (dest >= 32) { fail("invalid merge destination"); break; } + const Word bits_mask = (1U << count) - 1; + value = (x & ~(bits_mask << dest)) | (((y >> source) & bits_mask) << dest); + break; + } + case 17: case 18: case 19: case 20: case 21: + value = condition(op[lane]) ? ~0U : 0; break; + case 22: value = read((x + y) & 0xffff); break; + case 23: + if (loop_count || imm[lane] < 2) { fail("invalid loop"); break; } + // The native SDK emits zero-count loops around FIFO reads. + // Unlike Mesa's test simulator, these must skip the body. + if (x == 0) { next_pc = pc + imm[lane]; break; } + loop_count = x - 1; loop_start = pc; loop_end = pc + imm[lane] - 1; + break; + case 24: next_pc = pc + (imm[lane] & 0x7fff); break; + case 25: case 26: case 27: case 28: case 29: + if (condition(op[lane]) == bool(imm[lane] & 0x8000)) { + const auto offset = std::int32_t(imm[lane] << 19) >> 19; + if (offset == -4096) stopped = true; + else { + next_pc = Word(std::int32_t(pc) + offset); + // Native back edges use bit 14 to suppress the + // fallthrough termination when the branch is taken. + branch_cancels_end = (imm[lane] & 0x4000) != 0; + } + } + break; + case 30: + if (x >= dram.size()) { fail("data RAM read out of bounds"); break; } + value = dram[x]; break; + case 31: + if (x >= dram.size()) { fail("data RAM write out of bounds"); break; } + dram[x] = y; break; + } + alu[lane] = value; + if (dst[lane] < 24) { regs[dst[lane]] = value; initialized |= 1U << dst[lane]; } + else if (dst[lane] != 24) fail("invalid destination register"); + } + const auto output = [&](Word selector) -> Word { + switch (selector) { + case 1: return alu[0]; case 2: return alu[1]; + case 3: return load[0]; case 4: return load[1]; + case 5: return imm[0]; case 6: return imm[1]; + case 8: return imm[0] >> 12; case 9: return imm[1] >> 12; + case 10: return (imm[0] << 16) | imm[1]; + default: fail("invalid output selector"); return 0; + } + }; + for (Word lane = 0; lane < 2 && error.empty(); ++lane) { + if (!(mask & (4U << lane))) continue; + if (out_method[lane]) { + const Word address = output(out_method[lane]); + method = address & 0xfff; increment = (address >> 12) & 15; + } + if (out_emit[lane]) { + const Word value = output(out_emit[lane]); + if (error.empty() && !emit(method, value)) fail("unsupported emitted method"); + method += increment; + } + } + if (!error.empty()) break; + if (end_next || stopped) return {true, pc, consumed, {}}; + end_next = field(0, 1) != 0 && !branch_cancels_end; + if (loop_count && pc == loop_end) { --loop_count; next_pc = loop_start + 1; } + pc = next_pc; + } + if (error.empty()) error = "instruction limit exceeded"; + return {false, pc, consumed, error}; + } + + std::array<Word, 0xc00> dram{}; +}; + +} // namespace Tegra diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 8ffdcb6276..9fd3ca62e3 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -22,6 +22,11 @@ #include "video_core/texture_cache/util.h" namespace Tegra { + +bool MemoryManager::IsShuttingDown() const { + return system.IsShuttingDown(); +} + using Tegra::Memory::GuestMemoryFlags; std::atomic<size_t> MemoryManager::unique_identifier_generator{}; @@ -56,6 +61,10 @@ MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, GPU : MemoryManager(system_, system_.Host1x().MemoryManager(), address_space_bits_, split_address_, big_page_bits_, page_bits_) {} +void MemoryManager::SetNativeSyncpointShim(GPUVAddr address) { + native_syncpoint_shim.store(address, std::memory_order_release); +} + MemoryManager::~MemoryManager() = default; MemoryManager::EntryType MemoryManager::GetEntry(size_t position, bool is_big_page) const { @@ -231,6 +240,15 @@ std::optional<DAddr> MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t s template <typename T> T MemoryManager::Read(GPUVAddr addr) const { + if constexpr (sizeof(T) == sizeof(u32)) { + const auto shim = native_syncpoint_shim.load(std::memory_order_acquire); + if (shim && addr >= shim && addr - shim < 192 * sizeof(u32) && + Common::IsAligned(addr - shim, sizeof(u32))) { + return static_cast<T>(system.Host1x().GetSyncpointManager().GetGuestSyncpointValue( + static_cast<u32>((addr - shim) / sizeof(u32)))); + } + } + if (auto page_pointer{GetPointer(addr)}; page_pointer) { // NOTE: Avoid adding any extra logic to this fast-path block T value; diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index 08978567d2..79cffae1e4 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -43,6 +43,11 @@ public: u64 big_page_bits_ = 16, u64 page_bits_ = 12); ~MemoryManager(); + void SetNativeSyncpointShim(GPUVAddr address); + void EnableNativeTiling() { native_tiling = true; } + bool UsesNativeTiling() const { return native_tiling; } + bool IsShuttingDown() const; + static constexpr bool HAS_FLUSH_INVALIDATION = true; inline size_t GetID() const noexcept { @@ -178,6 +183,7 @@ private: GPUVAddr gpu_addr, std::size_t size, boost::container::small_vector<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result) const; + std::atomic<GPUVAddr> native_syncpoint_shim{}; Core::System& system; MaxwellDeviceMemoryManager& memory; @@ -220,6 +226,7 @@ private: boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash{}; boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash2{}; + bool native_tiling{}; mutable std::mutex guard; static constexpr size_t continuous_bits = 64; diff --git a/src/video_core/native_qmd.h b/src/video_core/native_qmd.h new file mode 100644 index 0000000000..c9ae3b8cc0 --- /dev/null +++ b/src/video_core/native_qmd.h @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once +#include <array> +#include <cstdint> +#include <string_view> +#include <type_traits> +#include <unordered_map> + +namespace Tegra { + +// Native QMD 3.0 dependency scheduling; nonempty grids require a dispatch backend. +class NativeQmdScheduler { +public: + using Word = std::uint32_t; + using Address = std::uint64_t; + using Descriptor = std::array<Word, 64>; + struct RejectShaderDispatch { bool operator()(const Descriptor&) const { return false; } }; + template <typename Read, typename Release> + std::string_view Signal(Address address, Word argument, Read read, Release release) { + return Signal(address, argument, read, release, RejectShaderDispatch{}); + } + template <typename Read, typename Release, typename Dispatch> + std::string_view Signal(Address address, Word argument, Read read, Release release, Dispatch dispatch) { + struct Pending { Address address; Word action; Word count; }; + std::array<Pending, 256> pending{}; + std::size_t count = 1; + pending[0] = {address, argument & 15, ((argument >> 14) & 0x3ff) + 1}; + for (std::size_t i = 0; i < count; ++i) { + const auto request = pending[i]; + if (states.size() >= 4096 && !states.contains(request.address)) return "QMD cache limit"; + auto& state = states[request.address]; + if (request.action == 3 || request.action == 2) { + if (request.action == 3 || !state.loaded) { + if (!read(request.address, state.descriptor)) return "unmapped QMD"; + const auto& qmd = state.descriptor; + if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue"; + if constexpr (std::is_same_v<Dispatch, RejectShaderDispatch>) { + if (qmd[12] && (qmd[13] & 0xffff) && (qmd[14] & 0xffff)) + return "native shader dispatch required"; + } + state.dependencies = std::int32_t((qmd[4] >> 14) & 0xffff) + state.deferred; + state.deferred = 0; + state.loaded = true; + state.completed = false; + } + state.scheduled = true; + } else if (request.action == 7 || request.action == 12) { + const auto delta = request.action == 7 ? -std::int32_t(request.count) : std::int32_t(request.count); + if (!state.loaded || state.completed) { state.deferred += delta; continue; } + state.dependencies += delta; + } else if (request.action == 0 || request.action == 1) { + continue; + } else { + return "unsupported PCAS action"; + } + if (!state.loaded || !state.scheduled || state.completed || state.dependencies > 0) continue; + if (state.dependencies < 0) return "QMD dependency underflow"; + // The SDK patches release payloads while a scheduled QMD is + // waiting for its remaining dependencies. Fetch them at execution. + if (!read(request.address, state.descriptor)) return "unmapped QMD"; + const auto qmd = state.descriptor; + if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue"; + // Validate both release slots before making either visible. + for (Word slot = 0; slot < 2; ++slot) { + const Word flags = qmd[25 + slot * 4]; + if (!(flags & (1U << 23))) continue; + const bool add = (flags & (1U << 26)) != 0; + if ((flags & (0x3bU << 24)) || (flags >> 30) != 1 || + (add && (flags & (7U << 20)))) return "unsupported QMD release format"; + if (add) { + if constexpr (!std::is_invocable_r_v<bool, Release, Address, Word, Word>) + return "unsupported QMD reduction callback"; + } + } + if (qmd[12] && (qmd[13] & 0xffff) && (qmd[14] & 0xffff) && !dispatch(qmd)) + return "native shader dispatch required"; + state.completed = true; + for (Word slot = 0; slot < 2; ++slot) { + const Word flags = qmd[25 + slot * 4]; + if (!(flags & (1U << 23))) continue; + const Address destination = (Address{flags & 0xff} << 32) | qmd[24 + slot * 4]; + bool released{}; + if constexpr (std::is_invocable_r_v<bool, Release, Address, Word, Word>) { + released = release(destination, qmd[26 + slot * 4], flags); + } else { + released = release(destination, qmd[26 + slot * 4]); + } + if (!released) return "unmapped QMD release"; + } + if (qmd[16] & 1) { + const Word action = (qmd[16] >> 1) & 7; + if (action != 4 && action != 3 && action != 1) return "unsupported dependent QMD action"; + if (count == pending.size()) return "QMD chain limit"; + pending[count++] = {Address{qmd[15]} << 8, action == 4 ? 7U : action == 3 ? 3U : 2U, 1}; + } + if (qmd[16] & (1U << 5)) return "second dependent QMD unsupported"; + } + return {}; + } +private: + struct State { + Descriptor descriptor{}; + std::int32_t dependencies{}; + std::int32_t deferred{}; + bool loaded{}; + bool scheduled{}; + bool completed{}; + }; + std::unordered_map<Address, State> states; +}; +} // namespace Tegra diff --git a/src/video_core/native_query_trace.h b/src/video_core/native_query_trace.h new file mode 100644 index 0000000000..eae258dc04 --- /dev/null +++ b/src/video_core/native_query_trace.h @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include <cstdlib> +#include <fstream> +#include <iomanip> +#include <mutex> +#include "common/common_types.h" + +namespace VideoCommon { +namespace NativeQueryDiagnostics { +struct State { + State() : file{"native-query-trace.csv", std::ios::trunc} { + file << "event,asid,gpu,cpu,type,value,flags\n"; + } + std::mutex mutex; + std::ofstream file; + size_t records{}; +}; +inline State& GetState() { + static State state; + return state; +} +} // namespace NativeQueryDiagnostics + +// Events: 0=request, 1=written, 2=invalidated, 3=unsynchronized, +// 4=channel suspended, 5=channel resumed, 6=buffer readback, +// 7/8=QMD release slots observed, 9=PCAS request, +// 10=DMA destination (value=source, flags=line bytes), 11=inline write. +// Keep this opt-in and bounded. +inline void TraceNativeQuery(u32 event, u64 asid, u64 gpu, u64 cpu, u32 type, u64 value, u32 flags) { + if (!std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) return; + auto& state = NativeQueryDiagnostics::GetState(); + const std::scoped_lock lock{state.mutex}; + if (state.records++ < 1000000 || event == 4 || event == 5) { + state.file << event << ',' << asid << ',' << std::hex << gpu << ',' << cpu << ',' + << type << ',' << value << ',' << flags << std::dec << '\n'; + } + if (event == 4 || event == 5) state.file.flush(); +} +} // namespace VideoCommon diff --git a/src/video_core/native_semaphore.h b/src/video_core/native_semaphore.h new file mode 100644 index 0000000000..1ba7451c51 --- /dev/null +++ b/src/video_core/native_semaphore.h @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include <cstddef> +#include <cstdint> + +namespace Tegra { +struct NativeSemaphoreWait { + std::uint64_t address; + std::uint32_t value; + std::uint32_t operation; + + static constexpr bool Supported(std::uint32_t operation) { + return operation == 1 || operation == 4 || operation == 8; + } + constexpr bool Matches(std::uint32_t current) const { + switch (operation) { + case 1: return current == value; + case 4: return current >= value; + case 8: return (current & value) != 0; + default: return false; + } + } +}; + +// Revisit dependencies after another channel makes progress. A blocked +// channel keeps its FIFO cursor until its real semaphore predicate is true. +template <class Channels, class Ready, class Dispatch> +std::size_t ResumeReadyNativeChannels(const Channels& channels, Ready ready, Dispatch dispatch) { + std::size_t resumed = 0; + for (std::size_t pass = 0; pass < channels.size(); ++pass) { + bool progress = false; + for (const auto& channel : channels) { + if (!ready(channel)) continue; + dispatch(channel); + ++resumed; + progress = true; + } + if (!progress) break; + } + return resumed; +} +} // namespace Tegra diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index b8c1bc4f45..c2fefa6695 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -20,6 +20,7 @@ #include "common/settings.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/gpu.h" +#include "video_core/native_query_trace.h" #include "video_core/host1x/gpu_device_memory_manager.h" #include "video_core/memory_manager.h" #include "video_core/query_cache/bank_base.h" @@ -246,6 +247,9 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type return; } DAddr cpu_addr = *cpu_addr_opt; + const bool native_trace = gpu_memory->UsesNativeTiling(); + const u64 trace_asid = gpu_memory->GetID(); + if (native_trace) TraceNativeQuery(0, trace_asid, addr, cpu_addr, static_cast<u32>(counter_type), payload, static_cast<u32>(flags)); const size_t new_query_id = streamer->WriteCounter(cpu_addr, has_timestamp, payload, subreport); auto* query = streamer->GetQuery(new_query_id); if (is_fence) { @@ -262,14 +266,16 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type u8* pointer_timestamp = impl->device_memory.template GetPointer<u8>(cpu_addr + 8); bool is_synced = (Settings::IsGPUFenceBehaviorDefault() ? !Settings::IsGPULevelHigh() : !Settings::IsGPUFenceBehaviorBalanced() && !Settings::IsGPUFenceBehaviorAccurate()) && is_fence; std::function<void()> operation([this, is_synced, streamer, query_base = query, query_location, - pointer, pointer_timestamp] { + pointer, pointer_timestamp, native_trace, trace_asid, addr, cpu_addr, counter_type] { if (True(query_base->flags & QueryFlagBits::IsInvalidated)) { + if (native_trace) TraceNativeQuery(2, trace_asid, addr, cpu_addr, static_cast<u32>(counter_type), query_base->value, static_cast<u32>(query_base->flags)); if (!is_synced) [[likely]] { impl->pending_unregister.push_back(query_location); } return; } if (False(query_base->flags & QueryFlagBits::IsFinalValueSynced)) [[unlikely]] { + if (native_trace) TraceNativeQuery(3, trace_asid, addr, cpu_addr, static_cast<u32>(counter_type), query_base->value, static_cast<u32>(query_base->flags)); LOG_ERROR(HW_GPU, "Query report value not synchronized. Consider increasing GPU accuracy."); if (!is_synced) [[likely]] { @@ -287,6 +293,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type u32 value = static_cast<u32>(query_base->value); std::memcpy(pointer, &value, sizeof(value)); } + if (native_trace) TraceNativeQuery(1, trace_asid, addr, cpu_addr, static_cast<u32>(counter_type), query_base->value, static_cast<u32>(query_base->flags)); if (!is_synced) [[likely]] { impl->pending_unregister.push_back(query_location); } diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp index 9f4aaf390f..8a11c356ff 100644 --- a/src/video_core/renderer_vulkan/present/layer.cpp +++ b/src/video_core/renderer_vulkan/present/layer.cpp @@ -349,7 +349,7 @@ void Layer::UpdateRawImage(const Tegra::FramebufferConfig& framebuffer, size_t i if (host_ptr) { Tegra::Texture::UnswizzleTexture( mapped_span.subspan(image_offset, linear_size), std::span(host_ptr, tiled_size), - bytes_per_pixel, framebuffer.width, framebuffer.height, 1, block_height_log2, 0); + bytes_per_pixel, framebuffer.width, framebuffer.height, 1, block_height_log2, 0, 1, framebuffer.native_tiling); buffer.Flush(); // Ensure host writes are visible before the GPU copy. } diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index ab58560170..bd092271e4 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -633,7 +633,7 @@ struct alignas(16) BlockLinearUnswizzle3DPushConstants { u32 block_depth; // Offset 48 u32 block_depth_mask; // Offset 52 - s32 _pad; // Offset 56 + u32 native_tiling; // Offset 56 s32 destination[3]; // Offset 60 s32 _pad_end; // Offset 72 @@ -753,6 +753,7 @@ void BlockLinearUnswizzle3DPass::UnswizzleChunk( pc.block_height_mask = params.block_height_mask; pc.block_depth = params.block_depth; pc.block_depth_mask = params.block_depth_mask; + pc.native_tiling = params.native_tiling; pc.blocks_dim[0] = blocks_x; pc.blocks_dim[1] = blocks_y; diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index d4ffcd8317..5e0975a231 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -786,7 +786,7 @@ GraphicsPipeline* PipelineCache::BuiltPipeline(GraphicsPipeline* pipeline) const if (pipeline->IsBuilt()) { return pipeline; } - if (!use_asynchronous_shaders) { + if (maxwell3d->GetNativeMacroEngine().native_turing || !use_asynchronous_shaders) { return pipeline; } // If something is using depth, we can assume that games are not rendering anything which diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 264ffb28b5..740c988550 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -7,6 +7,8 @@ #include <algorithm> #include <array> #include <memory> +#include <cstdlib> +#include <fstream> #include <mutex> #include <fmt/format.h> @@ -14,12 +16,14 @@ #include "video_core/renderer_vulkan/renderer_vulkan.h" #include "common/assert.h" +#include "shader_recompiler/exception.h" #include "common/logging.h" #include "common/scope_exit.h" #include "common/settings.h" #include "video_core/buffer_cache/buffer_cache.h" #include "video_core/gpu_logging/gpu_logging.h" #include "video_core/control/channel_state.h" +#include "video_core/native_query_trace.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/kepler_compute.h" #include "video_core/engines/maxwell_3d.h" @@ -238,6 +242,14 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { SCOPE_EXIT { gpu.TickWork(); }; + bool submitted = false; + SCOPE_EXIT { + auto& native = maxwell3d->GetNativeMacroEngine(); + if (native.native_turing && !submitted) { + native.native_execution_failed = true; + LOG_ERROR(HW_GPU, "Native graphics draw failed; refusing submission completion"); + } + }; FlushWork(); gpu_memory->FlushCaching(); @@ -257,6 +269,45 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { HandleTransformFeedback(); query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable); draw_func(); + submitted = true; + static u32 native_capture_count{}; + if (maxwell3d->GetNativeMacroEngine().native_turing && + std::getenv("EDEN_NATIVE_DRAW_CAPTURE") && native_capture_count < 16) { + const u32 capture = native_capture_count++; + const auto prefix = fmt::format("native-draw-{}", capture); + const auto dump = [&](const std::string& name, GPUVAddr address, size_t size) { + if (!size || !gpu_memory->IsFullyMappedRange(address, size)) return; + std::vector<u8> bytes(size); + gpu_memory->ReadBlockUnsafe(address, bytes.data(), bytes.size()); + std::ofstream file(prefix + name + ".bin", std::ios::binary); + file.write(reinterpret_cast<const char*>(bytes.data()), bytes.size()); + LOG_INFO(HW_GPU, "Native draw capture {}{}: address={:#x}, bytes={}, nonzero={}", + capture, name, address, size, + std::count_if(bytes.begin(), bytes.end(), [](u8 byte) { return byte != 0; })); + }; + const auto& regs = maxwell3d->regs; + const auto& rt = regs.rt[0]; + const size_t size = static_cast<size_t>(rt.array_pitch) * 4; + if (size && size <= 16 * 1024 * 1024) { + if (const auto address = gpu_memory->GpuToCpuAddress(rt.Address(), size)) { + LOG_INFO(HW_GPU, "Native draw {} render target physical address={:#x}", capture, *address); + texture_cache.DownloadMemory(*address, size); + dump("-rt0", rt.Address(), size); + } + } + std::ofstream registers(prefix + "-registers.bin", std::ios::binary); + registers.write(reinterpret_cast<const char*>(®s), sizeof(regs)); + for (size_t stream = 0; stream < 4; ++stream) { + if (regs.vertex_streams[stream].IsEnabled()) + dump(fmt::format("-vertex-{}", stream), regs.vertex_streams[stream].Address(), 128); + } + for (size_t stage : {0U, 4U}) { + for (size_t bank : {2U, 3U}) { + const auto& cb = maxwell3d->state.shader_stages[stage].const_buffers[bank]; + if (cb.enabled) dump(fmt::format("-cbuf-{}-{}", stage, bank), cb.address, std::min<size_t>(cb.size, 128)); + } + } + } } void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) { @@ -588,11 +639,13 @@ void RasterizerVulkan::DispatchCompute() { ComputePipeline* const pipeline{pipeline_cache.CurrentComputePipeline()}; if (!pipeline) { + if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute pipeline unavailable"); return; } std::scoped_lock lock{texture_cache.mutex, buffer_cache.mutex}; if (!pipeline->Configure(*kepler_compute, *gpu_memory, scheduler, buffer_cache, texture_cache)) { + if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute resources unavailable"); return; } @@ -942,6 +995,12 @@ Tegra::Engines::AccelerateDMAInterface& RasterizerVulkan::AccessAccelerateDMA() void RasterizerVulkan::AccelerateInlineToMemory(GPUVAddr address, size_t copy_size, std::span<const u8> memory) { auto cpu_addr = gpu_memory->GpuToCpuAddress(address); + if (gpu_memory->UsesNativeTiling()) { + u32 first_word{}; + if (memory.size() >= sizeof(first_word)) std::memcpy(&first_word, memory.data(), sizeof(first_word)); + VideoCommon::TraceNativeQuery(11, gpu_memory->GetID(), address, cpu_addr.value_or(0), + 0, first_word, static_cast<u32>(copy_size)); + } if (!cpu_addr) [[unlikely]] { gpu_memory->WriteBlock(address, memory.data(), copy_size); return; @@ -969,6 +1028,12 @@ std::optional<FramebufferTextureInfo> RasterizerVulkan::AccelerateDisplay( std::scoped_lock lock{texture_cache.mutex}; const auto [image_view, scaled] = texture_cache.TryFindFramebufferImageView(config, framebuffer_addr); + static u32 display_trace_count{}; + if ((std::getenv("EDEN_NATIVE_FRAME_CAPTURE") || std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) && display_trace_count++ < 64) { + LOG_INFO(HW_GPU, "Native display source: address={:#x}, offset={:#x}, dimensions={}x{}, stride={}, accelerated={}, mask={:#x}", + framebuffer_addr, config.offset, config.width, config.height, pixel_stride, + image_view != nullptr, config.layer_stack_mask); + } if (!image_view) { return {}; } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 583cb6edaf..29afaac01b 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -2085,6 +2085,11 @@ void Image::DownloadMemory(VkBuffer buffer, size_t offset, void Image::DownloadMemory(std::span<VkBuffer> buffers_span, std::span<size_t> offsets_span, std::span<const VideoCommon::BufferImageCopy> copies) { + if (!original_image) { + LOG_CRITICAL(Render_Vulkan, "Missing image at download: gpu={:#x}, cpu={:#x}, type={}, format={}, flags={:#x}, size={}, dimensions={}x{}", + gpu_addr, cpu_addr, static_cast<u32>(info.type), static_cast<u32>(info.format), + static_cast<u32>(flags), unswizzled_size_bytes, info.size.width, info.size.height); + } const bool is_rescaled = True(flags & ImageFlagBits::Rescaled); if (is_rescaled) { ScaleDown(); diff --git a/src/video_core/shader_cache.cpp b/src/video_core/shader_cache.cpp index f1aff2810f..75e5432ab2 100644 --- a/src/video_core/shader_cache.cpp +++ b/src/video_core/shader_cache.cpp @@ -11,6 +11,7 @@ #include "common/assert.h" #include "shader_recompiler/frontend/maxwell/control_flow.h" +#include "shader_recompiler/exception.h" #include "shader_recompiler/object_pool.h" #include "video_core/control/channel_state.h" #include "video_core/dirty_flags.h" @@ -43,7 +44,7 @@ void ShaderCache::SyncGuestHost() { ShaderCache::ShaderCache(Tegra::MaxwellDeviceMemoryManager& device_memory_) : device_memory{device_memory_} {} -bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { +bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { auto& dirty{maxwell3d->dirty.flags}; if (!dirty[VideoCommon::Dirty::Shaders]) { return last_shaders_valid; @@ -51,6 +52,7 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { dirty[VideoCommon::Dirty::Shaders] = false; const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()}; + const bool native = maxwell3d->GetNativeMacroEngine().native_turing; for (size_t index = 0; index < Tegra::Engines::Maxwell3D::Regs::MaxShaderProgram; ++index) { if (!maxwell3d->regs.IsShaderConfigEnabled(index)) { unique_hashes[index] = 0; @@ -63,7 +65,9 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { unique_hashes[index] = 0; continue; } - const GPUVAddr shader_addr{base_addr + shader_config.offset}; + const GPUVAddr shader_addr{native + ? (static_cast<u64>(shader_config.reserved[0] & 0xff) << 32) | shader_config.reserved[1] + : base_addr + shader_config.offset}; const std::optional<VAddr> cpu_shader_addr{gpu_memory->GpuToCpuAddress(shader_addr)}; if (!cpu_shader_addr) { LOG_ERROR(HW_GPU, "Invalid GPU address for shader {:#016x}", shader_addr); @@ -72,8 +76,8 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { } const ShaderInfo* shader_info{TryGet(*cpu_shader_addr)}; if (!shader_info) { - const u32 start_address{shader_config.offset}; - GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, base_addr, start_address}; + const u32 start_address{native ? 0 : shader_config.offset}; + GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, native ? shader_addr : base_addr, start_address}; shader_info = MakeShaderInfo(env, *cpu_shader_addr); } shader_infos[index] = shader_info; @@ -81,6 +85,13 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) { } last_shaders_valid = true; return true; +} catch (const Shader::Exception& error) { + auto& native = maxwell3d->GetNativeMacroEngine(); + if (!native.native_turing) throw; + native.native_execution_failed = true; + last_shaders_valid = false; + LOG_ERROR(HW_GPU, "Native graphics shader rejected: {}", error.what()); + return false; } const ShaderInfo* ShaderCache::ComputeShader() { @@ -103,14 +114,19 @@ void ShaderCache::GetGraphicsEnvironments(GraphicsEnvironments& result, const std::array<u64, NUM_PROGRAMS>& unique_hashes) { size_t env_index{}; const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()}; + const bool native = maxwell3d->GetNativeMacroEngine().native_turing; for (size_t index = 0; index < NUM_PROGRAMS; ++index) { if (unique_hashes[index] == 0) { continue; } const auto program{static_cast<Tegra::Engines::Maxwell3D::Regs::ShaderType>(index)}; auto& env{result.envs[index]}; - const u32 start_address{maxwell3d->regs.pipelines[index].offset}; - env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, base_addr, start_address}; + const auto& config = maxwell3d->regs.pipelines[index]; + const u32 start_address{native ? 0 : config.offset}; + const GPUVAddr address = native + ? (static_cast<u64>(config.reserved[0] & 0xff) << 32) | config.reserved[1] + : base_addr; + env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, address, start_address}; env.SetCachedSize(shader_infos[index]->size_bytes); result.env_ptrs[env_index++] = &env; } @@ -241,7 +257,7 @@ const ShaderInfo* ShaderCache::MakeShaderInfo(GenericEnvironment& env, VAddr cpu info->unique_hash = env.CalculateHash(); info->size_bytes = env.ReadSizeBytes(); } - const size_t size_bytes{info->size_bytes}; + const size_t size_bytes{env.SourceSizeBytes() ? env.SourceSizeBytes() : info->size_bytes}; const ShaderInfo* const result{info.get()}; Register(std::move(info), cpu_addr, size_bytes); return result; diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp index 4d5db7e511..4a7eff3008 100644 --- a/src/video_core/shader_environment.cpp +++ b/src/video_core/shader_environment.cpp @@ -5,7 +5,11 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <chrono> +#include <cstdlib> +#include <thread> #include <filesystem> +#include <cstring> #include <fstream> #include <memory> #include <optional> @@ -21,6 +25,11 @@ #include "common/settings.h" #include <ranges> #include "shader_recompiler/environment.h" +#include "shader_recompiler/exception.h" +#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h" +#include "shader_recompiler/frontend/ampere/program_override.h" +#include "shader_recompiler/frontend/ampere/program_capture.h" +#include "shader_recompiler/frontend/ampere/program_address_map.h" #include "video_core/engines/kepler_compute.h" #include "video_core/memory_manager.h" #include "video_core/shader_environment.h" @@ -149,11 +158,26 @@ u64 GenericEnvironment::ReadInstruction(u32 address) { if (address >= cached_lowest && address < cached_highest) { return code[(address - cached_lowest) / INST_SIZE]; } + if (native_source_size) { + throw Shader::RuntimeError("Lowered native instruction address out of range: {:#x}", address); + } has_unbound_instructions = true; return gpu_memory->Read<u64>(program_base + address); } +u32 GenericEnvironment::TranslateProgramAddress(u32 address) const { + if (const auto translated = Shader::Ampere::NativeProgramAddress(code, address)) return *translated; + if (native_source_size || Shader::Ampere::HasNativeAddressMap(code)) + throw Shader::RuntimeError("Invalid native indirect branch address {:#x}", address); + return address; +} + std::optional<u64> GenericEnvironment::Analyze() { + if (native_source_size) { + // Include the converted header and a version tag to isolate this experimental ABI. + return Common::CityHash64(reinterpret_cast<const char*>(code.data()), + code.size() * sizeof(u64)) ^ 0x4f554e4345000002ULL; + } const std::optional<u64> size{TryFindSize()}; if (!size) { return std::nullopt; @@ -164,6 +188,12 @@ std::optional<u64> GenericEnvironment::Analyze() { } void GenericEnvironment::SetCachedSize(size_t size_bytes) { + if (native_source_size) { + if (size_bytes != CachedSizeBytes()) { + throw Shader::RuntimeError("Native shader changed while constructing pipeline"); + } + return; + } cached_lowest = start_address; cached_highest = start_address + static_cast<u32>(size_bytes); code.resize(CachedSizeWords()); @@ -291,6 +321,119 @@ Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, return entry; } +void GenericEnvironment::InitializeNativeCode(u32 source_header_size) { + using Shader::Ampere::Instruction; + const u32 native_header_size = source_header_size; + constexpr u32 maximum_instructions = Shader::Ampere::ProgramCaptureState::MaximumInstructions; + std::vector<Instruction> native; + Shader::Ampere::ProgramCaptureState capture; + for (u32 index = 0; index < maximum_instructions; ++index) { + const GPUVAddr address = program_base + native_header_size + index * sizeof(Instruction); + if (!gpu_memory->IsFullyMappedRange(address, sizeof(Instruction))) { + if (const char* directory = std::getenv("EDEN_NATIVE_SHADER_CAPTURE_DIR")) { + std::filesystem::create_directories(directory); + std::ofstream dump(std::filesystem::path{directory} / + fmt::format("{:016x}.capture.bin", program_base), std::ios::binary); + dump.write(reinterpret_cast<const char*>(native.data()), native.size() * sizeof(Instruction)); + } + throw Shader::RuntimeError("Unmapped native shader instruction at {:#x} (program={:#x}, index={})", + address, program_base, index); + } + auto& instruction = native.emplace_back(); + gpu_memory->ReadBlock(address, &instruction, sizeof(instruction)); + const auto status = capture.Observe(instruction, index); + if (status == Shader::Ampere::ProgramCaptureState::Result::InvalidTarget) + throw Shader::RuntimeError("Native shader branch outside capture bounds"); + if (status == Shader::Ampere::ProgramCaptureState::Result::Complete) break; + } + auto lowered = Shader::Ampere::LowerToMaxwell(native); + if (!lowered) { + if (const char* directory = std::getenv("EDEN_NATIVE_SHADER_CAPTURE_DIR")) { + std::filesystem::create_directories(directory); + std::ofstream dump(std::filesystem::path{directory} / + fmt::format("{:016x}.capture.bin", program_base), std::ios::binary); + dump.write(reinterpret_cast<const char*>(native.data()), native.size() * sizeof(Instruction)); + } + } + if (!lowered) { + if (const char* directory = std::getenv("EDEN_NATIVE_SHADER_OVERRIDE_DIR")) { + const auto key = fmt::format("{:016x}", Shader::Ampere::NativeProgramKey(native)); + const auto folder = std::filesystem::path{directory}; + std::filesystem::create_directories(folder); + const auto source_path = folder / (key + ".source.bin"); + const auto override_path = folder / (key + ".override.bin"); + { + std::ofstream dump(source_path, std::ios::binary); + dump.write(reinterpret_cast<const char*>(native.data()), native.size() * sizeof(Shader::Ampere::Instruction)); + } + { + // Retain mapped bytes beyond the first EXIT for investigating + // native CALL targets in compiler library subroutines. + std::ofstream mapped(folder / (key + ".mapped.bin"), std::ios::binary); + std::array<u8, 0x1000> block{}; + for (u32 offset = 0; offset < 0x10000; offset += 0x1000) { + if (!gpu_memory->IsFullyMappedRange(program_base + offset, block.size())) break; + gpu_memory->ReadBlock(program_base + offset, block.data(), block.size()); + mapped.write(reinterpret_cast<const char*>(block.data()), block.size()); + } + } + LOG_WARNING(Shader, "Native shader paused: key={}, offset={:#x}, reason={}, source={}", + key, lowered.error_offset, lowered.error, source_path.string()); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::minutes{30}; + while (!gpu_memory->IsShuttingDown() && std::chrono::steady_clock::now() < deadline) { + if (std::filesystem::exists(override_path)) { + std::ifstream file(override_path, std::ios::binary | std::ios::ate); + const auto length = file.tellg(); + if (length <= 0 || length > 16 * 1024 * 1024) { + throw Shader::RuntimeError("Invalid native shader override size"); + } + std::vector<u8> bytes(static_cast<size_t>(length)); + file.seekg(0); + file.read(reinterpret_cast<char*>(bytes.data()), bytes.size()); + if (!file) throw Shader::RuntimeError("Native shader override read failed"); + const u32 source_bytes = Shader::Ampere::ProgramOverrideSourceSize(bytes); + if (source_bytes > native.size() * sizeof(Shader::Ampere::Instruction)) { + if (!gpu_memory->IsFullyMappedRange(program_base + native_header_size, source_bytes)) + throw Shader::RuntimeError("Unmapped extended native shader source"); + native.resize(source_bytes / sizeof(Shader::Ampere::Instruction)); + gpu_memory->ReadBlock(program_base + native_header_size, native.data(), source_bytes); + } + lowered = Shader::Ampere::DecodeProgramOverride(bytes, native); + if (!lowered) throw Shader::RuntimeError("Native shader override rejected: {}", lowered.error); + LOG_WARNING(Shader, "Native shader resumed: key={}, instructions={}", key, lowered.instructions.size()); + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds{100}); + } + } + } + if (!lowered) { + throw Shader::RuntimeError("Native shader at {:#x}, instruction {:#x}: {}", + program_base, lowered.error_offset, lowered.error); + } + if (lowered.required_texture_buffer != ~0U) { + // Native TEX carries its cbuf index explicitly; TEXS obtains it from the environment. + texture_bound = lowered.required_texture_buffer; + } + native_source_size = native_header_size + lowered.source_size; + code.resize(sizeof(sph) / sizeof(u64)); + std::memcpy(code.data(), &sph, sizeof(sph)); + for (const u64 instruction : lowered.instructions) { + if ((code.size() * sizeof(u64)) % 32 == 0) { + code.push_back(0); // Maxwell scheduling word, absent from native instructions. + } + code.push_back(instruction); + } + // Keep the original padding ABI, then retain the native address map as + // unreachable data. Existing disk-cache serialization carries it with code. + code.push_back(0); + Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets); + cached_lowest = 0; + cached_highest = static_cast<u32>((code.size() - 1) * sizeof(u64)); + LOG_INFO(Shader, "Lowered native shader at {:#x}: {} source bytes, {} instructions", + program_base, native_source_size, lowered.instructions.size()); +} + GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, Tegra::MemoryManager& gpu_memory_, Maxwell::ShaderType program, GPUVAddr program_base_, @@ -298,6 +441,23 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, : GenericEnvironment{gpu_memory_, program_base_, start_address_}, maxwell3d{&maxwell3d_} { gpu_memory->ReadBlock(program_base + start_address, &sph, sizeof(sph)); initial_offset = sizeof(sph); + texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot; + if (maxwell3d->GetNativeMacroEngine().native_turing) { + if (program != Maxwell::ShaderType::VertexB && program != Maxwell::ShaderType::Pixel) { + throw Shader::RuntimeError("Unsupported native graphics shader stage {}", static_cast<u32>(program)); + } + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { + const u32 native_stage = program == Maxwell::ShaderType::Pixel ? 4 : 0; + const auto& table = maxwell3d->state.shader_stages[native_stage].const_buffers[1]; + if (table.enabled && table.size && table.size <= Tegra::Engines::Maxwell3D::Regs::MaxConstBufferSize) { + std::vector<u8> bytes(table.size); + gpu_memory->ReadBlock(table.address, bytes.data(), bytes.size()); + std::ofstream dump(fmt::format("native-graphics-cbuf-{}-1.bin", static_cast<u32>(program)), std::ios::binary); + dump.write(reinterpret_cast<const char*>(bytes.data()), bytes.size()); + } + } + InitializeNativeCode(); + } gp_passthrough_mask = maxwell3d->regs.post_vtg_shader_attrib_skip_mask; switch (program) { case Maxwell::ShaderType::VertexA: @@ -331,8 +491,7 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, const u64 local_size{sph.LocalMemorySize()}; ASSERT(local_size <= (std::numeric_limits<u32>::max)()); local_memory_size = static_cast<u32>(local_size) + sph.common3.shader_local_memory_crs_size; - texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot; - is_proprietary_driver = texture_bound == 2; + is_proprietary_driver = maxwell3d->GetNativeMacroEngine().native_turing || texture_bound == 2; has_hle_engine_state = maxwell3d->engine_state == Tegra::Engines::Maxwell3D::EngineHint::OnHLEMacro; } @@ -417,6 +576,12 @@ ComputeEnvironment::ComputeEnvironment(Tegra::Engines::KeplerCompute& kepler_com is_proprietary_driver = texture_bound == 2; shared_memory_size = qmd.shared_alloc; workgroup_size = {qmd.block_dim_x, qmd.block_dim_y, qmd.block_dim_z}; + if (kepler_compute->IsNativeMode()) { + initial_offset = sizeof(sph); + is_proprietary_driver = true; + InitializeNativeCode(0); + start_address = sizeof(sph); + } } u32 ComputeEnvironment::ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) { @@ -511,7 +676,7 @@ void FileEnvironment::Deserialize(std::ifstream& file) { if (stage == Shader::Stage::Compute) { file.read(reinterpret_cast<char*>(&workgroup_size), sizeof(workgroup_size)) .read(reinterpret_cast<char*>(&shared_memory_size), sizeof(shared_memory_size)); - initial_offset = 0; + initial_offset = Shader::Ampere::HasNativeAddressMap(code) ? sizeof(sph) : 0; } else { file.read(reinterpret_cast<char*>(&sph), sizeof(sph)); initial_offset = sizeof(sph); @@ -526,6 +691,13 @@ void FileEnvironment::Dump(u64 pipeline_hash, u64 shader_hash) { DumpImpl(pipeline_hash, shader_hash, code, read_highest, read_lowest, initial_offset, stage); } +u32 FileEnvironment::TranslateProgramAddress(u32 address) const { + if (const auto translated = Shader::Ampere::NativeProgramAddress(code, address)) return *translated; + if (Shader::Ampere::HasNativeAddressMap(code)) + throw Shader::RuntimeError("Invalid cached native indirect branch address {:#x}", address); + return address; +} + u64 FileEnvironment::ReadInstruction(u32 address) { if (address < read_lowest || address > read_highest) { throw Shader::LogicError("Out of bounds address {}", address); diff --git a/src/video_core/shader_environment.h b/src/video_core/shader_environment.h index 924e61ef46..e49f01d95f 100644 --- a/src/video_core/shader_environment.h +++ b/src/video_core/shader_environment.h @@ -47,6 +47,8 @@ public: [[nodiscard]] u64 ReadInstruction(u32 address) final; + [[nodiscard]] u32 TranslateProgramAddress(u32 address) const override; + [[nodiscard]] std::optional<u64> Analyze(); void SetCachedSize(size_t size_bytes); @@ -57,6 +59,10 @@ public: [[nodiscard]] size_t ReadSizeBytes() const noexcept; + [[nodiscard]] size_t SourceSizeBytes() const noexcept { + return native_source_size; + } + [[nodiscard]] bool CanBeSerialized() const noexcept; [[nodiscard]] u64 CalculateHash() const; @@ -72,6 +78,9 @@ public: protected: std::optional<u64> TryFindSize(); + void InitializeNativeCode(u32 source_header_size = 128); + size_t native_source_size{}; + Tegra::Texture::TICEntry ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw); @@ -172,6 +181,8 @@ public: [[nodiscard]] u64 ReadInstruction(u32 address) override; + [[nodiscard]] u32 TranslateProgramAddress(u32 address) const override; + [[nodiscard]] u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) override; [[nodiscard]] Shader::TextureType ReadTextureType(u32 handle) override; diff --git a/src/video_core/texture_cache/accelerated_swizzle.cpp b/src/video_core/texture_cache/accelerated_swizzle.cpp index 4c3f724d79..8ae0ed218d 100644 --- a/src/video_core/texture_cache/accelerated_swizzle.cpp +++ b/src/video_core/texture_cache/accelerated_swizzle.cpp @@ -37,6 +37,7 @@ BlockLinearSwizzle2DParams MakeBlockLinearSwizzle2DParams(const SwizzleParameter .x_shift = GOB_SIZE_SHIFT + block.height + block.depth, .block_height = block.height, .block_height_mask = (1U << block.height) - 1, + .native_tiling = info.native_tiling, }; } @@ -63,6 +64,7 @@ BlockLinearSwizzle3DParams MakeBlockLinearSwizzle3DParams(const SwizzleParameter .block_height_mask = (1U << block.height) - 1, .block_depth = block.depth, .block_depth_mask = (1U << block.depth) - 1, + .native_tiling = info.native_tiling, }; } diff --git a/src/video_core/texture_cache/accelerated_swizzle.h b/src/video_core/texture_cache/accelerated_swizzle.h index d4250da689..8296950c38 100644 --- a/src/video_core/texture_cache/accelerated_swizzle.h +++ b/src/video_core/texture_cache/accelerated_swizzle.h @@ -20,6 +20,7 @@ struct BlockLinearSwizzle2DParams { u32 x_shift; u32 block_height; u32 block_height_mask; + u32 native_tiling; }; struct BlockLinearSwizzle3DParams { @@ -33,6 +34,7 @@ struct BlockLinearSwizzle3DParams { u32 block_height_mask; u32 block_depth; u32 block_depth_mask; + u32 native_tiling; }; [[nodiscard]] BlockLinearSwizzle2DParams MakeBlockLinearSwizzle2DParams( diff --git a/src/video_core/texture_cache/format_lookup_table.cpp b/src/video_core/texture_cache/format_lookup_table.cpp index d6e019cddf..20bb96aae3 100644 --- a/src/video_core/texture_cache/format_lookup_table.cpp +++ b/src/video_core/texture_cache/format_lookup_table.cpp @@ -157,6 +157,7 @@ PixelFormat PixelFormatFromTextureInfo(TextureFormat format, ComponentType red, return PixelFormat::S8_UINT_D24_UNORM; case Hash(TextureFormat::G24R8, UINT, UNORM, UNORM, UNORM, LINEAR): return PixelFormat::S8_UINT_D24_UNORM; + case Hash(TextureFormat::S8Z24, UNORM, UINT, UNORM, UNORM, LINEAR): case Hash(TextureFormat::S8Z24, UNORM, UINT, UINT, UINT, LINEAR): return PixelFormat::D24_UNORM_S8_UINT; case Hash(TextureFormat::Z32_X24S8, FLOAT, UINT, UNORM, UNORM, LINEAR): diff --git a/src/video_core/texture_cache/image_info.h b/src/video_core/texture_cache/image_info.h index eb490a6424..7407062d19 100644 --- a/src/video_core/texture_cache/image_info.h +++ b/src/video_core/texture_cache/image_info.h @@ -14,6 +14,17 @@ namespace VideoCommon { using Tegra::Texture::TICEntry; using VideoCore::Surface::PixelFormat; +// Native descriptors address the selected view directly. The legacy NVN +// layer-base bookkeeping is not an additional array prefix for this ABI. +inline TICEntry TextureViewHeader(TICEntry entry, bool native) { + if (native) { + entry.layer_base_0_2.Assign(0); + entry.layer_base_3_7.Assign(0); + entry.layer_base_8_10.Assign(0); + } + return entry; +} + struct ImageInfo { ImageInfo() = default; explicit ImageInfo(const TICEntry& config) noexcept; @@ -37,6 +48,7 @@ struct ImageInfo { u32 maybe_unaligned_layer_stride = 0; u32 num_samples = 1; u32 tile_width_spacing = 0; + bool native_tiling = false; bool rescaleable = false; bool downscaleable = false; bool forced_flushed = false; diff --git a/src/video_core/texture_cache/pending_download.h b/src/video_core/texture_cache/pending_download.h new file mode 100644 index 0000000000..52a48e82d3 --- /dev/null +++ b/src/video_core/texture_cache/pending_download.h @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later +#pragma once + +#include <algorithm> +#include <memory> +#include <unordered_set> +#include <cstddef> +#include <utility> +#include "common/slot_vector.h" + +namespace VideoCommon { + +template <class Image> +struct PendingImageDownload { + bool is_swizzle; + size_t async_buffer_id; + Common::SlotId object_id; + std::shared_ptr<Image> retired_image{}; + + Image& GetImage(Common::SlotVector<Image>& images) const { + return retired_image ? *retired_image : images[object_id]; + } +}; + +// Whole-image copies are recorded together at commit time. Repeated requests +// for the same live image therefore observe the same contents. Keep the first +// occurrence so reverse CPU writeback order remains unchanged for aliases. +template <class Image, class Downloads> +size_t DeduplicatePendingImageDownloads(Downloads& downloads, Common::SlotVector<Image>& images) { + std::unordered_set<const Image*> seen; + const size_t previous_size = downloads.size(); + const auto end = std::remove_if(downloads.begin(), downloads.end(), [&](const auto& download) { + return download.is_swizzle && !seen.insert(&download.GetImage(images)).second; + }); + downloads.erase(end, downloads.end()); + return previous_size - downloads.size(); +} + +// Cache slots may be recycled before queued GPU downloads are submitted or +// consumed. Keep the actual image, including its metadata, until those finish. +template <class Image, class Downloads, class Batches> +bool RetainPendingImageDownloads(Common::SlotId id, Image& image, Downloads& pending, + Batches& submitted) { + std::shared_ptr<Image> retained; + const auto retain = [&](auto& downloads) { + for (auto& download : downloads) { + if (!download.is_swizzle || download.object_id != id || download.retired_image) continue; + if (!retained) retained = std::make_shared<Image>(std::move(image)); + download.retired_image = retained; + } + }; + retain(pending); + for (auto& downloads : submitted) retain(downloads); + return retained != nullptr; +} + +} // namespace VideoCommon diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index e92e54f8ce..905f73999b 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -803,6 +803,17 @@ std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImage if (it == page_table.end()) { return {}; } + const auto view_format = [&]() { + switch (config.pixel_format) { + case Service::android::PixelFormat::Rgb565: + return PixelFormat::R5G6B5_UNORM; + case Service::android::PixelFormat::Bgra8888: + return PixelFormat::B8G8R8A8_UNORM; + default: + return PixelFormat::A8B8G8R8_UNORM; + } + }(); + const auto& image_map_ids = it->second; boost::container::small_vector<ImageId, 4> valid_image_ids; for (const ImageMapId map_id : image_map_ids) { @@ -814,20 +825,17 @@ std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImage if (image.image_view_ids.empty()) { continue; } + if (!VideoCore::Surface::IsViewCompatible(image.info.format, view_format, false, true)) { + static u32 rejected_alias_count{}; + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && rejected_alias_count++ < 8) { + LOG_WARNING(HW_GPU, "Ignoring incompatible framebuffer alias: address={:#x}, image={}, view={}", + cpu_addr, image.info.format, view_format); + } + continue; + } valid_image_ids.push_back(map.image_id); } - const auto view_format = [&]() { - switch (config.pixel_format) { - case Service::android::PixelFormat::Rgb565: - return PixelFormat::R5G6B5_UNORM; - case Service::android::PixelFormat::Bgra8888: - return PixelFormat::B8G8R8A8_UNORM; - default: - return PixelFormat::A8B8G8R8_UNORM; - } - }(); - const auto GetImageViewForFramebuffer = [&](ImageId image_id) { ImageViewInfo info{ImageViewType::e2D, view_format}; if (config.blending == Tegra::BlendMode::Opaque) { @@ -869,6 +877,7 @@ void TextureCache<P>::CommitAsyncFlushes() { // This is intentionally passing the value by copy if constexpr (IMPLEMENTS_ASYNC_DOWNLOADS) { auto& download_ids = uncommitted_downloads; + const size_t redundant_images = DeduplicatePendingImageDownloads(download_ids, slot_images); if (download_ids.empty()) { committed_downloads.emplace_back(std::move(uncommitted_downloads)); uncommitted_downloads.clear(); @@ -882,17 +891,32 @@ void TextureCache<P>::CommitAsyncFlushes() { for (PendingDownload& download_info : download_ids) { if (download_info.is_swizzle) { total_size_bytes += - Common::AlignUp(slot_images[download_info.object_id].unswizzled_size_bytes, 64); + Common::AlignUp(download_info.GetImage(slot_images).unswizzled_size_bytes, 64); any_none_dma = true; download_info.async_buffer_id = last_async_buffer_id; } } if (any_none_dma) { + if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && total_size_bytes >= 64_MiB) { + LOG_WARNING(HW_GPU, "Large image readback batch: bytes={}, entries={}, duplicates_removed={}, submitted_batches={}, retained_buffers={}", + total_size_bytes, download_ids.size(), redundant_images, + committed_downloads.size(), async_buffers_death_ring.size()); + size_t traced_images = 0; + for (const auto& download : download_ids) { + if (!download.is_swizzle || traced_images++ >= 8) continue; + const auto& image = download.GetImage(slot_images); + LOG_WARNING(HW_GPU, "Readback image: gpu={:#x}, cpu={:#x}, format={}, size={}x{}x{}, levels={}, layers={}, layer_stride={}, guest_bytes={}, download_bytes={}", + image.gpu_addr, image.cpu_addr, image.info.format, + image.info.size.width, image.info.size.height, image.info.size.depth, + image.info.resources.levels, image.info.resources.layers, + image.info.layer_stride, image.guest_size_bytes, image.unswizzled_size_bytes); + } + } auto download_map = runtime.DownloadStagingBuffer(total_size_bytes, true); for (const PendingDownload& download_info : download_ids) { if (download_info.is_swizzle) { - Image& image = slot_images[download_info.object_id]; + Image& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); image.DownloadMemory(download_map, copies); download_map.offset += Common::AlignUp(image.unswizzled_size_bytes, 64); @@ -925,7 +949,7 @@ void TextureCache<P>::PopAsyncFlushes() { auto& download_info = download_ids[i - 1]; auto& download_buffer = download_map[download_info.async_buffer_id]; if (download_info.is_swizzle) { - const ImageBase& image = slot_images[download_info.object_id]; + const ImageBase& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); download_buffer.offset -= Common::AlignUp(image.unswizzled_size_bytes, 64); std::span<u8> download_span = @@ -954,7 +978,7 @@ void TextureCache<P>::PopAsyncFlushes() { size_t total_size_bytes = 0; for (const PendingDownload& download_info : download_ids) { if (download_info.is_swizzle) { - total_size_bytes += slot_images[download_info.object_id].unswizzled_size_bytes; + total_size_bytes += download_info.GetImage(slot_images).unswizzled_size_bytes; } } auto download_map = runtime.DownloadStagingBuffer(total_size_bytes); @@ -963,7 +987,7 @@ void TextureCache<P>::PopAsyncFlushes() { if (!download_info.is_swizzle) { continue; } - Image& image = slot_images[download_info.object_id]; + Image& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); image.DownloadMemory(download_map, copies); download_map.offset += image.unswizzled_size_bytes; @@ -976,7 +1000,7 @@ void TextureCache<P>::PopAsyncFlushes() { if (!download_info.is_swizzle) { continue; } - const ImageBase& image = slot_images[download_info.object_id]; + const ImageBase& image = download_info.GetImage(slot_images); const auto copies = FullDownloadCopies(image.info); SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, download_span, swizzle_data_buffer); download_map.offset += image.unswizzled_size_bytes; @@ -1178,7 +1202,18 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging) } template <class P> -ImageViewId TextureCache<P>::CreateImageView(const TICEntry& config) { +ImageViewId TextureCache<P>::CreateImageView(const TICEntry& raw_config) { + const bool native = gpu_memory->UsesNativeTiling(); + if (raw_config.BaseLayer() && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { + static u32 texture_view_traces{}; + if (texture_view_traces++ < 8) { + LOG_WARNING(HW_GPU, "Native texture view header: native={}, graphics_native={}, address={:#x}, dimensions={}x{}x{}, type={}, legacy_layer={}, raw={:016x}/{:016x}/{:016x}/{:016x}", + native, maxwell3d->GetNativeMacroEngine().native_turing, raw_config.Address(), raw_config.Width(), raw_config.Height(), raw_config.Depth(), + static_cast<u32>(raw_config.texture_type.Value()), raw_config.BaseLayer(), + raw_config.raw[0], raw_config.raw[1], raw_config.raw[2], raw_config.raw[3]); + } + } + const TICEntry config = TextureViewHeader(raw_config, native); const ImageInfo info(config); if (info.type == ImageType::Buffer) { const ImageViewInfo view_info(config, 0); @@ -1526,7 +1561,9 @@ ImageId TextureCache<P>::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr, } } ASSERT_MSG(cpu_addr, "Tried to insert an image to an invalid gpu_addr={:#x}", gpu_addr); - const ImageId image_id = JoinImages(info, gpu_addr, *cpu_addr); + ImageInfo storage_info = info; + storage_info.native_tiling = gpu_memory->UsesNativeTiling(); + const ImageId image_id = JoinImages(storage_info, gpu_addr, *cpu_addr); const Image& image = slot_images[image_id]; // Using "image.gpu_addr" instead of "gpu_addr" is important because it might be different const auto [it, is_new] = image_allocs_table.try_emplace(image.gpu_addr); @@ -2385,7 +2422,11 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { } slot_image_views.erase(image_view_id); } - if (!immediate_delete) { + const bool retained = RetainPendingImageDownloads(image_id, slot_images[image_id], + uncommitted_downloads, committed_downloads); + if (retained) { + LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value()); + } else if (!immediate_delete) { sentenced_images.Push(std::move(slot_images[image_id])); } slot_images.erase(image_id); diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 8a2318cc72..d103d66189 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -28,6 +28,7 @@ #include <ranges> #include "common/scratch_buffer.h" #include "common/slot_vector.h" +#include "video_core/texture_cache/pending_download.h" #include "common/thread_worker.h" #include "video_core/compatible_formats.h" #include "video_core/control/channel_state_cache.h" @@ -463,11 +464,7 @@ private: size_t size; }; - struct PendingDownload { - bool is_swizzle; - size_t async_buffer_id; - Common::SlotId object_id; - }; + using PendingDownload = PendingImageDownload<Image>; Common::SlotVector<Image> slot_images; Common::SlotVector<ImageMapView> slot_map_views; diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp index e55d0752ec..5bcea68977 100644 --- a/src/video_core/texture_cache/util.cpp +++ b/src/video_core/texture_cache/util.cpp @@ -439,7 +439,7 @@ template <u32 GOB_EXTENT> } layers = 1; } else { - layers = (std::max)(resources.layers, info.resources.layers + base->layer); + layers = (std::max)(info.resources.layers, resources.layers + base->layer); } return OverlapResult{ .gpu_addr = overlap.gpu_addr, @@ -566,7 +566,7 @@ void SwizzleBlockLinearImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr dst(gpu_memory, gpu_addr + guest_offset, subresource_size, &tmp_buffer); SwizzleTexture(dst, src, bytes_per_block, num_tiles.width, num_tiles.height, - num_tiles.depth, block.height, block.depth); + num_tiles.depth, block.height, block.depth, 1, gpu_memory.UsesNativeTiling()); } host_offset += host_bytes_per_layer; @@ -838,7 +838,7 @@ bool IsValidEntry(const Tegra::MemoryManager& gpu_memory, const TICEntry& config if (gpu_memory.GpuToCpuAddress(address).has_value()) { return true; } - const ImageInfo info{config}; + const ImageInfo info{TextureViewHeader(config, gpu_memory.UsesNativeTiling())}; const size_t guest_size_bytes = CalculateGuestSizeInBytes(info); return gpu_memory.GpuToCpuAddress(address, guest_size_bytes).has_value(); } @@ -913,7 +913,7 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory const std::span<u8> dst = output.subspan(host_offset); const std::span<const u8> src = input.subspan(guest_offset + guest_layer_offset); UnswizzleTexture(dst, src, 1U << bpp_log2, num_tiles.width, num_tiles.height, - num_tiles.depth, block.height, block.depth, stride_alignment); + num_tiles.depth, block.height, block.depth, stride_alignment, gpu_memory.UsesNativeTiling()); guest_layer_offset += layer_stride; host_offset += host_bytes_per_layer; } diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index 25b0860b0f..ef50be156b 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -18,6 +18,11 @@ namespace Tegra::Texture { namespace { +constexpr u32 NativeGobOffset(u32 offset, bool native_tiling) { + if (!native_tiling) return offset; + const u32 difference = ((offset >> 5) ^ (offset >> 6)) & 1; + return offset ^ (difference << 5) ^ (difference << 6); +} template <u32 mask> constexpr u32 pdep(u32 value) { u32 result = 0; @@ -38,7 +43,7 @@ void incrpdep(u32& value) { template <bool TO_LINEAR, u32 BYTES_PER_PIXEL> void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 height, u32 depth, - u32 block_height, u32 block_depth, u32 stride) { + u32 block_height, u32 block_depth, u32 stride, bool native_tiling) { // The origin of the transformation can be configured here, leave it as zero as the current API // doesn't expose it. static constexpr u32 origin_x = 0; @@ -77,7 +82,7 @@ void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift; const u32 base_swizzled_offset = offset_z + offset_y + offset_x; - const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y); + const u32 swizzled_offset = NativeGobOffset(base_swizzled_offset + (swizzled_x | swizzled_y), native_tiling); const u32 unswizzled_offset = slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL; @@ -94,7 +99,7 @@ void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 template <bool TO_LINEAR, u32 BYTES_PER_PIXEL> void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 num_lines, - u32 block_height, u32 block_depth, u32 pitch_linear) { + u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) { // The origin of the transformation can be configured here, leave it as zero as the current API // doesn't expose it. static constexpr u32 origin_z = 0; @@ -136,7 +141,7 @@ void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 wid const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift; const u32 base_swizzled_offset = offset_z + offset_y + offset_x; - const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y); + const u32 swizzled_offset = NativeGobOffset(base_swizzled_offset + (swizzled_x | swizzled_y), native_tiling); const u32 unswizzled_offset = slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL; @@ -156,12 +161,12 @@ void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 wid template <bool TO_LINEAR> void Swizzle(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, - u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment) { + u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment, bool native_tiling) { switch (bytes_per_pixel) { #define BPP_CASE(x) \ case x: \ return SwizzleImpl<TO_LINEAR, x>(output, input, width, height, depth, block_height, \ - block_depth, stride_alignment); + block_depth, stride_alignment, native_tiling); BPP_CASE(1) BPP_CASE(2) BPP_CASE(3) @@ -181,35 +186,35 @@ void Swizzle(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixe void UnswizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment) { + u32 stride_alignment, bool native_tiling) { const u32 stride = Common::AlignUpLog2(width, stride_alignment) * bytes_per_pixel; const u32 new_bpp = (std::min)(4U, static_cast<u32>(std::countr_zero(width * bytes_per_pixel))); width = (width * bytes_per_pixel) >> new_bpp; bytes_per_pixel = 1U << new_bpp; Swizzle<false>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth, - stride); + stride, native_tiling); } void SwizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment) { + u32 stride_alignment, bool native_tiling) { const u32 stride = Common::AlignUpLog2(width, stride_alignment) * bytes_per_pixel; const u32 new_bpp = (std::min)(4U, static_cast<u32>(std::countr_zero(width * bytes_per_pixel))); width = (width * bytes_per_pixel) >> new_bpp; bytes_per_pixel = 1U << new_bpp; Swizzle<true>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth, - stride); + stride, native_tiling); } void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 extent_y, - u32 block_height, u32 block_depth, u32 pitch_linear) { + u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) { switch (bytes_per_pixel) { #define BPP_CASE(x) \ case x: \ return SwizzleSubrectImpl<true, x>(output, input, width, height, depth, origin_x, \ origin_y, extent_x, extent_y, block_height, \ - block_depth, pitch_linear); + block_depth, pitch_linear, native_tiling); BPP_CASE(1) BPP_CASE(2) BPP_CASE(3) @@ -227,13 +232,13 @@ void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_p void UnswizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, - u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear) { + u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) { switch (bytes_per_pixel) { #define BPP_CASE(x) \ case x: \ return SwizzleSubrectImpl<false, x>(output, input, width, height, depth, origin_x, \ origin_y, extent_x, extent_y, block_height, \ - block_depth, pitch_linear); + block_depth, pitch_linear, native_tiling); BPP_CASE(1) BPP_CASE(2) BPP_CASE(3) diff --git a/src/video_core/textures/decoders.h b/src/video_core/textures/decoders.h index 1e14a38753..6151967869 100644 --- a/src/video_core/textures/decoders.h +++ b/src/video_core/textures/decoders.h @@ -29,12 +29,12 @@ constexpr u32 SWIZZLE_Y_BITS = 0b011010000; /// Unswizzles a block linear texture into linear memory. void UnswizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment = 1); + u32 stride_alignment = 1, bool native_tiling = false); /// Swizzles linear memory into a block linear texture. void SwizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth, - u32 stride_alignment = 1); + u32 stride_alignment = 1, bool native_tiling = false); /// This function calculates the correct size of a texture depending if it's tiled or not. std::size_t CalculateSize(bool tiled, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, @@ -43,12 +43,12 @@ std::size_t CalculateSize(bool tiled, u32 bytes_per_pixel, u32 width, u32 height /// Copies an untiled subrectangle into a tiled surface. void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 extent_y, - u32 block_height, u32 block_depth, u32 pitch_linear); + u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling = false); /// Copies a tiled subrectangle into a linear surface. void UnswizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, - u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear); + u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling = false); /// Obtains the offset of the gob for positions 'dst_x' & 'dst_y' u64 GetGOBOffset(u32 width, u32 height, u32 dst_x, u32 dst_y, u32 block_height, diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index f4721543ae..57b07c3707 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -221,7 +221,31 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const VkImage handle{}; VmaAllocation allocation{}; VmaAllocationInfo alloc_info{}; - vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + const VkResult result = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + if (result != VK_SUCCESS) { + LOG_ERROR(Render_Vulkan, + "Vulkan image allocation failed (result={}): {}x{}x{}, format={:#x}, type={}, " + "layers={}, mips={}, samples={}, usage={:#x}, flags={:#x}", + static_cast<s32>(result), ci.extent.width, ci.extent.height, ci.extent.depth, + static_cast<u32>(ci.format), static_cast<u32>(ci.imageType), ci.arrayLayers, + ci.mipLevels, static_cast<u32>(ci.samples), ci.usage, ci.flags); + VmaBudget budgets[VK_MAX_MEMORY_HEAPS]{}; + vmaGetHeapBudgets(allocator, budgets); + const auto format_properties = device.GetPhysical().GetFormatProperties(ci.format); + const auto physical_properties = device.GetPhysical().GetProperties(); + LOG_ERROR(Render_Vulkan, + "Image device limits: max2D={}, max allocations={}, format optimal features={:#x}", + physical_properties.limits.maxImageDimension2D, + physical_properties.limits.maxMemoryAllocationCount, + format_properties.optimalTilingFeatures); + for (u32 heap = 0; heap < properties.memoryHeapCount; ++heap) { + LOG_ERROR(Render_Vulkan, + "Vulkan heap {} budget at image failure: usage={} MiB, budget={} MiB, blocks={}, allocations={}", + heap, budgets[heap].usage / (1024 * 1024), budgets[heap].budget / (1024 * 1024), + budgets[heap].statistics.blockCount, budgets[heap].statistics.allocationCount); + } + } + vk::Check(result); // Log GPU memory allocation for images if (GPU::Logging::IsActive() && @@ -258,7 +282,12 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa VmaAllocation allocation{}; VkMemoryPropertyFlags property_flags{}; - vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + const VkResult result = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + if (result != VK_SUCCESS) { + LOG_CRITICAL(Render_Vulkan, "Buffer allocation failed: bytes={}, usage={}, buffer_flags={:#x}, result={}", + ci.size, static_cast<u32>(usage), ci.usage, static_cast<s32>(result)); + } + vk::Check(result); vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags); // Log GPU memory allocation for buffers diff --git a/src/yuzu_cmd/yuzu.cpp b/src/yuzu_cmd/yuzu.cpp index 473a5b1014..488f1ef47d 100644 --- a/src/yuzu_cmd/yuzu.cpp +++ b/src/yuzu_cmd/yuzu.cpp @@ -159,11 +159,13 @@ static void OnStatusMessageReceived(const Network::StatusMessageEntry& msg) { struct SdlState { Core::System system{}; + InputCommon::InputSubsystem input_subsystem{}; std::unique_ptr<EmuWindow_SDL3> emu_window; }; extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { SdlState* state = new SdlState(); + *appstate = state; #ifdef _WIN32 if (AttachConsole(ATTACH_PARENT_PROCESS)) { @@ -202,8 +204,6 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { state->system.Initialize(); - InputCommon::InputSubsystem input_subsystem{}; - // Apply the command line arguments state->system.ApplySettings(); @@ -212,14 +212,14 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { case Settings::RendererBackend::OpenGL_GLSL: case Settings::RendererBackend::OpenGL_GLASM: case Settings::RendererBackend::OpenGL_SPIRV: - state->emu_window = std::make_unique<EmuWindow_SDL3_GL>(&input_subsystem, state->system, lp.fullscreen); + state->emu_window = std::make_unique<EmuWindow_SDL3_GL>(&state->input_subsystem, state->system, lp.fullscreen); break; #endif case Settings::RendererBackend::Vulkan: - state->emu_window = std::make_unique<EmuWindow_SDL3_VK>(&input_subsystem, state->system, lp.fullscreen); + state->emu_window = std::make_unique<EmuWindow_SDL3_VK>(&state->input_subsystem, state->system, lp.fullscreen); break; case Settings::RendererBackend::Null: - state->emu_window = std::make_unique<EmuWindow_SDL3_Null>(&input_subsystem, state->system, lp.fullscreen); + state->emu_window = std::make_unique<EmuWindow_SDL3_Null>(&state->input_subsystem, state->system, lp.fullscreen); break; default: LOG_CRITICAL(Frontend, "Invalid renderer backend"); @@ -295,7 +295,7 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { // QLaunch launched applications (should) now reload shader cache. static std::mutex shader_cache_reload_mutex; - state->system.RegisterApplicationChangedCallback([&state](u64 changed_program_id) { + state->system.RegisterApplicationChangedCallback([state](u64 changed_program_id) { if (!Settings::values.use_disk_shader_cache.GetValue()) { return; } @@ -319,7 +319,7 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { void(state->system.Run()); if (state->system.DebuggerEnabled()) state->system.InitializeDebugger(); - return SDL_APP_SUCCESS; + return SDL_APP_CONTINUE; } extern "C" SDL_AppResult SDL_AppIterate(void *appstate) { SdlState *state = (SdlState *)appstate; @@ -328,10 +328,13 @@ extern "C" SDL_AppResult SDL_AppIterate(void *appstate) { extern "C" SDL_AppResult SDL_AppEvent(void *appstate, SDL_Event *event) { SdlState *state = (SdlState *)appstate; state->emu_window->OnEvent(*event); - return SDL_APP_SUCCESS; + return SDL_APP_CONTINUE; } extern "C" void SDL_AppQuit(void *appstate, SDL_AppResult result) { SdlState *state = (SdlState *)appstate; + if (!state) { + return; + } state->system.DetachDebugger(); void(state->system.Pause()); state->system.ShutdownMainProcess(); |
