aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDjihads <76681596+Djihads80@users.noreply.github.com>2026-10-29 02:25:36 +0100
committerGitHub <noreply@github.com>2026-10-29 02:25:36 +0100
commit0bb5055b6541c74f47603a8dff0569a509cbaf18 (patch)
treeda827ff9c4f2f9a5309ee2804ee689f608c0ac2f
parent1af7fe72927af9fc64aa4af117bc9b30043c4dc7 (diff)
parent535b6769709eb22333af26269f1b16f37adce4db (diff)
Merge pull request #1 from Djihads80/yumekiru/ns2-progress-v2
Experimental NS2 support and accumulated GPU fixes
-rw-r--r--src/audio_core/renderer/behavior/info_updater.cpp13
-rw-r--r--src/common/sparse_large_vector.cpp2
-rw-r--r--src/common/zbic_compression.cpp7
-rw-r--r--src/core/CMakeLists.txt14
-rw-r--r--src/core/arm/dynarmic/arm_dynarmic_64.cpp3
-rw-r--r--src/core/device_memory_manager.inc3
-rw-r--r--src/core/file_sys/program_metadata.cpp55
-rw-r--r--src/core/file_sys/program_metadata.h8
-rw-r--r--src/core/hle/kernel/k_memory_block.h4
-rw-r--r--src/core/hle/kernel/k_page_table_base.cpp3
-rw-r--r--src/core/hle/kernel/k_process.cpp1
-rw-r--r--src/core/hle/kernel/k_process.h5
-rw-r--r--src/core/hle/kernel/kernel.cpp12
-rw-r--r--src/core/hle/kernel/svc.cpp4
-rw-r--r--src/core/hle/kernel/svc.h2
-rw-r--r--src/core/hle/kernel/svc/svc_info.cpp27
-rw-r--r--src/core/hle/kernel/svc/svc_ipc.cpp19
-rw-r--r--src/core/hle/kernel/svc/svc_memory.cpp25
-rw-r--r--src/core/hle/kernel/svc/svc_synchronization.cpp21
-rw-r--r--src/core/hle/kernel/svc/svc_thread.cpp11
-rw-r--r--src/core/hle/kernel/svc_types.h1
-rw-r--r--src/core/hle/service/am/applet.h1
-rw-r--r--src/core/hle/service/am/service/application_functions.cpp14
-rw-r--r--src/core/hle/service/am/service/application_functions.h2
-rw-r--r--src/core/hle/service/am/service/application_proxy.cpp19
-rw-r--r--src/core/hle/service/am/service/application_proxy.h6
-rw-r--r--src/core/hle/service/am/service/application_proxy_service.cpp11
-rw-r--r--src/core/hle/service/am/service/application_proxy_service.h4
-rw-r--r--src/core/hle/service/am/service/gpu_memory_registration.h22
-rw-r--r--src/core/hle/service/am/service/gpu_session.cpp633
-rw-r--r--src/core/hle/service/am/service/gpu_session.h45
-rw-r--r--src/core/hle/service/am/service/graphics_scaling_parameters_functions.cpp45
-rw-r--r--src/core/hle/service/am/service/graphics_scaling_parameters_functions.h34
-rw-r--r--src/core/hle/service/am/service/graphics_session.cpp33
-rw-r--r--src/core/hle/service/am/service/graphics_session.h19
-rw-r--r--src/core/hle/service/am/service/graphics_supervisor.cpp1612
-rw-r--r--src/core/hle/service/am/service/graphics_supervisor.h93
-rw-r--r--src/core/hle/service/am/service/graphics_supervisor_protocol.h37
-rw-r--r--src/core/hle/service/am/service/graphics_syncpoint_cache.h49
-rw-r--r--src/core/hle/service/am/service/native_image_layout.h40
-rw-r--r--src/core/hle/service/am/service/performance_functions.cpp19
-rw-r--r--src/core/hle/service/am/service/performance_functions.h16
-rw-r--r--src/core/hle/service/am/service/self_controller.cpp45
-rw-r--r--src/core/hle/service/am/service/self_controller.h3
-rw-r--r--src/core/hle/service/audio/audio.cpp64
-rw-r--r--src/core/hle/service/hid/hid_server.cpp80
-rw-r--r--src/core/hle/service/hid/hid_server.h24
-rw-r--r--src/core/hle/service/ngc/ngc.cpp50
-rw-r--r--src/core/hle/service/nvdrv/core/syncpoint_manager.cpp8
-rw-r--r--src/core/hle/service/nvdrv/core/syncpoint_manager.h3
-rw-r--r--src/core/hle/service/nvdrv/devices/gpu_address_space_layout.h34
-rw-r--r--src/core/hle/service/nvdrv/devices/native_gpu_characteristics.h67
-rw-r--r--src/core/hle/service/nvdrv/devices/native_gpu_context.h23
-rw-r--r--src/core/hle/service/nvdrv/devices/native_gpu_kind.h51
-rw-r--r--src/core/hle/service/nvdrv/devices/nvdevice.h6
-rw-r--r--src/core/hle/service/nvdrv/devices/nvdisp_disp0.cpp61
-rw-r--r--src/core/hle/service/nvdrv/devices/nvdisp_disp0.h4
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp290
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h34
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp21
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp99
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_gpu.h11
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.cpp85
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.h38
-rw-r--r--src/core/hle/service/nvdrv/nvdrv.cpp19
-rw-r--r--src/core/hle/service/nvdrv/nvdrv_interface.cpp25
-rw-r--r--src/core/hle/service/nvdrv/nvdrv_interface.h2
-rw-r--r--src/core/hle/service/nvnflinger/buffer_queue_producer.cpp9
-rw-r--r--src/core/hle/service/nvnflinger/buffer_queue_producer.h2
-rw-r--r--src/core/hle/service/service.cpp28
-rw-r--r--src/core/launch_params.cpp2
-rw-r--r--src/core/loader/deconstructed_rom_directory.cpp39
-rw-r--r--src/core/loader/deconstructed_rom_directory.h1
-rw-r--r--src/core/loader/loader.cpp7
-rw-r--r--src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp6
-rw-r--r--src/dynarmic/src/dynarmic/backend/x64/a64_interface.cpp6
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc10
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.h3
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/translate/impl/a64_branch.cpp8
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h12
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp108
-rw-r--r--src/dynarmic/src/dynarmic/frontend/A64/translate/impl/system.cpp21
-rw-r--r--src/dynarmic/src/dynarmic/interface/A64/config.h4
-rw-r--r--src/dynarmic/tests/A64/lse.cpp526
-rw-r--r--src/dynarmic/tests/A64/pauth_returns.cpp94
-rw-r--r--src/dynarmic/tests/CMakeLists.txt2
-rw-r--r--src/shader_recompiler/CMakeLists.txt3
-rw-r--r--src/shader_recompiler/environment.h2
-rw-r--r--src/shader_recompiler/frontend/ampere/lower_to_maxwell.h1614
-rw-r--r--src/shader_recompiler/frontend/ampere/program_address_map.h29
-rw-r--r--src/shader_recompiler/frontend/ampere/program_capture.h47
-rw-r--r--src/shader_recompiler/frontend/ampere/program_override.h85
-rw-r--r--src/shader_recompiler/frontend/maxwell/control_flow.cpp2
-rw-r--r--src/tests/CMakeLists.txt8
-rw-r--r--src/tests/core/graphics_supervisor.cpp133
-rw-r--r--src/tests/core/native_gpu_characteristics.cpp389
-rw-r--r--src/tests/core/program_metadata.cpp151
-rw-r--r--src/tests/video_core/image_overlap.cpp93
-rw-r--r--src/tests/video_core/native_semaphore.cpp63
-rw-r--r--src/tests/video_core/native_shader.cpp2841
-rw-r--r--src/tests/video_core/native_swizzle.cpp67
-rw-r--r--src/tests/video_core/pending_download.cpp112
-rw-r--r--src/video_core/CMakeLists.txt3
-rw-r--r--src/video_core/buffer_cache/buffer_cache.h8
-rw-r--r--src/video_core/control/channel_state.cpp1
-rw-r--r--src/video_core/control/channel_state.h3
-rw-r--r--src/video_core/control/scheduler.cpp22
-rw-r--r--src/video_core/control/scheduler.h5
-rw-r--r--src/video_core/dirty_flags.cpp1
-rw-r--r--src/video_core/dma_pusher.cpp44
-rw-r--r--src/video_core/dma_pusher.h6
-rw-r--r--src/video_core/engines/draw_manager.cpp49
-rw-r--r--src/video_core/engines/engine_upload.cpp20
-rw-r--r--src/video_core/engines/engine_upload.h3
-rw-r--r--src/video_core/engines/kepler_compute.cpp212
-rw-r--r--src/video_core/engines/kepler_compute.h33
-rw-r--r--src/video_core/engines/maxwell_3d.cpp31
-rw-r--r--src/video_core/engines/maxwell_3d.h20
-rw-r--r--src/video_core/engines/maxwell_dma.cpp15
-rw-r--r--src/video_core/engines/puller.cpp122
-rw-r--r--src/video_core/engines/puller.h11
-rw-r--r--src/video_core/framebuffer_config.h1
-rw-r--r--src/video_core/gpu.cpp3
-rw-r--r--src/video_core/gpu_thread.cpp13
-rw-r--r--src/video_core/host_shaders/block_linear_unswizzle_2d.comp8
-rw-r--r--src/video_core/host_shaders/block_linear_unswizzle_3d.comp8
-rw-r--r--src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp10
-rw-r--r--src/video_core/macro.cpp38
-rw-r--r--src/video_core/macro.h19
-rw-r--r--src/video_core/macro_turing.h237
-rw-r--r--src/video_core/memory_manager.cpp18
-rw-r--r--src/video_core/memory_manager.h7
-rw-r--r--src/video_core/native_qmd.h112
-rw-r--r--src/video_core/native_query_trace.h42
-rw-r--r--src/video_core/native_semaphore.h44
-rw-r--r--src/video_core/query_cache/query_cache.h9
-rw-r--r--src/video_core/renderer_vulkan/present/layer.cpp2
-rw-r--r--src/video_core/renderer_vulkan/vk_compute_pass.cpp3
-rw-r--r--src/video_core/renderer_vulkan/vk_pipeline_cache.cpp2
-rw-r--r--src/video_core/renderer_vulkan/vk_rasterizer.cpp65
-rw-r--r--src/video_core/renderer_vulkan/vk_texture_cache.cpp5
-rw-r--r--src/video_core/shader_cache.cpp30
-rw-r--r--src/video_core/shader_environment.cpp178
-rw-r--r--src/video_core/shader_environment.h11
-rw-r--r--src/video_core/texture_cache/accelerated_swizzle.cpp2
-rw-r--r--src/video_core/texture_cache/accelerated_swizzle.h2
-rw-r--r--src/video_core/texture_cache/format_lookup_table.cpp1
-rw-r--r--src/video_core/texture_cache/image_info.h12
-rw-r--r--src/video_core/texture_cache/pending_download.h58
-rw-r--r--src/video_core/texture_cache/texture_cache.h81
-rw-r--r--src/video_core/texture_cache/texture_cache_base.h7
-rw-r--r--src/video_core/texture_cache/util.cpp8
-rw-r--r--src/video_core/textures/decoders.cpp33
-rw-r--r--src/video_core/textures/decoders.h8
-rw-r--r--src/video_core/vulkan_common/vulkan_memory_allocator.cpp33
-rw-r--r--src/yuzu_cmd/yuzu.cpp19
156 files changed, 12227 insertions, 179 deletions
diff --git a/src/audio_core/renderer/behavior/info_updater.cpp b/src/audio_core/renderer/behavior/info_updater.cpp
index a88eae91ad..b9a3745b7e 100644
--- a/src/audio_core/renderer/behavior/info_updater.cpp
+++ b/src/audio_core/renderer/behavior/info_updater.cpp
@@ -30,6 +30,9 @@ InfoUpdater::InfoUpdater(std::span<const u8> input_, std::span<u8> output_,
expected_input_size{input_.size()}, expected_output_size{output_.size()},
process_handle{process_handle_}, behaviour{behaviour_} {
std::construct_at<UpdateDataHeader>(out_header, behaviour.GetProcessRevision());
+ if ((in_header->revision & 0x00ffffffU) == 0x00764552U) {
+ out_header->revision ^= 0x00200000U;
+ }
}
Result InfoUpdater::UpdateVoiceChannelResources(VoiceContext& voice_context) {
@@ -584,11 +587,19 @@ Result InfoUpdater::UpdatePerformanceBuffer(std::span<u8> performance_output,
Result InfoUpdater::UpdateBehaviorInfo(BehaviorInfo& behaviour_) {
const auto in_params{reinterpret_cast<const BehaviorInfo::InParameter*>(input)};
+ static bool logged_header = false;
+ if (!logged_header) {
+ logged_header = true;
+ LOG_WARNING(Service_Audio, "Audio update header revision={:#x}, behavior revision={:#x}, expected={:#x}, behavior_size={}, total={}",
+ in_header->revision, in_params->revision, behaviour_.GetUserRevision(), in_header->behaviour_size, in_header->size);
+ }
if (!CheckValidRevision(in_params->revision)) {
return Service::Audio::ResultInvalidUpdateInfo;
}
- if (in_params->revision != behaviour_.GetUserRevision()) {
+ const u32 revision = (in_params->revision & 0x00ffffffU) == 0x00764552U
+ ? in_params->revision ^ 0x00200000U : in_params->revision;
+ if (revision != behaviour_.GetUserRevision()) {
return Service::Audio::ResultInvalidUpdateInfo;
}
diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp
index eee8ff6d18..b96340999d 100644
--- a/src/common/sparse_large_vector.cpp
+++ b/src/common/sparse_large_vector.cpp
@@ -26,7 +26,7 @@ static std::mutex vector_regions_mutex {};
// Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error
static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) {
DWORD code = info->ExceptionRecord->ExceptionCode;
- u64 exception_addr = reinterpret_cast<u64>(info->ExceptionRecord->ExceptionInformation[1]);
+ u64 exception_addr = static_cast<u64>(info->ExceptionRecord->ExceptionInformation[1]);
if (code != EXCEPTION_ACCESS_VIOLATION || info->ExceptionRecord->ExceptionInformation[0] == 1) {
// Not our problem
diff --git a/src/common/zbic_compression.cpp b/src/common/zbic_compression.cpp
index 5e0e4d947b..a298c6f81e 100644
--- a/src/common/zbic_compression.cpp
+++ b/src/common/zbic_compression.cpp
@@ -17,9 +17,16 @@
#endif
#include "zstd.h"
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4505)
+#endif
#define g_ZSTD_threading_useless_symbol g_ZSTD_zbic_threading_useless_symbol
#include "zstd.c"
#undef g_ZSTD_threading_useless_symbol
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
namespace Common::Compression {
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index 1b578f67ba..168037e5dc 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -460,6 +460,18 @@ add_library(core STATIC
hle/service/am/service/application_proxy.h
hle/service/am/service/application_proxy_service.cpp
hle/service/am/service/application_proxy_service.h
+ hle/service/am/service/performance_functions.cpp
+ hle/service/am/service/performance_functions.h
+ hle/service/am/service/gpu_session.cpp
+ hle/service/am/service/gpu_session.h
+ hle/service/am/service/graphics_session.cpp
+ hle/service/am/service/graphics_session.h
+ hle/service/am/service/graphics_supervisor.cpp
+ hle/service/am/service/graphics_supervisor.h
+ hle/service/am/service/graphics_supervisor_protocol.h
+ hle/service/am/service/graphics_syncpoint_cache.h
+ hle/service/am/service/graphics_scaling_parameters_functions.cpp
+ hle/service/am/service/graphics_scaling_parameters_functions.h
hle/service/am/service/audio_controller.cpp
hle/service/am/service/audio_controller.h
hle/service/am/service/common_state_getter.cpp
@@ -855,6 +867,8 @@ add_library(core STATIC
hle/service/nvdrv/devices/nvhost_ctrl_gpu.h
hle/service/nvdrv/devices/nvhost_gpu.cpp
hle/service/nvdrv/devices/nvhost_gpu.h
+ hle/service/nvdrv/devices/nvhost_tsg_gpu.cpp
+ hle/service/nvdrv/devices/nvhost_tsg_gpu.h
hle/service/nvdrv/devices/nvhost_nvdec.cpp
hle/service/nvdrv/devices/nvhost_nvdec.h
hle/service/nvdrv/devices/nvhost_nvdec_common.cpp
diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp
index 894857a86d..b5190e40cf 100644
--- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp
+++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp
@@ -194,6 +194,7 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s
// Callbacks
config.callbacks = std::addressof(*m_cb);
+ config.enable_pauth_returns = m_cb->m_process && m_cb->m_process->UsesPAuthReturns();
// Memory
if (page_table) {
@@ -444,6 +445,8 @@ void ArmDynarmic64::ClearInstructionCache() {
}
void ArmDynarmic64::InvalidateCacheRange(u64 addr, std::size_t size) {
+ // Refetch the source page too, including instructions restored by the debugger.
+ m_cb->last_code_addr = u64(-1);
m_jit->InvalidateCacheRange(addr, size);
}
diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc
index 97c2706abb..0087548978 100644
--- a/src/core/device_memory_manager.inc
+++ b/src/core/device_memory_manager.inc
@@ -21,6 +21,7 @@
#include "common/scope_exit.h"
#include "common/settings.h"
#include "core/device_memory.h"
+#include "core/hle/kernel/board/nintendo/nx/k_system_control.h"
#include "core/device_memory_manager.h"
#include "core/memory.h"
@@ -172,7 +173,7 @@ template <typename Traits>
DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memory_)
: physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())}
, device_inter{nullptr}
- , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS))
+ , compressed_device_addr(Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize() >> Memory::YUZU_PAGEBITS)
, tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS)
{
impl = std::make_unique<DeviceMemoryManagerAllocator<Traits>>();
diff --git a/src/core/file_sys/program_metadata.cpp b/src/core/file_sys/program_metadata.cpp
index 63ca9d3fe3..b528523e67 100644
--- a/src/core/file_sys/program_metadata.cpp
+++ b/src/core/file_sys/program_metadata.cpp
@@ -20,6 +20,10 @@ ProgramMetadata::ProgramMetadata() = default;
ProgramMetadata::~ProgramMetadata() = default;
Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) {
+ uses_extended_acid = false;
+ if (!file) {
+ return Loader::ResultStatus::ErrorBadNPDMHeader;
+ }
const std::size_t total_size = file->GetSize();
if (total_size < sizeof(Header)) {
return Loader::ResultStatus::ErrorBadNPDMHeader;
@@ -29,17 +33,55 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) {
return Loader::ResultStatus::ErrorBadNPDMHeader;
}
- if (sizeof(AcidHeader) != file->ReadObject(&acid_header, npdm_header.acid_offset)) {
+ if (npdm_header.magic != std::array{'M', 'E', 'T', 'A'}) {
+ return Loader::ResultStatus::ErrorBadNPDMHeader;
+ }
+
+ const auto contains = [](u64 offset, u64 size, u64 limit) {
+ return offset <= limit && size <= limit - offset;
+ };
+ const u64 acid_offset = npdm_header.acid_offset;
+ const u64 acid_size = npdm_header.acid_size;
+ const u64 aci_offset = npdm_header.aci_offset;
+ const u64 aci_size = npdm_header.aci_size;
+ if (acid_offset < sizeof(Header) || !contains(acid_offset, acid_size, total_size)) {
return Loader::ResultStatus::ErrorBadACIDHeader;
}
- if (sizeof(AciHeader) != file->ReadObject(&aci_header, npdm_header.aci_offset)) {
+ const auto read_acid = [&](u64 body_offset) {
+ return contains(body_offset, sizeof(AcidHeader), acid_size) &&
+ file->ReadObject(&acid_header, acid_offset + body_offset) == sizeof(AcidHeader) &&
+ acid_header.magic == std::array{'A', 'C', 'I', 'D'};
+ };
+ if (!read_acid(0x200)) {
+ if (!read_acid(0x300)) {
+ return Loader::ResultStatus::ErrorBadACIDHeader;
+ }
+ uses_extended_acid = true;
+ }
+
+ if (aci_offset < sizeof(Header) || !contains(aci_offset, aci_size, total_size) ||
+ aci_size < sizeof(AciHeader) ||
+ sizeof(AciHeader) != file->ReadObject(&aci_header, aci_offset) ||
+ aci_header.magic != std::array{'A', 'C', 'I', '0'}) {
return Loader::ResultStatus::ErrorBadACIHeader;
}
+ constexpr u64 fac_read_size = 4 + sizeof(FileAccessControl::permissions) +
+ sizeof(FileAccessControl::unknown);
+ if (acid_header.fac_size < fac_read_size ||
+ !contains(acid_header.fac_offset, acid_header.fac_size, acid_size)) {
+ return Loader::ResultStatus::ErrorBadFileAccessControl;
+ }
+ constexpr u64 fah_read_size = 4 + sizeof(FileAccessHeader::permissions) + 4 * sizeof(u32);
+ if (aci_header.fah_size < fah_read_size ||
+ !contains(aci_header.fah_offset, aci_header.fah_size, aci_size)) {
+ return Loader::ResultStatus::ErrorBadFileAccessHeader;
+ }
+
// Load acid_file_access per-component instead of the entire struct, since this struct does not
// reflect the layout of the real data.
- std::size_t current_offset = acid_header.fac_offset;
+ std::size_t current_offset = acid_offset + acid_header.fac_offset;
if (sizeof(FileAccessControl::version) != file->ReadBytes(&acid_file_access.version,
sizeof(FileAccessControl::version),
current_offset)) {
@@ -57,7 +99,7 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) {
}
// Load aci_file_access per-component instead of the entire struct, same as acid_file_access
- current_offset = aci_header.fah_offset;
+ current_offset = aci_offset + aci_header.fah_offset;
if (sizeof(FileAccessHeader::version) != file->ReadBytes(&aci_file_access.version,
sizeof(FileAccessHeader::version),
current_offset)) {
@@ -89,6 +131,10 @@ Loader::ResultStatus ProgramMetadata::Load(VirtualFile file) {
return Loader::ResultStatus::ErrorBadFileAccessHeader;
}
+ if (aci_header.kac_size % sizeof(u32) != 0 ||
+ !contains(aci_header.kac_offset, aci_header.kac_size, aci_size)) {
+ return Loader::ResultStatus::ErrorBadKernelCapabilityDescriptors;
+ }
aci_kernel_capabilities.resize(aci_header.kac_size / sizeof(u32));
const u64 read_size = aci_header.kac_size;
const u64 read_offset = npdm_header.aci_offset + aci_header.kac_offset;
@@ -128,6 +174,7 @@ void ProgramMetadata::LoadManual(bool is_64_bit, ProgramAddressSpaceType address
u32 main_thread_stack_size, u64 title_id,
u64 filesystem_permissions, u32 system_resource_size,
KernelCapabilityDescriptors capabilities) {
+ uses_extended_acid = false;
npdm_header.has_64_bit_instructions.Assign(is_64_bit);
npdm_header.address_space_type.Assign(address_space);
npdm_header.main_thread_priority = static_cast<u8>(main_thread_prio);
diff --git a/src/core/file_sys/program_metadata.h b/src/core/file_sys/program_metadata.h
index 115e6d6cd1..eb4cf11894 100644
--- a/src/core/file_sys/program_metadata.h
+++ b/src/core/file_sys/program_metadata.h
@@ -77,6 +77,9 @@ public:
u8 GetMainThreadCore() const;
u32 GetMainThreadStackSize() const;
u64 GetTitleID() const;
+ bool UsesExtendedAcid() const {
+ return uses_extended_acid;
+ }
u64 GetFilesystemPermissions() const;
u32 GetSystemResourceSize() const;
PoolPartition GetPoolPartition() const;
@@ -116,8 +119,6 @@ private:
static_assert(sizeof(Header) == 0x80, "NPDM header structure size is wrong");
struct AcidHeader {
- std::array<u8, 0x100> signature;
- std::array<u8, 0x100> nca_modulus;
std::array<char, 4> magic;
u32_le nca_size;
std::array<u8, 0x4> reserved;
@@ -139,7 +140,7 @@ private:
INSERT_PADDING_BYTES(0x8);
};
- static_assert(sizeof(AcidHeader) == 0x240, "ACID header structure size is wrong");
+ static_assert(sizeof(AcidHeader) == 0x40, "ACID header body size is wrong");
struct AciHeader {
std::array<char, 4> magic;
@@ -177,6 +178,7 @@ private:
};
Header npdm_header{};
+ bool uses_extended_acid{};
AciHeader aci_header{};
AcidHeader acid_header{};
diff --git a/src/core/hle/kernel/k_memory_block.h b/src/core/hle/kernel/k_memory_block.h
index f6543bdb3a..2abca31d87 100644
--- a/src/core/hle/kernel/k_memory_block.h
+++ b/src/core/hle/kernel/k_memory_block.h
@@ -195,7 +195,9 @@ enum class KMemoryAttribute : u8 {
Uncached = static_cast<u8>(Svc::MemoryAttribute::Uncached),
PermissionLocked = static_cast<u8>(Svc::MemoryAttribute::PermissionLocked),
- SetMask = Uncached | PermissionLocked,
+ GpuAccessible = static_cast<u8>(Svc::MemoryAttribute::GpuAccessible),
+
+ SetMask = Uncached | PermissionLocked | GpuAccessible,
};
DECLARE_ENUM_FLAG_OPERATORS(KMemoryAttribute);
diff --git a/src/core/hle/kernel/k_page_table_base.cpp b/src/core/hle/kernel/k_page_table_base.cpp
index 2a0a7d49d5..fd5d2c2dce 100644
--- a/src/core/hle/kernel/k_page_table_base.cpp
+++ b/src/core/hle/kernel/k_page_table_base.cpp
@@ -2102,7 +2102,8 @@ Result KPageTableBase::SetMemoryAttribute(KProcessAddress addr, size_t size, KMe
constexpr KMemoryAttribute AttributeTestMask =
~(KMemoryAttribute::SetMask | KMemoryAttribute::DeviceShared);
const KMemoryState state_test_mask =
- (True(mask & KMemoryAttribute::Uncached) ? KMemoryState::FlagCanChangeAttribute
+ (True(mask & (KMemoryAttribute::Uncached | KMemoryAttribute::GpuAccessible))
+ ? KMemoryState::FlagCanChangeAttribute
: KMemoryState::None) |
(True(mask & KMemoryAttribute::PermissionLocked) ? KMemoryState::FlagCanPermissionLock
: KMemoryState::None);
diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp
index 27d1b1bfc4..813d8e9402 100644
--- a/src/core/hle/kernel/k_process.cpp
+++ b/src/core/hle/kernel/k_process.cpp
@@ -1260,6 +1260,7 @@ Result KProcess::LoadFromMetadata(KernelCore& kernel, const FileSys::ProgramMeta
// Assign remaining properties.
m_ideal_core_id = metadata.GetMainThreadCore();
+ m_uses_pauth_returns = metadata.UsesExtendedAcid();
// Set up emulation context.
this->InitializeInterfaces(kernel);
diff --git a/src/core/hle/kernel/k_process.h b/src/core/hle/kernel/k_process.h
index b5693751c3..ca96591ed0 100644
--- a/src/core/hle/kernel/k_process.h
+++ b/src/core/hle/kernel/k_process.h
@@ -135,6 +135,7 @@ private:
bool m_is_signaled : 1 = false;
bool m_is_initialized : 1 = false;
bool m_is_application : 1 = false;
+ bool m_uses_pauth_returns : 1 = false;
bool m_is_default_application_system_resource : 1 = false;
bool m_is_suspended : 1 = false;
bool m_is_immortal : 1 = false;
@@ -248,6 +249,10 @@ public:
return m_is_application;
}
+ bool UsesPAuthReturns() const {
+ return m_uses_pauth_returns;
+ }
+
bool IsDefaultApplicationSystemResource() const {
return m_is_default_application_system_resource;
}
diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp
index 4f1d849209..c7beb9c3ce 100644
--- a/src/core/hle/kernel/kernel.cpp
+++ b/src/core/hle/kernel/kernel.cpp
@@ -120,13 +120,13 @@ struct KernelCore::Impl {
}
void TerminateAllProcesses() {
- std::scoped_lock lk{process_list_lock};
- for (auto& process : process_list) {
- process->Terminate(system.Kernel());
- process->Close(system.Kernel());
- process = nullptr;
+ auto& kernel = system.Kernel();
+ // Applet cleanup can remove processes while termination waits for its threads.
+ auto processes = kernel.GetProcessList();
+ for (auto& process : processes) {
+ process->Terminate(kernel);
+ kernel.RemoveProcess(process.GetPointerUnsafe());
}
- process_list.clear();
}
void Shutdown(KernelCore& kernel) {
diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp
index 3542d5a3b6..c91ae43dd8 100644
--- a/src/core/hle/kernel/svc.cpp
+++ b/src/core/hle/kernel/svc.cpp
@@ -2370,6 +2370,10 @@ static void Call32(Core::System& system, u32 imm, std::span<uint64_t, 8> args) {
static void Call64(Core::System& system, u32 imm, std::span<uint64_t, 8> args) {
switch (SvcId(imm)) {
+ case SvcId::SetGpuMemoryAttribute:
+ args[0] = SetGpuMemoryAttribute(system, args[0], args[1],
+ static_cast<u32>(args[2]), static_cast<u32>(args[3])).raw;
+ return;
case SvcId::SetHeapSize: return SvcWrap_SetHeapSize64(system, args);
case SvcId::SetMemoryPermission: return SvcWrap_SetMemoryPermission64(system, args);
case SvcId::SetMemoryAttribute: return SvcWrap_SetMemoryAttribute64(system, args);
diff --git a/src/core/hle/kernel/svc.h b/src/core/hle/kernel/svc.h
index 356c1c109a..422b8b8719 100644
--- a/src/core/hle/kernel/svc.h
+++ b/src/core/hle/kernel/svc.h
@@ -22,6 +22,7 @@ namespace Kernel::Svc {
Result SetHeapSize(Core::System& system, uint64_t* out_address, uint64_t size);
Result SetMemoryPermission(Core::System& system, uint64_t address, uint64_t size, MemoryPermission perm);
Result SetMemoryAttribute(Core::System& system, uint64_t address, uint64_t size, uint32_t mask, uint32_t attr);
+Result SetGpuMemoryAttribute(Core::System& system, uint64_t address, uint64_t size, uint32_t mask, uint32_t attr);
Result MapMemory(Core::System& system, uint64_t dst_address, uint64_t src_address, uint64_t size);
Result UnmapMemory(Core::System& system, uint64_t dst_address, uint64_t src_address, uint64_t size);
Result QueryMemory(Core::System& system, uint64_t out_memory_info, PageInfo* out_page_info, uint64_t address);
@@ -504,6 +505,7 @@ enum class SvcId : u32 {
CreateResourceLimit = 0x7d,
SetResourceLimitLimitValue = 0x7e,
CallSecureMonitor = 0x7f,
+ SetGpuMemoryAttribute = 0x80,
MapInsecureMemory = 0x90,
UnmapInsecureMemory = 0x91,
};
diff --git a/src/core/hle/kernel/svc/svc_info.cpp b/src/core/hle/kernel/svc/svc_info.cpp
index 863bb58fa8..aea7407542 100644
--- a/src/core/hle/kernel/svc/svc_info.cpp
+++ b/src/core/hle/kernel/svc/svc_info.cpp
@@ -14,11 +14,32 @@
#include "core/hle/kernel/svc_version.h"
namespace Kernel::Svc {
+namespace {
+// Identified from the extended SDK's RemoteRegionManager initialization.
+constexpr InfoType RemoteRegionAddress = static_cast<InfoType>(29);
+constexpr InfoType RemoteRegionSize = static_cast<InfoType>(30);
+constexpr InfoType RemoteMemoryUsed = static_cast<InfoType>(31);
+constexpr InfoType RemoteMemoryTotal = static_cast<InfoType>(32);
+constexpr InfoType MemoryMappingAlignment = static_cast<InfoType>(33);
+} // namespace
/// Gets system/memory information for the current process
Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle handle, u64 info_sub_id) {
LOG_TRACE(Kernel_SVC, "called info_id={:#x}, info_sub_id={:#x}, handle={:#08x}", info_id_type, info_sub_id, handle);
u32 info_id = u32(info_id_type);
+ if (info_id_type == RemoteRegionAddress || info_id_type == RemoteRegionSize ||
+ info_id_type == RemoteMemoryUsed || info_id_type == RemoteMemoryTotal ||
+ info_id_type == MemoryMappingAlignment) {
+ R_UNLESS(info_sub_id == 0, ResultInvalidEnumValue);
+ const auto& handle_table = GetCurrentProcess(system.Kernel()).GetHandleTable();
+ KScopedAutoObject process = handle_table.GetObject<KProcess>(system.Kernel(), handle);
+ R_UNLESS(process.IsNotNull(), ResultInvalidHandle);
+ R_UNLESS(process->UsesPAuthReturns(), ResultInvalidEnumValue);
+ // Remote mappings are absent; ordinary mappings use the kernel page size.
+ *result = info_id_type == MemoryMappingAlignment ? PageSize : 0;
+ LOG_DEBUG(Kernel_SVC, "Extended memory info_id={:#x}, value={:#x}", info_id, *result);
+ R_SUCCEED();
+ }
switch (info_id_type) {
case InfoType::CoreMask:
case InfoType::PriorityMask:
@@ -124,7 +145,8 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle
default:
break;
}
- LOG_ERROR(Kernel_SVC, "Unimplemented svcGetInfo id={:#016x}", info_id);
+ LOG_ERROR(Kernel_SVC, "Unimplemented svcGetInfo id={:#x}, sub_id={:#x}, handle={:#x}",
+ info_id, info_sub_id, handle);
R_THROW(ResultInvalidEnumValue);
}
case InfoType::DebuggerAttached:
@@ -258,7 +280,8 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle
R_SUCCEED();
}
default:
- LOG_ERROR(Kernel_SVC, "Unimplemented svcGetInfo id={:#016x}", info_id);
+ LOG_ERROR(Kernel_SVC, "Unimplemented svcGetInfo id={:#x}, sub_id={:#x}, handle={:#x}",
+ info_id, info_sub_id, handle);
R_THROW(ResultInvalidEnumValue);
}
}
diff --git a/src/core/hle/kernel/svc/svc_ipc.cpp b/src/core/hle/kernel/svc/svc_ipc.cpp
index 875590d6f7..fadde6c093 100644
--- a/src/core/hle/kernel/svc/svc_ipc.cpp
+++ b/src/core/hle/kernel/svc/svc_ipc.cpp
@@ -13,6 +13,7 @@
#include "core/hle/kernel/k_scoped_resource_reservation.h"
#include "core/hle/kernel/k_server_session.h"
#include "core/hle/kernel/k_session.h"
+#include "core/hle/kernel/k_thread.h"
#include "core/hle/kernel/svc.h"
#include "core/hle/kernel/svc_results.h"
@@ -142,7 +143,23 @@ Result ReplyAndReceiveImpl(KernelCore& kernel, int32_t* out_index, uintptr_t mes
/// Makes a blocking IPC call to a service.
Result SendSyncRequest(Core::System& system, Handle session_handle) {
- R_RETURN(SendSyncRequestImpl(system.Kernel(), 0, 0, session_handle));
+ auto& process = GetCurrentProcess(system.Kernel());
+ if (!process.UsesPAuthReturns()) {
+ R_RETURN(SendSyncRequestImpl(system.Kernel(), 0, 0, session_handle));
+ }
+ const auto tls = GetCurrentThread(system.Kernel()).GetTlsAddress();
+ const u64 header = process.GetMemory().Read64(tls);
+ const bool trace_tipc = (header & 0xffff) >= 16;
+ if (trace_tipc) {
+ LOG_DEBUG(Kernel_SVC, "TIPC send handle={:#x}, header={:#x}, special={:#x}",
+ session_handle, header, process.GetMemory().Read64(tls + 8));
+ }
+ const auto result = SendSyncRequestImpl(system.Kernel(), 0, 0, session_handle);
+ if (trace_tipc) {
+ LOG_DEBUG(Kernel_SVC, "TIPC receive result={:#x}, header={:#x}, data={:#x}",
+ result.raw, process.GetMemory().Read64(tls), process.GetMemory().Read64(tls + 8));
+ }
+ R_RETURN(result);
}
Result SendSyncRequestWithUserBuffer(Core::System& system, uint64_t message, uint64_t buffer_size,
diff --git a/src/core/hle/kernel/svc/svc_memory.cpp b/src/core/hle/kernel/svc/svc_memory.cpp
index 970912fe50..53c0d43456 100644
--- a/src/core/hle/kernel/svc/svc_memory.cpp
+++ b/src/core/hle/kernel/svc/svc_memory.cpp
@@ -126,6 +126,31 @@ Result SetMemoryAttribute(Core::System& system, u64 address, u64 size, u32 mask,
R_UNLESS(page_table.Contains(address, size), ResultInvalidCurrentMemory);
// Set the memory attribute.
+ const auto result = page_table.SetMemoryAttribute(address, size,
+ static_cast<KMemoryAttribute>(mask),
+ static_cast<KMemoryAttribute>(attr));
+ if (GetCurrentProcess(system.Kernel()).UsesPAuthReturns()) {
+ LOG_DEBUG(Kernel_SVC, "native memory attribute result={:#x}", result.raw);
+ }
+ R_RETURN(result);
+}
+
+// Captured native SDK callers set bit 0x20 before registering GPU-accessible
+// memory and clear it during destruction. This tracks eligibility on real mapped
+// memory; GPU mappings and submission remain separate operations.
+Result SetGpuMemoryAttribute(Core::System& system, u64 address, u64 size, u32 mask, u32 attr) {
+ auto& process = GetCurrentProcess(system.Kernel());
+ R_UNLESS(process.UsesPAuthReturns(), ResultNotImplemented);
+ LOG_DEBUG(Kernel_SVC, "GPU memory address={:#x}, size={:#x}, mask={:#x}, attr={:#x}",
+ address, size, mask, attr);
+ R_UNLESS(Common::IsAligned(address, PageSize), ResultInvalidAddress);
+ R_UNLESS(size > 0 && Common::IsAligned(size, PageSize), ResultInvalidSize);
+ R_UNLESS(address < address + size, ResultInvalidCurrentMemory);
+ constexpr u32 SupportedMask = static_cast<u32>(MemoryAttribute::GpuAccessible);
+ R_UNLESS((attr & ~mask) == 0 && ((mask | attr) & ~SupportedMask) == 0,
+ ResultInvalidCombination);
+ auto& page_table = process.GetPageTable();
+ R_UNLESS(page_table.Contains(address, size), ResultInvalidCurrentMemory);
R_RETURN(page_table.SetMemoryAttribute(address, size, static_cast<KMemoryAttribute>(mask),
static_cast<KMemoryAttribute>(attr)));
}
diff --git a/src/core/hle/kernel/svc/svc_synchronization.cpp b/src/core/hle/kernel/svc/svc_synchronization.cpp
index 2b019c52d1..2f72f60215 100644
--- a/src/core/hle/kernel/svc/svc_synchronization.cpp
+++ b/src/core/hle/kernel/svc/svc_synchronization.cpp
@@ -15,13 +15,24 @@
namespace Kernel::Svc {
+namespace {
+void LogInvalidNativeSyncHandle(Core::System& system, const char* operation, Handle handle) {
+ if (!GetCurrentProcess(system.Kernel()).UsesPAuthReturns()) { return; }
+ const auto& context = GetCurrentThread(system.Kernel()).GetContext();
+ LOG_DEBUG(Kernel_SVC, "native {} invalid handle={:#x}, pc={:#x}, lr={:#x}, fp={:#x}",
+ operation, handle, context.pc, context.lr, context.fp);
+}
+}
+
/// Close a handle
Result CloseHandle(Core::System& system, Handle handle) {
LOG_TRACE(Kernel_SVC, "Closing handle {:#08x}", handle);
// Remove the handle.
- R_UNLESS(GetCurrentProcess(system.Kernel()).GetHandleTable().Remove(system.Kernel(), handle),
- ResultInvalidHandle);
+ if (!GetCurrentProcess(system.Kernel()).GetHandleTable().Remove(system.Kernel(), handle)) {
+ LogInvalidNativeSyncHandle(system, "CloseHandle", handle);
+ R_THROW(ResultInvalidHandle);
+ }
R_SUCCEED();
}
@@ -49,6 +60,7 @@ Result ResetSignal(Core::System& system, Handle handle) {
}
}
+ LogInvalidNativeSyncHandle(system, "ResetSignal", handle);
R_THROW(ResultInvalidHandle);
}
@@ -72,7 +84,10 @@ Result WaitSynchronization(Core::System& system, int32_t* out_index, u64 user_ha
R_UNLESS(GetCurrentMemory(system.Kernel()).ReadBlock(user_handles, handles.data(), sizeof(Handle) * num_handles), ResultInvalidPointer);
// Convert the handles to objects.
- R_UNLESS(handle_table.GetMultipleObjects<KSynchronizationObject>(system.Kernel(), objs.data(), handles.data(), num_handles), ResultInvalidHandle);
+ if (!handle_table.GetMultipleObjects<KSynchronizationObject>(system.Kernel(), objs.data(), handles.data(), num_handles)) {
+ for (s32 i = 0; i < num_handles; ++i) { LogInvalidNativeSyncHandle(system, "WaitSynchronization set", handles[i]); }
+ R_THROW(ResultInvalidHandle);
+ }
}
// Ensure handles are closed when we're done.
diff --git a/src/core/hle/kernel/svc/svc_thread.cpp b/src/core/hle/kernel/svc/svc_thread.cpp
index dbca13dd26..b4eb0b5719 100644
--- a/src/core/hle/kernel/svc/svc_thread.cpp
+++ b/src/core/hle/kernel/svc/svc_thread.cpp
@@ -16,8 +16,11 @@
namespace Kernel::Svc {
namespace {
-constexpr bool IsValidVirtualCoreId(int32_t core_id) {
- return (0 <= core_id && core_id < static_cast<int32_t>(Core::Hardware::NUM_CPU_CORES));
+bool IsValidVirtualCoreId(const KProcess& process, int32_t core_id) {
+ // Extended processes use the kernel's existing virtual-core mapping.
+ const auto core_count = process.UsesPAuthReturns() ? Core::Hardware::NumVirtualCores
+ : Core::Hardware::NUM_CPU_CORES;
+ return 0 <= core_id && core_id < static_cast<int32_t>(core_count);
}
} // Anonymous namespace
@@ -38,7 +41,7 @@ Result CreateThread(Core::System& system, Handle* out_handle, u64 entry_point, u
}
// Validate arguments.
- R_UNLESS(IsValidVirtualCoreId(core_id), ResultInvalidCoreId);
+ R_UNLESS(IsValidVirtualCoreId(process, core_id), ResultInvalidCoreId);
R_UNLESS(((1ull << core_id) & process.GetCoreMask()) != 0, ResultInvalidCoreId);
R_UNLESS(HighestThreadPriority <= priority && priority <= LowestThreadPriority,
@@ -262,7 +265,7 @@ Result SetThreadCoreMask(Core::System& system, Handle thread_handle, s32 core_id
R_UNLESS(affinity_mask != 0, ResultInvalidCombination);
// Validate the core id.
- if (IsValidVirtualCoreId(core_id)) {
+ if (IsValidVirtualCoreId(GetCurrentProcess(system.Kernel()), core_id)) {
R_UNLESS(((1ULL << core_id) & affinity_mask) != 0, ResultInvalidCombination);
} else {
R_UNLESS(core_id == IdealCoreNoUpdate || core_id == IdealCoreDontCare,
diff --git a/src/core/hle/kernel/svc_types.h b/src/core/hle/kernel/svc_types.h
index b0f4796da4..c7b68cc0df 100644
--- a/src/core/hle/kernel/svc_types.h
+++ b/src/core/hle/kernel/svc_types.h
@@ -50,6 +50,7 @@ enum class MemoryAttribute : u32 {
DeviceShared = (1 << 2),
Uncached = (1 << 3),
PermissionLocked = (1 << 4),
+ GpuAccessible = (1 << 5),
};
DECLARE_ENUM_FLAG_OPERATORS(MemoryAttribute);
diff --git a/src/core/hle/service/am/applet.h b/src/core/hle/service/am/applet.h
index 571fa65219..69d2876363 100644
--- a/src/core/hle/service/am/applet.h
+++ b/src/core/hle/service/am/applet.h
@@ -84,6 +84,7 @@ struct Applet {
// Application functions
bool game_play_recording_supported{};
bool media_playback_state{};
+ bool graphics_scale_changed_notification_enabled{};
GamePlayRecordingState game_play_recording_state{GamePlayRecordingState::Disabled};
bool jit_service_launched{};
bool application_crash_report_enabled{};
diff --git a/src/core/hle/service/am/service/application_functions.cpp b/src/core/hle/service/am/service/application_functions.cpp
index 90d292c4a2..f86806dd3c 100644
--- a/src/core/hle/service/am/service/application_functions.cpp
+++ b/src/core/hle/service/am/service/application_functions.cpp
@@ -118,7 +118,8 @@ IApplicationFunctions::IApplicationFunctions(Core::System& system_, std::shared_
{220, nullptr, "Unknown220"}, // [20.0.0+]
{300, nullptr, "CreateMovieWriter"}, // [19.0.0+]
{310, nullptr, "Unknown310"}, // [20.0.0+]
- {320, nullptr, "Unknown320"}, // [20.0.0+]
+ {320, D<&IApplicationFunctions::GetScreenShotAndContinuousRecordingInitiallyEnabled>,
+ "GetScreenShotAndContinuousRecordingInitiallyEnabled"},
{330, D<&IApplicationFunctions::Unknown330>, "Unknown330"}, // [20.0.0+]
{500, nullptr, "StartContinuousRecordingFlushForDebug"},
{1000, nullptr, "CreateMovieMaker"},
@@ -544,6 +545,17 @@ Result IApplicationFunctions::GetUnknownEvent210(
R_SUCCEED();
}
+Result IApplicationFunctions::GetScreenShotAndContinuousRecordingInitiallyEnabled(
+ Out<bool> out_screenshot_enabled, Out<bool> out_recording_enabled) {
+ std::scoped_lock lk{m_applet->lock};
+ *out_screenshot_enabled = m_applet->screenshot_permission != ScreenshotPermission::Disable;
+ *out_recording_enabled = m_applet->game_play_recording_supported &&
+ m_applet->game_play_recording_state == GamePlayRecordingState::Enabled;
+ LOG_DEBUG(Service_AM, "called, screenshot={}, recording={}", *out_screenshot_enabled,
+ *out_recording_enabled);
+ R_SUCCEED();
+}
+
Result IApplicationFunctions::Unknown330(Out<u8> out) {
LOG_DEBUG(Service_AM, "called");
*out = 0;
diff --git a/src/core/hle/service/am/service/application_functions.h b/src/core/hle/service/am/service/application_functions.h
index b97ed7e25e..172b712f14 100644
--- a/src/core/hle/service/am/service/application_functions.h
+++ b/src/core/hle/service/am/service/application_functions.h
@@ -82,6 +82,8 @@ private:
Result GetHealthWarningDisappearedSystemEvent(OutCopyHandle<Kernel::KReadableEvent> out_event);
Result CreateApplicationAndRequestToStart(u64 application_id);
Result GetUnknownEvent210(OutCopyHandle<Kernel::KReadableEvent> out_event);
+ Result GetScreenShotAndContinuousRecordingInitiallyEnabled(
+ Out<bool> out_screenshot_enabled, Out<bool> out_recording_enabled);
Result Unknown330(Out<u8> out);
Result PrepareForJit();
diff --git a/src/core/hle/service/am/service/application_proxy.cpp b/src/core/hle/service/am/service/application_proxy.cpp
index df438e1e17..02502ae764 100644
--- a/src/core/hle/service/am/service/application_proxy.cpp
+++ b/src/core/hle/service/am/service/application_proxy.cpp
@@ -11,7 +11,9 @@
#include "core/hle/service/am/service/common_state_getter.h"
#include "core/hle/service/am/service/debug_functions.h"
#include "core/hle/service/am/service/display_controller.h"
+#include "core/hle/service/am/service/graphics_scaling_parameters_functions.h"
#include "core/hle/service/am/service/library_applet_creator.h"
+#include "core/hle/service/am/service/performance_functions.h"
#include "core/hle/service/am/service/process_winding_controller.h"
#include "core/hle/service/am/service/self_controller.h"
#include "core/hle/service/am/service/window_controller.h"
@@ -33,6 +35,9 @@ IApplicationProxy::IApplicationProxy(Core::System& system_, std::shared_ptr<Appl
{10, D<&IApplicationProxy::GetProcessWindingController>, "GetProcessWindingController"},
{11, D<&IApplicationProxy::GetLibraryAppletCreator>, "GetLibraryAppletCreator"},
{20, D<&IApplicationProxy::GetApplicationFunctions>, "GetApplicationFunctions"},
+ {40, D<&IApplicationProxy::GetPerformanceFunctions>, "GetPerformanceFunctions"},
+ {60, D<&IApplicationProxy::GetGraphicsScalingParametersFunctions>,
+ "GetGraphicsScalingParametersFunctions"},
{1000, D<&IApplicationProxy::GetDebugFunctions>, "GetDebugFunctions"},
};
// clang-format on
@@ -106,4 +111,18 @@ Result IApplicationProxy::GetApplicationFunctions(
R_SUCCEED();
}
+Result IApplicationProxy::GetPerformanceFunctions(
+ Out<SharedPointer<IPerformanceFunctions>> out_functions) {
+ LOG_DEBUG(Service_AM, "called");
+ *out_functions = std::make_shared<IPerformanceFunctions>(system);
+ R_SUCCEED();
+}
+
+Result IApplicationProxy::GetGraphicsScalingParametersFunctions(
+ Out<SharedPointer<IGraphicsScalingParametersFunctions>> out_functions) {
+ LOG_DEBUG(Service_AM, "called");
+ *out_functions = std::make_shared<IGraphicsScalingParametersFunctions>(system, m_applet);
+ R_SUCCEED();
+}
+
} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/application_proxy.h b/src/core/hle/service/am/service/application_proxy.h
index b56f597605..aa1ab1938a 100644
--- a/src/core/hle/service/am/service/application_proxy.h
+++ b/src/core/hle/service/am/service/application_proxy.h
@@ -17,7 +17,9 @@ class IApplicationFunctions;
class ICommonStateGetter;
class IDebugFunctions;
class IDisplayController;
+class IGraphicsScalingParametersFunctions;
class ILibraryAppletCreator;
+class IPerformanceFunctions;
class IProcessWindingController;
class ISelfController;
class IWindowController;
@@ -43,6 +45,10 @@ private:
Result GetApplicationFunctions(
Out<SharedPointer<IApplicationFunctions>> out_application_functions);
+ Result GetPerformanceFunctions(Out<SharedPointer<IPerformanceFunctions>> out_functions);
+ Result GetGraphicsScalingParametersFunctions(
+ Out<SharedPointer<IGraphicsScalingParametersFunctions>> out_functions);
+
private:
WindowSystem& m_window_system;
Kernel::KProcess* const m_process;
diff --git a/src/core/hle/service/am/service/application_proxy_service.cpp b/src/core/hle/service/am/service/application_proxy_service.cpp
index 7727951d8c..da11cce94d 100644
--- a/src/core/hle/service/am/service/application_proxy_service.cpp
+++ b/src/core/hle/service/am/service/application_proxy_service.cpp
@@ -19,6 +19,8 @@ IApplicationProxyService::IApplicationProxyService(Core::System& system_,
: ServiceFramework{system_, "appletOE"}, m_window_system{window_system} {
static const FunctionInfo functions[] = {
{0, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy"},
+ {10, D<&IApplicationProxyService::OpenApplicationProxyWithFlags>,
+ "OpenApplicationProxyWithFlags"},
};
RegisterHandlers(functions);
}
@@ -40,6 +42,15 @@ Result IApplicationProxyService::OpenApplicationProxy(
}
}
+Result IApplicationProxyService::OpenApplicationProxyWithFlags(
+ Out<SharedPointer<IApplicationProxy>> out_application_proxy, u32 flags, ClientProcessId pid,
+ InCopyHandle<Kernel::KProcess> process_handle) {
+ LOG_DEBUG(Service_AM, "called, flags={:#x}", flags);
+ // The extended SDK supplies an additional u32 before the process ID.
+ R_UNLESS(flags == 0 || flags == 2, ResultUnknown);
+ R_RETURN(OpenApplicationProxy(out_application_proxy, pid, process_handle));
+}
+
std::shared_ptr<Applet> IApplicationProxyService::GetAppletFromProcessId(ProcessId process_id) {
return m_window_system.GetByAppletResourceUserId(process_id.pid);
}
diff --git a/src/core/hle/service/am/service/application_proxy_service.h b/src/core/hle/service/am/service/application_proxy_service.h
index fce860a7c1..5812f6133a 100644
--- a/src/core/hle/service/am/service/application_proxy_service.h
+++ b/src/core/hle/service/am/service/application_proxy_service.h
@@ -26,6 +26,10 @@ private:
Result OpenApplicationProxy(Out<SharedPointer<IApplicationProxy>> out_application_proxy,
ClientProcessId pid, InCopyHandle<Kernel::KProcess> process_handle);
+ Result OpenApplicationProxyWithFlags(
+ Out<SharedPointer<IApplicationProxy>> out_application_proxy, u32 flags, ClientProcessId pid,
+ InCopyHandle<Kernel::KProcess> process_handle);
+
private:
std::shared_ptr<Applet> GetAppletFromProcessId(ProcessId pid);
diff --git a/src/core/hle/service/am/service/gpu_memory_registration.h b/src/core/hle/service/am/service/gpu_memory_registration.h
new file mode 100644
index 0000000000..37ac7fd377
--- /dev/null
+++ b/src/core/hle/service/am/service/gpu_memory_registration.h
@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+#include <optional>
+#include <span>
+#include "common/common_types.h"
+
+namespace Service::AM {
+struct GpuMemoryRegistration {
+ u64 begin;
+ u64 end;
+ u8 flags;
+ u8 device;
+};
+
+constexpr std::optional<GpuMemoryRegistration> DecodeGpuMemoryRegistration(std::span<const u32> packet) {
+ if (packet.size() < 8 || packet[0] != 0x75 || packet[1] != 6) return std::nullopt;
+ // The SDK writes STRB at offsets 0x18 and 0x1c, leaving three padding bytes.
+ return GpuMemoryRegistration{u64{packet[2]} | (u64{packet[3]} << 32),
+ u64{packet[4]} | (u64{packet[5]} << 32), static_cast<u8>(packet[6]), static_cast<u8>(packet[7])};
+}
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/gpu_session.cpp b/src/core/hle/service/am/service/gpu_session.cpp
new file mode 100644
index 0000000000..a83381d7bc
--- /dev/null
+++ b/src/core/hle/service/am/service/gpu_session.cpp
@@ -0,0 +1,633 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <algorithm>
+#include <cstring>
+#include <mutex>
+#include <vector>
+#include "common/swap.h"
+#include "common/alignment.h"
+#include "core/hle/service/nvdrv/core/nvmap.h"
+#include "core/core.h"
+#include "core/hle/kernel/k_client_session.h"
+#include "core/hle/kernel/k_event.h"
+#include "core/hle/kernel/k_process.h"
+#include "core/hle/kernel/svc_results.h"
+#include "core/hle/service/am/service/graphics_session.h"
+#include "core/hle/service/am/service/gpu_session.h"
+#include "core/hle/service/am/service/gpu_memory_registration.h"
+#include "core/hle/service/ipc_helpers.h"
+#include "core/hle/service/nvdrv/nvdrv.h"
+#include "core/hle/service/nvdrv/devices/nvhost_as_gpu.h"
+#include "core/hle/service/nvdrv/devices/native_gpu_kind.h"
+
+namespace Service::AM {
+struct GpuDeviceRegistry {
+ GpuDeviceRegistry(Core::System& system, std::shared_ptr<Nvidia::Module> module_, Kernel::KProcess* process_)
+ : context{system, "GpuDevices"}, module{std::move(module_)}, process{process_} {
+ process->Open(context.kernel);
+ }
+ ~GpuDeviceRegistry() {
+ for (const auto& fd : address_spaces) { if (fd) { module->Close(*fd); } }
+ if (nv_session) { module->GetContainer().CloseSession(Nvidia::NvCore::SessionId{*nv_session}); }
+ process->Close(context.kernel);
+ for (auto* event : events) { context.CloseEvent(event); }
+ }
+ KernelHelpers::ServiceContext context;
+ std::array<Kernel::KEvent*, 64> events{};
+ struct Device { u64 name; u64 debug_label; };
+ std::mutex mutex;
+ std::shared_ptr<Nvidia::Module> module;
+ Kernel::KProcess* process;
+ std::array<std::optional<s32>, 8> address_spaces{};
+ std::array<bool, 8> address_space_registered{};
+ std::optional<size_t> nv_session;
+ std::array<std::optional<Device>, 256> devices{};
+};
+namespace {
+class IGpuWorker final : public ServiceFramework<IGpuWorker> {
+public:
+ IGpuWorker(Core::System& system, std::shared_ptr<Nvidia::Module> module,
+ Kernel::KProcess* process, std::shared_ptr<GpuDeviceRegistry> devices)
+ : ServiceFramework{system, "IGpuWorker"}, m_devices{std::move(devices)},
+ m_module{std::move(module)}, m_process{process} {
+ m_process->Open(kernel);
+ static const FunctionInfo functions[] = {
+ {101, &IGpuWorker::CreateGpuAccessibleMemory, "CreateGpuAccessibleMemory"},
+ {102, &IGpuWorker::DestroyGpuAccessibleMemory, "DestroyGpuAccessibleMemory"},
+ {211, &IGpuWorker::NormalMapGpuMemory4K, "NormalMapGpuMemory4K"},
+ {212, &IGpuWorker::NormalModifyGpuMemory4K, "NormalModifyGpuMemory4K"},
+ {213, &IGpuWorker::NormalUnmapGpuMemory4K, "NormalUnmapGpuMemory4K"},
+ {214, &IGpuWorker::CreateSparseRegion, "CreateSparseRegion"},
+ {215, &IGpuWorker::DestroySparseRegion, "DestroySparseRegion"},
+ };
+ RegisterHandlersTipc(functions);
+ }
+ ~IGpuWorker() override {
+ for (const auto& mapping : m_mappings) {
+ if (auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(mapping.as_id)) {
+ as->UnmapNative4K(mapping.address, mapping.size);
+ }
+ }
+ for (const auto& region : m_sparse_regions) {
+ if (auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(region.as_id)) {
+ as->DestroyNativeSparseRegion(region.address, region.size);
+ }
+ }
+ auto& nvmap = m_module->GetContainer().GetNvMapFile();
+ for (const auto id : m_memory) {
+ if (const auto info = nvmap.FreeHandle(id, false); info && info->can_unlock) {
+ const auto result = m_process->GetPageTable().UnlockForDeviceAddressSpace(
+ info->address, info->size);
+ if (result.IsError()) {
+ LOG_ERROR(Service_AM, "GPU memory cleanup failed: {:#x}", result.raw);
+ }
+ }
+ }
+ if (m_nv_session) {
+ m_module->GetContainer().CloseSession(Nvidia::NvCore::SessionId{*m_nv_session});
+ }
+ m_process->Close(kernel);
+ }
+ Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override {
+ if (!ctx.IsTipc() || handlers_tipc.contains(ctx.GetCommand())) {
+ return ServiceFrameworkBase::HandleSyncRequest(session, ctx);
+ }
+ const auto guard = LockService();
+ InvokeRequestTipc(ctx);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(Kernel::ResultNotImplemented.raw);
+ return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess;
+ }
+private:
+ struct Mapping { u32 as_id; u64 address; u64 size; u32 memory_id; };
+ std::vector<Mapping> m_mappings;
+ std::vector<Mapping> m_sparse_regions;
+ static u64 ReadPair(const u32* packet, size_t index) {
+ return u64{packet[index]} | (u64{packet[index + 1]} << 32);
+ }
+ bool OwnsAddressSpace(u32 id) const {
+ for (size_t i = 0; i < m_devices->address_spaces.size(); ++i) {
+ if (m_devices->address_spaces[i] &&
+ static_cast<u32>(*m_devices->address_spaces[i]) == id &&
+ m_devices->address_space_registered[i]) { return true; }
+ }
+ return false;
+ }
+ void CreateSparseRegion(HLERequestContext& ctx) {
+ const std::scoped_lock device_lock{m_devices->mutex};
+ const auto* p = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ const u64 begin = ReadPair(p, 3), end = ReadPair(p, 5);
+ const u32 notification = p[11];
+ if (p[0] == 0xe6 && p[1] == 10 && notification < m_devices->events.size() && m_devices->events[notification]) {
+ result = Kernel::ResultNotFound;
+ const u32 shift = p[7] & 0xff, device = p[9] & 0xff;
+ if (OwnsAddressSpace(p[2]) && device < m_devices->devices.size() && m_devices->devices[device]) {
+ result = Kernel::ResultInvalidArgument;
+ if ((shift == 12 || shift == 16) && (p[8] & 0xff) <= 1 && (p[10] & 0xff) <= 1 &&
+ end >= begin && end != ~u64{0} && Common::IsAligned(begin, 1ULL << shift) &&
+ Common::IsAligned(end - begin + 1, 1ULL << shift)) {
+ const auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(p[2]);
+ result = as && as->CreateNativeSparseRegion(begin, end - begin + 1) == Nvidia::NvResult::Success
+ ? ResultSuccess : Kernel::ResultInvalidCurrentMemory;
+ if (result.IsSuccess()) { m_sparse_regions.push_back({p[2], begin, end - begin + 1, 0}); }
+ }
+ }
+ m_devices->events[notification]->Signal(kernel);
+ }
+ LOG_DEBUG(Service_AM, "GPU sparse region as={}, VA={:#x}..{:#x}, result={:#x}", p[2], begin, end, result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void DestroySparseRegion(HLERequestContext& ctx) {
+ const std::scoped_lock device_lock{m_devices->mutex};
+ const auto* p = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ const u64 begin = ReadPair(p, 3), end = ReadPair(p, 5);
+ const u32 notification = p[8];
+ if (p[0] == 0xe7 && p[1] == 7 && (p[7] & 0xff) <= 1 &&
+ notification < m_devices->events.size() && m_devices->events[notification]) {
+ const auto it = std::find_if(m_sparse_regions.begin(), m_sparse_regions.end(), [&](const auto& region) {
+ return region.as_id == p[2] && region.address == begin && end >= begin && region.size == end - begin + 1;
+ });
+ result = Kernel::ResultNotFound;
+ if (it != m_sparse_regions.end()) {
+ const auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(p[2]);
+ if (as && as->DestroyNativeSparseRegion(begin, it->size) == Nvidia::NvResult::Success) {
+ std::erase_if(m_mappings, [&](const auto& mapping) {
+ return mapping.as_id == p[2] && mapping.address >= begin && mapping.address <= end;
+ });
+ m_sparse_regions.erase(it);
+ result = ResultSuccess;
+ }
+ }
+ m_devices->events[notification]->Signal(kernel);
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ Result MapMemory(const u32* packet, u8& selected_kind) {
+ R_UNLESS(OwnsAddressSpace(packet[2]), Kernel::ResultNotFound);
+ R_UNLESS(std::find(m_memory.begin(), m_memory.end(), packet[7]) != m_memory.end(),
+ Kernel::ResultNotFound);
+ R_UNLESS(packet[15] < m_devices->devices.size() && m_devices->devices[packet[15]],
+ Kernel::ResultNotFound);
+ R_UNLESS(packet[10] == 0 && packet[11] <= 1 && packet[14] == 0 && packet[16] <= 1,
+ Kernel::ResultNotImplemented);
+ const auto kind = Nvidia::Devices::SelectNativeMemoryKind(
+ static_cast<u8>(packet[12]), static_cast<u8>(packet[13]));
+ R_UNLESS(kind, Kernel::ResultNotImplemented);
+ const u64 begin = ReadPair(packet, 3), end = ReadPair(packet, 5);
+ R_UNLESS(end >= begin && end != ~u64{0}, Kernel::ResultInvalidSize);
+ R_UNLESS(m_mappings.size() < 1024, Kernel::ResultOutOfResource);
+ const auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(packet[2]);
+ R_UNLESS(as, Kernel::ResultNotFound);
+ const auto result = as->MapNative4K(packet[7], begin, ReadPair(packet, 8),
+ end - begin + 1, kind->backend);
+ R_UNLESS(result == Nvidia::NvResult::Success, Kernel::ResultInvalidCurrentMemory);
+ m_mappings.push_back({packet[2], begin, end - begin + 1, packet[7]});
+ selected_kind = kind->selected;
+ R_SUCCEED();
+ }
+ void NormalMapGpuMemory4K(HLERequestContext& ctx) {
+ std::array<u32, 18> words;
+ std::copy_n(ctx.CommandBuffer(), words.size(), words.begin());
+ // These arguments are single-byte TIPC fields; SDK padding is not zeroed.
+ for (size_t i = 10; i <= 16; ++i) { words[i] &= 0xff; }
+ const auto* packet = words.data();
+ const std::scoped_lock device_lock{m_devices->mutex};
+ Result result = Kernel::ResultInvalidCombination;
+ u8 selected_kind{};
+ if (packet[0] == 0xe3 && packet[1] == 16 && packet[17] < m_devices->events.size() &&
+ m_devices->events[packet[17]]) {
+ result = MapMemory(packet, selected_kind);
+ m_devices->events[packet[17]]->Signal(kernel);
+ LOG_DEBUG(Service_AM, "GPU map4K as={}, memory={}, VA={:#x}..{:#x}, permission={}, cache={}, kinds={:#x}/{:#x}, mode={}, sync={}, notification={}, selected={:#x}, result={:#x}",
+ packet[2], packet[7], ReadPair(packet, 3), ReadPair(packet, 5), packet[10],
+ packet[11], packet[12], packet[13], packet[14], packet[16], packet[17], selected_kind, result.raw);
+ }
+ IPC::ResponseBuilder rb{ctx, 3};
+ rb.Push(result.raw);
+ rb.Push<u32>(selected_kind);
+ }
+ void NormalModifyGpuMemory4K(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const std::scoped_lock device_lock{m_devices->mutex};
+ Result result = Kernel::ResultInvalidCombination;
+ u8 selected{};
+ const u64 begin = ReadPair(packet, 3), end = ReadPair(packet, 5);
+ const auto requested = static_cast<u8>(packet[7]);
+ const auto fallback = static_cast<u8>(packet[8]);
+ if (packet[0] == 0xe4 && packet[1] == 9 && (packet[9] & 0xff) <= 1 &&
+ packet[10] < m_devices->events.size() && m_devices->events[packet[10]]) {
+ result = Kernel::ResultNotFound;
+ const auto kind = Nvidia::Devices::SelectNativeMemoryKind(requested, fallback);
+ const auto owner = std::find_if(m_mappings.begin(), m_mappings.end(), [&](const auto& mapping) {
+ return mapping.as_id == packet[2] && begin >= mapping.address &&
+ begin - mapping.address < mapping.size && end >= begin &&
+ end - mapping.address < mapping.size;
+ });
+ if (kind && OwnsAddressSpace(packet[2]) && owner != m_mappings.end()) {
+ const auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(packet[2]);
+ if (as && as->ModifyNativeKind(begin, end - begin + 1, requested, fallback) ==
+ Nvidia::NvResult::Success) {
+ selected = kind->selected;
+ result = ResultSuccess;
+ }
+ }
+ m_devices->events[packet[10]]->Signal(kernel);
+ }
+ LOG_DEBUG(Service_AM, "GPU modify4K as={}, VA={:#x}..{:#x}, kinds={:#x}/{:#x}, sync={}, notification={}, selected={}, result={:#x}",
+ packet[2], begin, end, requested, fallback, packet[9], packet[10], selected, result.raw);
+ IPC::ResponseBuilder rb{ctx, 3};
+ rb.Push(result.raw);
+ rb.Push<u32>(selected);
+ }
+ void NormalUnmapGpuMemory4K(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const std::scoped_lock device_lock{m_devices->mutex};
+ Result result = Kernel::ResultInvalidCombination;
+ u64 unmapped{};
+ if (packet[0] == 0xe5 && packet[1] == 7 && (packet[7] & 0xff) <= 1 &&
+ packet[8] < m_devices->events.size() && m_devices->events[packet[8]]) {
+ const u64 begin = ReadPair(packet, 3), end = ReadPair(packet, 5);
+ const auto it = std::find_if(m_mappings.begin(), m_mappings.end(), [&](const auto& mapping) {
+ return mapping.as_id == packet[2] && mapping.address == begin &&
+ end >= begin && end - begin + 1 == mapping.size;
+ });
+ result = Kernel::ResultNotFound;
+ if (it != m_mappings.end()) {
+ const auto as = m_module->GetDevice<Nvidia::Devices::nvhost_as_gpu>(it->as_id);
+ result = Kernel::ResultInvalidState;
+ if (as && as->UnmapNative4K(it->address, it->size) == Nvidia::NvResult::Success) {
+ unmapped = it->size;
+ m_mappings.erase(it);
+ result = ResultSuccess;
+ }
+ }
+ m_devices->events[packet[8]]->Signal(kernel);
+ LOG_DEBUG(Service_AM, "GPU unmap4K as={}, VA={:#x}..{:#x}, bytes={:#x}, result={:#x}",
+ packet[2], begin, end, unmapped, result.raw);
+ }
+ IPC::ResponseBuilder rb{ctx, 4};
+ rb.Push(result.raw);
+ rb.Push(unmapped);
+ }
+ Result RegisterMemory(u64 begin, u64 end, u32 flags, u32 device, u32& id) {
+ R_UNLESS(flags == 0, Kernel::ResultNotImplemented);
+ R_UNLESS(device < m_devices->devices.size(), Kernel::ResultNotFound);
+ const std::scoped_lock device_lock{m_devices->mutex};
+ R_UNLESS(m_devices->devices[device], Kernel::ResultNotFound);
+ R_UNLESS(end >= begin && end != ~u64{0}, Kernel::ResultInvalidSize);
+ const u64 size = end - begin + 1;
+ R_UNLESS(Common::IsAligned(begin, Kernel::PageSize), Kernel::ResultInvalidAddress);
+ R_UNLESS(Common::IsAligned(size, Kernel::PageSize), Kernel::ResultInvalidSize);
+ R_UNLESS(m_memory.size() < 1024, Kernel::ResultOutOfResource);
+ auto& page_table = m_process->GetPageTable();
+ R_UNLESS(page_table.Contains(begin, size), Kernel::ResultInvalidCurrentMemory);
+ for (u64 address = begin; address <= end;) {
+ Kernel::KMemoryInfo info{};
+ Kernel::Svc::PageInfo page{};
+ R_TRY(page_table.QueryInfo(&info, &page, address));
+ R_UNLESS(True(info.m_attribute & Kernel::KMemoryAttribute::GpuAccessible),
+ Kernel::ResultInvalidCurrentMemory);
+ const u64 next = info.m_address + info.m_size;
+ R_UNLESS(next > address, Kernel::ResultInvalidCurrentMemory);
+ address = next;
+ }
+ if (!m_nv_session) {
+ m_nv_session = m_module->GetContainer().OpenSession(m_process).id;
+ }
+ auto& nvmap = m_module->GetContainer().GetNvMapFile();
+ std::shared_ptr<Nvidia::NvCore::NvMap::Handle> handle;
+ R_UNLESS(nvmap.CreateHandle(size, handle) == Nvidia::NvResult::Success,
+ Kernel::ResultOutOfResource);
+ bool is_io{};
+ const auto lock_result = page_table.LockForMapDeviceAddressSpace(
+ &is_io, begin, size, Kernel::KMemoryPermission::None, true, true);
+ if (lock_result.IsError()) {
+ nvmap.FreeHandle(handle->id, false);
+ R_RETURN(lock_result);
+ }
+ const auto alloc_result = handle->Alloc({}, Kernel::PageSize, 0, begin,
+ Nvidia::NvCore::SessionId{*m_nv_session});
+ if (alloc_result != Nvidia::NvResult::Success) {
+ page_table.UnlockForDeviceAddressSpace(begin, size);
+ nvmap.FreeHandle(handle->id, false);
+ R_RETURN(Kernel::ResultOutOfResource);
+ }
+ id = handle->id;
+ m_memory.push_back(id);
+ R_SUCCEED();
+ }
+ Result ReleaseMemory(u32 id, u32 notification) {
+ const std::scoped_lock device_lock{m_devices->mutex};
+ R_UNLESS(notification < m_devices->events.size() && m_devices->events[notification],
+ Kernel::ResultInvalidCombination);
+ const auto it = std::find(m_memory.begin(), m_memory.end(), id);
+ R_UNLESS(it != m_memory.end(), Kernel::ResultNotFound);
+ auto& nvmap = m_module->GetContainer().GetNvMapFile();
+ auto handle = nvmap.GetHandle(id);
+ R_UNLESS(handle, Kernel::ResultNotFound);
+ {
+ const std::scoped_lock handle_lock{handle->mutex};
+ // No asynchronous GPU work is implemented yet. Never report release
+ // completion for memory still retained or pinned by another user.
+ R_UNLESS(handle->pins == 0 && handle->internal_dupes == 0 && handle->dupes == 1,
+ Kernel::ResultBusy);
+ }
+ // FreeHandle uses the final shared reference to decide when CPU memory
+ // may be unshared. Drop this temporary inspection reference first.
+ handle.reset();
+ const auto info = nvmap.FreeHandle(id, false);
+ R_UNLESS(info && info->can_unlock, Kernel::ResultInvalidState);
+ m_memory.erase(it);
+ R_TRY(m_process->GetPageTable().UnlockForDeviceAddressSpace(info->address, info->size));
+ m_devices->events[notification]->Signal(kernel);
+ R_SUCCEED();
+ }
+ void DestroyGpuAccessibleMemory(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const auto result = packet[0] == 0x76 && packet[1] == 2
+ ? ReleaseMemory(packet[2], packet[3]) : Kernel::ResultInvalidCombination;
+ LOG_DEBUG(Service_AM, "GPU memory release id={}, notification={}, result={:#x}",
+ packet[2], packet[3], result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void CreateGpuAccessibleMemory(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ u32 id{};
+ Result result = Kernel::ResultInvalidCombination;
+ if (const auto request = DecodeGpuMemoryRegistration(std::span<const u32>{packet, 8})) {
+ result = RegisterMemory(request->begin, request->end, request->flags, request->device, id);
+ LOG_DEBUG(Service_AM, "GPU memory {:#x}..{:#x}, device={}, raw_device={:#x}, flags={}, id={}, result={:#x}",
+ request->begin, request->end, request->device, packet[7], request->flags, id, result.raw);
+ }
+ IPC::ResponseBuilder rb{ctx, 3};
+ rb.Push(result.raw);
+ rb.Push(id);
+ }
+ std::shared_ptr<GpuDeviceRegistry> m_devices;
+ std::optional<size_t> m_nv_session;
+ std::vector<u32> m_memory;
+ std::shared_ptr<Nvidia::Module> m_module;
+ Kernel::KProcess* const m_process;
+};
+} // namespace
+
+IGpuSession::IGpuSession(Core::System& system, std::shared_ptr<Nvidia::Module> module,
+ Kernel::KProcess* process, bool legacy_protocol)
+ : ServiceFramework{system, "IGpuSession"},
+ m_process{process}, m_module{std::move(module)}, m_legacy_protocol{legacy_protocol} {
+ m_process->Open(kernel);
+ m_devices = std::make_shared<GpuDeviceRegistry>(system, m_module, m_process);
+ static const FunctionInfo functions[] = {
+ {11, &IGpuSession::GetNotificationRange, "GetNotificationRange"},
+ {12, &IGpuSession::GetNotificationEvent, "GetNotificationEvent"},
+ {21, &IGpuSession::RequestWorker, "RequestWorker"},
+ {22, &IGpuSession::AcquireWorker, "AcquireWorker"},
+ {33, &IGpuSession::CreateAddressSpace, "CreateAddressSpace"},
+ {41, &IGpuSession::CreateGraphicsDevice, "CreateGraphicsDevice"},
+ {42, &IGpuSession::DestroyGraphicsDevice, "DestroyGraphicsDevice"},
+ {201, &IGpuSession::GetAddressSpacePartition, "GetLegacyGsvPartition"},
+ {202, &IGpuSession::GetAddressSpacePartition, "GetGsvPartition"},
+ {203, &IGpuSession::GetAddressSpacePartition, "GetUserPartition"},
+ };
+ RegisterHandlersTipc(functions);
+}
+
+IGpuSession::~IGpuSession() {
+ if (m_worker) { m_worker->Close(kernel); }
+ m_process->Close(kernel);
+}
+
+Result IGpuSession::HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) {
+ if (!ctx.IsTipc() || handlers_tipc.contains(ctx.GetCommand())) {
+ return ServiceFrameworkBase::HandleSyncRequest(session, ctx);
+ }
+ const auto guard = LockService();
+ InvokeRequestTipc(ctx);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(Kernel::ResultNotImplemented.raw);
+ return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess;
+}
+
+Result IGpuSession::RegisterAddressSpace(u32 id) {
+ const auto guard = LockService();
+ const std::scoped_lock device_lock{m_devices->mutex};
+ for (size_t i = 0; i < m_devices->address_spaces.size(); ++i) {
+ if (m_devices->address_spaces[i] && static_cast<u32>(*m_devices->address_spaces[i]) == id) {
+ m_devices->address_space_registered[i] = true;
+ R_SUCCEED();
+ }
+ }
+ R_THROW(Kernel::ResultNotFound);
+}
+
+void IGpuSession::GetAddressSpacePartition(HLERequestContext& ctx) {
+ const std::scoped_lock device_lock{m_devices->mutex};
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ std::array<u64, 9> values{};
+ const bool legacy_user = m_legacy_protocol && (ctx.GetCommand() == 201 || ctx.GetCommand() == 202);
+ const bool user = legacy_user || ctx.GetCommand() == 203;
+ if (!user) {
+ values[0] = ~u64{0}; // No supervisor-owned partition is reserved by this adapter.
+ }
+ if (packet[0] == 16 + ctx.GetCommand() && packet[1] == 1) {
+ const u32 id = packet[2];
+ const bool exists = std::any_of(m_devices->address_spaces.begin(), m_devices->address_spaces.end(),
+ [id](const auto& fd) { return fd && static_cast<u32>(*fd) == id; });
+ result = exists ? ResultSuccess : Kernel::ResultNotFound;
+ if (exists && user) {
+ std::array<u8, 64> input{};
+ std::array<u8, 64> output{};
+ const u32_le buffer_size{48};
+ std::memcpy(input.data() + 8, &buffer_size, sizeof(buffer_size));
+ Nvidia::Ioctl command{};
+ command.raw = 0xc0404108;
+ if (m_module->Ioctl1(static_cast<s32>(id), command, input, output) ==
+ Nvidia::NvResult::Success) {
+ std::array<Nvidia::Devices::nvhost_as_gpu::VaRegion, 2> regions;
+ std::memcpy(regions.data(), output.data() + 16, sizeof(regions));
+ const u64 end = regions[1].offset + regions[1].pages * regions[1].page_size - 1;
+ if (legacy_user) {
+ const auto& region = regions[ctx.GetCommand() == 201 ? 0 : 1];
+ values[0] = region.offset;
+ values[1] = region.offset + region.pages * region.page_size - 1;
+ } else {
+ values[0] = 1;
+ values[1] = regions[0].offset;
+ values[2] = end;
+ }
+ } else {
+ result = Kernel::ResultInvalidState;
+ }
+ }
+ LOG_DEBUG(Service_AM, "GPU partition command={}, id={}, begin={:#x}, end={:#x}, result={:#x}",
+ ctx.GetCommand(), id, user ? values[1] : values[0],
+ user ? values[2] : values[1], result.raw);
+ }
+ IPC::ResponseBuilder rb{ctx, user && !legacy_user ? 20U : 6U};
+ rb.Push(result.raw);
+ for (size_t i = 0; i < (user && !legacy_user ? values.size() : size_t{2}); ++i) {
+ rb.Push(values[i]);
+ }
+}
+
+void IGpuSession::CreateAddressSpace(HLERequestContext& ctx) {
+ const std::scoped_lock device_lock{m_devices->mutex};
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ u32 id{};
+ if (packet[0] == 0x31 && packet[1] == 1 && packet[2] == 0) {
+ result = Kernel::ResultOutOfResource;
+ for (auto& slot : m_devices->address_spaces) {
+ if (slot) {
+ continue;
+ }
+ if (!m_devices->nv_session) {
+ m_devices->nv_session = m_module->GetContainer().OpenSession(m_process).id;
+ }
+ const auto fd = m_module->Open("/dev/nvhost-as-gpu",
+ Nvidia::NvCore::SessionId{*m_devices->nv_session});
+ if (fd == Nvidia::INVALID_NVDRV_FD) {
+ break;
+ }
+ std::array<u8, 40> input{};
+ std::array<u8, 40> output{};
+ const u32_le big_page_size{0x20000};
+ std::memcpy(input.data() + 8, &big_page_size, sizeof(big_page_size));
+ Nvidia::Ioctl command{};
+ command.raw = 0x40284109;
+ const auto nv_result = m_module->Ioctl1(fd, command, input, output);
+ if (nv_result == Nvidia::NvResult::Success) {
+ slot = fd;
+ id = fd;
+ result = ResultSuccess;
+ } else {
+ m_module->Close(fd);
+ result = Kernel::ResultInvalidState;
+ }
+ break;
+ }
+ }
+ LOG_DEBUG(Service_AM, "GPU address space id={}, result={:#x}", id, result.raw);
+ IPC::ResponseBuilder rb{ctx, 3};
+ rb.Push(result.raw);
+ rb.Push(id);
+}
+
+void IGpuSession::CreateGraphicsDevice(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ u32 id{};
+ if (packet[0] == 0x39 && packet[1] == 6) {
+ const u64 name = u64{packet[2]} | (u64{packet[3]} << 32);
+ const u64 flags = u64{packet[4]} | (u64{packet[5]} << 32);
+ const u64 debug_label = u64{packet[6]} | (u64{packet[7]} << 32);
+ result = flags == 0 ? Kernel::ResultOutOfResource : Kernel::ResultNotImplemented;
+ if (flags == 0) {
+ const std::scoped_lock device_lock{m_devices->mutex};
+ for (u32 candidate = 1; candidate < m_devices->devices.size(); ++candidate) {
+ if (!m_devices->devices[candidate]) {
+ id = candidate;
+ m_devices->devices[id] = GpuDeviceRegistry::Device{name, debug_label};
+ result = ResultSuccess;
+ break;
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "graphics device name={:#x}, flags={:#x}, id={}, result={:#x}",
+ name, flags, id, result.raw);
+ }
+ IPC::ResponseBuilder rb{ctx, 3};
+ rb.Push(result.raw);
+ rb.Push(id);
+}
+
+void IGpuSession::DestroyGraphicsDevice(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x3a && packet[1] == 1) {
+ const u8 id = static_cast<u8>(packet[2]);
+ const std::scoped_lock device_lock{m_devices->mutex};
+ result = m_devices->devices[id] ? ResultSuccess : Kernel::ResultNotFound;
+ m_devices->devices[id].reset();
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGpuSession::GetNotificationRange(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const Result result = packet[0] == 0x1b && packet[1] == 0
+ ? ResultSuccess : Kernel::ResultInvalidCombination;
+ IPC::ResponseBuilder rb{ctx, 4};
+ rb.Push(result.raw);
+ rb.Push<u32>(0);
+ rb.Push<u32>(static_cast<u32>(m_devices->events.size() - 1));
+ LOG_DEBUG(Service_AM, "GPU notification range 0..{}", m_devices->events.size() - 1);
+}
+
+void IGpuSession::GetNotificationEvent(HLERequestContext& ctx) {
+ const std::scoped_lock device_lock{m_devices->mutex};
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ Kernel::KReadableEvent* readable{};
+ if (packet[0] == 0x1c && packet[1] == 1 && packet[2] < m_devices->events.size()) {
+ const u32 index = packet[2];
+ if (!m_devices->events[index]) { m_devices->events[index] = m_devices->context.CreateEvent("GpuNotification"); }
+ if (m_devices->events[index]) {
+ readable = &m_devices->events[index]->GetReadableEvent();
+ readable->Open(kernel);
+ result = ResultSuccess;
+ } else { result = Kernel::ResultOutOfResource; }
+ LOG_DEBUG(Service_AM, "GPU notification event index={}, result={:#x}", index, result.raw);
+ }
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(result.raw);
+ ctx.AddMoveObject(readable);
+}
+
+void IGpuSession::RequestWorker(HLERequestContext& ctx) {
+ const std::scoped_lock device_lock{m_devices->mutex};
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x25 && packet[1] == 1 && packet[2] < m_devices->events.size()) {
+ const u32 index = packet[2];
+ if (!m_devices->events[index]) {
+ result = Kernel::ResultInvalidState;
+ } else {
+ result = ResultSuccess;
+ if (!m_worker) {
+ result = CreateGraphicsSession(system, ctx,
+ std::make_shared<IGpuWorker>(system, m_module, m_process, m_devices), &m_worker);
+ }
+ if (result.IsSuccess()) { m_devices->events[index]->Signal(kernel); }
+ }
+ LOG_DEBUG(Service_AM, "GPU worker request index={}, result={:#x}", index, result.raw);
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGpuSession::AcquireWorker(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ Kernel::KClientSession* client{};
+ if (packet[0] == 0x26 && packet[1] == 0) {
+ if (m_worker) {
+ client = m_worker;
+ client->Open(kernel);
+ result = ResultSuccess;
+ } else { result = Kernel::ResultInvalidState; }
+ }
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(result.raw);
+ ctx.AddMoveObject(client);
+ LOG_DEBUG(Service_AM, "GPU worker acquisition result={:#x}", result.raw);
+}
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/gpu_session.h b/src/core/hle/service/am/service/gpu_session.h
new file mode 100644
index 0000000000..e8f580bef1
--- /dev/null
+++ b/src/core/hle/service/am/service/gpu_session.h
@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <array>
+#include <optional>
+#include "core/hle/service/service.h"
+#include "core/hle/service/kernel_helpers.h"
+
+namespace Kernel {
+class KClientSession;
+class KProcess;
+class KEvent;
+}
+namespace Service::Nvidia { class Module; }
+namespace Service::AM {
+
+struct GpuDeviceRegistry;
+
+class IGpuSession final : public ServiceFramework<IGpuSession> {
+public:
+ IGpuSession(Core::System& system, std::shared_ptr<Nvidia::Module> module,
+ Kernel::KProcess* process, bool legacy_protocol = false);
+ ~IGpuSession() override;
+ Result RegisterAddressSpace(u32 id);
+ Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override;
+
+private:
+ void GetAddressSpacePartition(HLERequestContext& ctx);
+ void CreateAddressSpace(HLERequestContext& ctx);
+ void CreateGraphicsDevice(HLERequestContext& ctx);
+ void DestroyGraphicsDevice(HLERequestContext& ctx);
+ void GetNotificationRange(HLERequestContext& ctx);
+ void GetNotificationEvent(HLERequestContext& ctx);
+ void RequestWorker(HLERequestContext& ctx);
+ void AcquireWorker(HLERequestContext& ctx);
+
+ std::shared_ptr<GpuDeviceRegistry> m_devices;
+ Kernel::KClientSession* m_worker{};
+ Kernel::KProcess* const m_process;
+ std::shared_ptr<Nvidia::Module> m_module;
+ const bool m_legacy_protocol;
+};
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/graphics_scaling_parameters_functions.cpp b/src/core/hle/service/am/service/graphics_scaling_parameters_functions.cpp
new file mode 100644
index 0000000000..931c81850d
--- /dev/null
+++ b/src/core/hle/service/am/service/graphics_scaling_parameters_functions.cpp
@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "core/hle/service/am/applet.h"
+#include "core/hle/service/am/service/graphics_scaling_parameters_functions.h"
+#include "core/hle/service/cmif_serialization.h"
+
+namespace Service::AM {
+
+IGraphicsScalingParametersFunctions::IGraphicsScalingParametersFunctions(
+ Core::System& system, std::shared_ptr<Applet> applet)
+ : ServiceFramework{system, "IGraphicsScalingParametersFunctions"}, m_applet{std::move(applet)} {
+ static const FunctionInfo functions[] = {
+ {10, D<&IGraphicsScalingParametersFunctions::GetGraphicsScalingParameters>,
+ "GetGraphicsScalingParameters"},
+ {1000, nullptr, "SetSupportingGraphicsScalingParametersForDebug"},
+ {1010, nullptr, "SetGpuQuotaForDebug"},
+ {1020, nullptr, "ClearGpuQuotaForDebug"},
+ {1030, nullptr, "SetScreenRatioForDebug"},
+ {1040, nullptr, "ClearScreenRatioForDebug"},
+ {1050, D<&IGraphicsScalingParametersFunctions::SetGraphicsScaleChangedNotificationEnabled>,
+ "SetGraphicsScaleChangedNotificationEnabled"},
+ };
+ RegisterHandlers(functions);
+}
+
+IGraphicsScalingParametersFunctions::~IGraphicsScalingParametersFunctions() = default;
+
+Result IGraphicsScalingParametersFunctions::GetGraphicsScalingParameters(
+ Out<GraphicsScalingParameters> out_parameters) {
+ // Prototype policy: lowest SDK scale (10 maps to public scale 0), unit screen ratio.
+ // The ratio field interpretation is inferred from the adjacent debug API.
+ *out_parameters = {10, 1, 1};
+ LOG_DEBUG(Service_AM, "called, prototype scale=10, screen ratio=1/1");
+ R_SUCCEED();
+}
+
+Result IGraphicsScalingParametersFunctions::SetGraphicsScaleChangedNotificationEnabled(bool enabled) {
+ LOG_DEBUG(Service_AM, "called, enabled={}", enabled);
+ std::scoped_lock lk{m_applet->lock};
+ m_applet->graphics_scale_changed_notification_enabled = enabled;
+ R_SUCCEED();
+}
+
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/graphics_scaling_parameters_functions.h b/src/core/hle/service/am/service/graphics_scaling_parameters_functions.h
new file mode 100644
index 0000000000..7c92cb53f4
--- /dev/null
+++ b/src/core/hle/service/am/service/graphics_scaling_parameters_functions.h
@@ -0,0 +1,34 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "core/hle/service/cmif_types.h"
+#include "core/hle/service/service.h"
+
+namespace Service::AM {
+
+struct Applet;
+
+struct GraphicsScalingParameters {
+ u32 graphics_scale;
+ u32 screen_ratio_numerator;
+ u32 screen_ratio_denominator;
+};
+static_assert(sizeof(GraphicsScalingParameters) == 12);
+
+class IGraphicsScalingParametersFunctions final
+ : public ServiceFramework<IGraphicsScalingParametersFunctions> {
+public:
+ explicit IGraphicsScalingParametersFunctions(Core::System& system,
+ std::shared_ptr<Applet> applet);
+ ~IGraphicsScalingParametersFunctions() override;
+
+private:
+ Result GetGraphicsScalingParameters(Out<GraphicsScalingParameters> out_parameters);
+ Result SetGraphicsScaleChangedNotificationEnabled(bool enabled);
+
+ const std::shared_ptr<Applet> m_applet;
+};
+
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/graphics_session.cpp b/src/core/hle/service/am/service/graphics_session.cpp
new file mode 100644
index 0000000000..902e7f9706
--- /dev/null
+++ b/src/core/hle/service/am/service/graphics_session.cpp
@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "core/core.h"
+#include "core/hle/kernel/k_client_session.h"
+#include "core/hle/kernel/k_scoped_resource_reservation.h"
+#include "core/hle/kernel/k_session.h"
+#include "core/hle/kernel/svc_results.h"
+#include "core/hle/service/am/service/graphics_session.h"
+#include "core/hle/service/server_manager.h"
+
+namespace Service::AM {
+Result CreateGraphicsSession(Core::System& system, HLERequestContext& ctx,
+ std::shared_ptr<SessionRequestHandler> handler,
+ Kernel::KClientSession** out_client) {
+ auto& kernel = system.Kernel();
+ Kernel::KScopedResourceReservation reservation(
+ kernel, Kernel::GetCurrentProcessPointer(kernel), Kernel::LimitableResource::SessionCountMax);
+ R_UNLESS(reservation.Succeeded(), Kernel::ResultLimitReached);
+ auto* session = Kernel::KSession::Create(kernel);
+ R_UNLESS(session != nullptr, Kernel::ResultOutOfResource);
+ session->Initialize(kernel, nullptr, 0);
+ reservation.Commit();
+ Kernel::KSession::Register(kernel, session);
+ auto& server = ctx.GetManager()->GetServerManager();
+ auto manager = std::make_shared<SessionRequestManager>(kernel, server);
+ manager->SetSessionHandler(std::move(handler));
+ R_TRY(server.RegisterSession(&session->GetServerSession(), std::move(manager)));
+ *out_client = &session->GetClientSession();
+ R_SUCCEED();
+}
+
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/graphics_session.h b/src/core/hle/service/am/service/graphics_session.h
new file mode 100644
index 0000000000..698b5ee8ee
--- /dev/null
+++ b/src/core/hle/service/am/service/graphics_session.h
@@ -0,0 +1,19 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <memory>
+#include "core/hle/result.h"
+
+namespace Core { class System; }
+namespace Kernel { class KClientSession; }
+namespace Service {
+class HLERequestContext;
+class SessionRequestHandler;
+}
+namespace Service::AM {
+Result CreateGraphicsSession(Core::System& system, HLERequestContext& ctx,
+ std::shared_ptr<SessionRequestHandler> handler,
+ Kernel::KClientSession** out_client);
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/graphics_supervisor.cpp b/src/core/hle/service/am/service/graphics_supervisor.cpp
new file mode 100644
index 0000000000..7759f575e7
--- /dev/null
+++ b/src/core/hle/service/am/service/graphics_supervisor.cpp
@@ -0,0 +1,1612 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <algorithm>
+#include <atomic>
+#include <bit>
+#include <array>
+#include "common/alignment.h"
+#include "core/hle/service/nvdrv/core/nvmap.h"
+#include "core/core.h"
+#include "core/core_timing.h"
+#include "core/hle/service/nvnflinger/buffer_queue_producer.h"
+#include "core/hle/service/nvnflinger/ui/graphic_buffer.h"
+#include "core/hle/kernel/k_client_session.h"
+#include "core/hle/kernel/k_scoped_resource_reservation.h"
+#include "core/hle/kernel/k_session.h"
+#include "core/hle/kernel/k_readable_event.h"
+#include "core/hle/kernel/k_transfer_memory.h"
+#include "core/hle/kernel/k_event.h"
+#include "core/hle/kernel/k_process.h"
+#include "core/hle/kernel/svc_results.h"
+#include "core/hle/service/am/service/graphics_supervisor.h"
+#include "core/hle/service/am/service/graphics_session.h"
+#include "core/hle/service/am/service/gpu_session.h"
+#include "core/hle/service/am/service/graphics_supervisor_protocol.h"
+#include "core/hle/service/am/service/graphics_syncpoint_cache.h"
+#include "core/hle/service/am/service/native_image_layout.h"
+#include "core/hle/service/ipc_helpers.h"
+#include "core/hle/service/nvdrv/nvdrv_interface.h"
+#include "core/hle/service/nvdrv/core/container.h"
+#include "core/hle/service/nvdrv/core/syncpoint_manager.h"
+#include "core/hle/service/server_manager.h"
+#include "core/hle/service/sm/sm.h"
+#include "core/hle/service/vi/application_display_service.h"
+#include "core/hle/service/vi/container.h"
+#include "core/hle/service/vi/manager_root_service.h"
+#include "core/hle/service/vi/vi_types.h"
+
+namespace Service::AM {
+struct NativeImageRegistry {
+ struct Image {
+ u32 id;
+ u32 memory_id;
+ std::array<u32, 16> preference;
+ u64 size;
+ bool window_pool{};
+ };
+ NativeImageRegistry(Kernel::KernelCore& kernel_, Kernel::KProcess* process_)
+ : kernel{kernel_}, process{process_} {
+ process->Open(kernel);
+ }
+ ~NativeImageRegistry() {
+ if (module) {
+ for (const auto& image : images) {
+ ReleaseMemory(image.memory_id);
+ }
+ }
+ if (memory_session) {
+ module->GetContainer().CloseSession(*memory_session);
+ }
+ process->Close(kernel);
+ }
+ void ReleaseMemory(u32 id) {
+ const auto info = module->GetContainer().GetNvMapFile().FreeHandle(id, true);
+ if (info && info->can_unlock && info->address != 0) {
+ process->GetPageTable().UnlockForDeviceAddressSpace(info->address, info->size);
+ }
+ }
+ Kernel::KernelCore& kernel;
+ Kernel::KProcess* process;
+ std::mutex mutex;
+ std::shared_ptr<Nvidia::Module> module;
+ std::optional<Nvidia::NvCore::SessionId> memory_session;
+ std::vector<Image> images;
+};
+using NativeImage = NativeImageRegistry::Image;
+
+
+namespace {
+
+// TIPC payloads follow a 32-bit result, so these handlers push result.raw.
+
+class ISyncpointSupervisor final : public ServiceFramework<ISyncpointSupervisor> {
+public:
+ ISyncpointSupervisor(Core::System& system, std::shared_ptr<Nvidia::Module> module)
+ : ServiceFramework{system, "ISyncpointSupervisor"}, m_module{std::move(module)},
+ m_context{system, "NativeSyncpoint"} {
+ static const FunctionInfo functions[] = {
+ {0, &ISyncpointSupervisor::ExchangeClientToken, "ExchangeClientToken"},
+ {1, &ISyncpointSupervisor::GetMaximumEventCount, "GetMaximumEventCount"},
+ {21, &ISyncpointSupervisor::SetEventCount, "SetEventCount"},
+ {22, &ISyncpointSupervisor::GetEvent, "GetEvent"},
+ {201, &ISyncpointSupervisor::ArmEvent, "ArmEvent"},
+ {202, &ISyncpointSupervisor::CancelEvent, "CancelEvent"},
+ {103, &ISyncpointSupervisor::QueryCounter, "QueryCounter"},
+ };
+ RegisterHandlersTipc(functions);
+ }
+
+ ~ISyncpointSupervisor() override {
+ if (m_poll) system.CoreTiming().UnscheduleEvent(m_poll);
+ for (auto& event : m_events) m_context.CloseEvent(event.event);
+ }
+
+ Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override {
+ if (!ctx.IsTipc() || handlers_tipc.contains(ctx.GetCommand())) {
+ return ServiceFrameworkBase::HandleSyncRequest(session, ctx);
+ }
+ const auto guard = LockService();
+ LOG_DEBUG(Service_AM, "syncpoint unsupported command={}, words={:08x}", ctx.GetCommand(),
+ fmt::join(std::span<const u32>(ctx.CommandBuffer() + 2, std::min<u32>(ctx.CommandBuffer()[1], 32)), ","));
+ InvokeRequestTipc(ctx);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(Kernel::ResultNotImplemented.raw);
+ return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess;
+ }
+
+private:
+ static constexpr u32 MaximumEventCount = 63;
+
+ void ExchangeClientToken(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const Result result = packet[0] == 0x10 && packet[1] == 2
+ ? ResultSuccess : Kernel::ResultInvalidCombination;
+ const u64 token = u64{packet[2]} | (u64{packet[3]} << 32);
+ IPC::ResponseBuilder rb{ctx, 4};
+ rb.Push(result.raw);
+ rb.Push(token);
+ LOG_DEBUG(Service_AM, "syncpoint token={:#x}, result={:#x}", token, result.raw);
+ }
+ void GetMaximumEventCount(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const Result result = packet[0] == 0x11 && packet[1] == 0
+ ? ResultSuccess : Kernel::ResultInvalidCombination;
+ IPC::ResponseBuilder rb{ctx, 3};
+ rb.Push(result.raw);
+ rb.Push(MaximumEventCount);
+ LOG_DEBUG(Service_AM, "syncpoint maximum events={}", MaximumEventCount);
+ }
+
+ void SetEventCount(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x25 && packet[1] == 1 && packet[2] <= MaximumEventCount) {
+ m_event_count = packet[2];
+ result = ResultSuccess;
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ LOG_DEBUG(Service_AM, "syncpoint event count={}, result={:#x}", m_event_count, result.raw);
+ }
+
+ void GetEvent(HLERequestContext& ctx) {
+ const auto* p = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ Kernel::KReadableEvent* readable{};
+ if (p[0] == 0x26 && p[1] == 1 && p[2] < m_event_count) {
+ auto& event = m_events[p[2]];
+ if (!event.event) event.event = m_context.CreateEvent("SyncpointCompletion");
+ result = event.event ? ResultSuccess : Kernel::ResultOutOfResource;
+ if (event.event) {
+ readable = &event.event->GetReadableEvent();
+ readable->Open(kernel);
+ }
+ }
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(result.raw);
+ ctx.AddMoveObject(readable);
+ }
+ void RefreshEvents() {
+ auto& manager = m_module->GetContainer().GetSyncpointManager();
+ for (auto& event : m_events) {
+ if (!event.pending) continue;
+ const auto current = manager.ReadAllocatedSyncpoint(event.id);
+ if (current && event.last_observed != *current) {
+ event.last_observed = *current;
+ LOG_INFO(Service_AM, "Syncpoint wait progress: id={}, current={}, target={}", event.id, *current, event.threshold);
+ }
+ if (current && static_cast<s32>(*current - event.threshold) >= 0) {
+ event.pending = false;
+ event.event->Signal(kernel);
+ }
+ }
+ }
+ void ArmEvent(HLERequestContext& ctx) {
+ const auto* p = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ const u32 id = ~p[3];
+ if (p[0] == 0xd9 && p[1] == 5 && p[2] < m_event_count &&
+ p[5] == 0 && p[6] == 0 &&
+ m_module->GetContainer().GetSyncpointManager().ReadAllocatedSyncpoint(id)) {
+ auto& event = m_events[p[2]];
+ if (!event.event) event.event = m_context.CreateEvent("SyncpointCompletion");
+ result = event.event ? ResultSuccess : Kernel::ResultOutOfResource;
+ if (event.event) {
+ event.event->Clear(kernel);
+ event.id = id;
+ event.threshold = p[4];
+ event.last_observed = ~u32{0};
+ LOG_INFO(Service_AM, "Arm syncpoint wait: slot={}, id={}, target={}", p[2], id, event.threshold);
+ event.pending = true;
+ RefreshEvents();
+ if (!m_poll) {
+ m_poll = Core::Timing::CreateEvent("NativeSyncpointCompletion",
+ [this](s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> {
+ const auto guard = LockService();
+ RefreshEvents();
+ return std::nullopt;
+ });
+ system.CoreTiming().ScheduleLoopingEvent(std::chrono::milliseconds{1},
+ std::chrono::milliseconds{1}, m_poll);
+ }
+ }
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ LOG_DEBUG(Service_AM, "syncpoint arm event={}, id={}, threshold={}, result={:#x}",
+ p[2], id, p[4], result.raw);
+ }
+ void CancelEvent(HLERequestContext& ctx) {
+ const auto* p = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ if (p[0] == 0xda && p[1] == 1 && p[2] < m_event_count) {
+ auto& event = m_events[p[2]];
+ event.pending = false;
+ // The native SDK waits for this event after cancelling a timed wait
+ // before it releases the event slot. Cancellation acknowledges that
+ // the wait is retired; it does not advance the syncpoint counter.
+ if (event.event) event.event->Signal(kernel);
+ result = ResultSuccess;
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+
+ void QueryCounter(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ u64 value{};
+ if (packet[0] == 0x77 && packet[1] == 1) {
+ const u32 id = ~packet[2];
+ auto current = m_module->GetContainer().GetSyncpointManager().ReadAllocatedSyncpoint(id);
+ result = current ? ResultSuccess : Kernel::ResultNotFound;
+ value = current.value_or(0);
+ LOG_DEBUG(Service_AM, "query syncpoint id={}, value={}, result={:#x}", id, value, result.raw);
+ }
+ IPC::ResponseBuilder rb{ctx, 5};
+ rb.Push(result.raw);
+ rb.Push(value);
+ rb.Push<u32>(0); // Existing NVDRV counters use 32-bit modular comparisons.
+ }
+
+ std::shared_ptr<Nvidia::Module> m_module;
+ u32 m_event_count{};
+ struct CompletionEvent {
+ Kernel::KEvent* event{};
+ u32 id{};
+ u32 threshold{};
+ u32 last_observed = ~u32{0};
+ bool pending{};
+ };
+ KernelHelpers::ServiceContext m_context;
+ std::array<CompletionEvent, MaximumEventCount> m_events{};
+ std::shared_ptr<Core::Timing::EventType> m_poll;
+};
+
+class IVi2Session final : public ServiceFramework<IVi2Session> {
+ struct ClientState {
+ u64 token{};
+ u32 slots{};
+ std::optional<u64> display;
+ std::optional<u64> layer;
+ u64 available{0x7fff};
+ u32 retire_notification{};
+ u64 output_encoding{0x14};
+ u32 available_notification{};
+ std::shared_ptr<android::BufferQueueProducer> producer;
+ std::array<std::optional<s32>, 15> queued{};
+ std::optional<size_t> player_display;
+ };
+
+public:
+ explicit IVi2Session(Core::System& system, u64 aruid, std::shared_ptr<NativeImageRegistry> images) : ServiceFramework{system, "IVi2Session"}, m_images{std::move(images)}, m_aruid{aruid}, m_context{system, "Vi2Session"} {
+ static const FunctionInfo functions[] = {
+ {1, &IVi2Session::GetSessionLimit, "GetNotificationLimit"},
+ {11, &IVi2Session::GetNotificationEvent, "GetNotificationEvent"},
+ {13, &IVi2Session::ReleaseNotificationEvent, "ReleaseNotificationEvent"},
+ {21, &IVi2Session::GetNotifications, "GetNotifications"},
+ {31, &IVi2Session::DuplicateVsyncEvent, "GetDisplayEvent"},
+ {32, &IVi2Session::DuplicateDisplayEvent, "DuplicateDisplayEvent"},
+ {3, &IVi2Session::GetSessionLimit, "GetSessionLimit"},
+ {5, &IVi2Session::GetSessionLimit, "GetRequestGroupLimit"},
+ {51, &IVi2Session::ConfigureRequestGroups, "ConfigureRequestGroups"},
+ {101, &IVi2Session::OpenDisplay, "OpenDisplay"},
+ {102, &IVi2Session::CloseDisplay, "CloseDisplay"},
+ {111, &IVi2Session::OpenEndpoint, "OpenEndpoint"},
+ {112, &IVi2Session::CloseEndpoint, "CloseEndpoint"},
+ {201, &IVi2Session::RegisterClientState, "RegisterClientState"},
+ {202, &IVi2Session::UnregisterClientState, "UnregisterClientState"},
+ {212, &IVi2Session::GetClientAvailability, "GetClientAvailability"},
+ {229, &IVi2Session::PresentClientBuffer, "PresentClientBuffer"},
+ {228, &IVi2Session::CancelClientBuffer, "CancelClientBuffer"},
+ {351, &IVi2Session::SetClientOutputEncoding, "SetClientOutputEncoding"},
+ {261, &IVi2Session::SetDisplayCapability, "SetDisplayCapability261"},
+ {265, &IVi2Session::SetDisplayCapability, "SetDisplayCapability265"},
+ {267, &IVi2Session::SetDisplayCapability, "SetDisplayCapability267"},
+ {271, &IVi2Session::SetDisplayCapability, "SetDisplayCapability271"},
+ {251, &IVi2Session::BindClientDisplay, "BindClientDisplay"},
+ {252, &IVi2Session::UnbindClientDisplay, "UnbindClientDisplay"},
+ };
+ RegisterHandlersTipc(functions);
+ }
+
+ ~IVi2Session() override {
+ if (m_release_poll) system.CoreTiming().UnscheduleEvent(m_release_poll);
+ if (m_display_service) {
+ for (auto& state : m_client_states) {
+ if (state.layer) m_display_service->GetContainer()->DestroyManagedLayer(*state.layer);
+ }
+ }
+ if (m_default_display) { m_display_service->CloseDisplay(*m_default_display); }
+ for (auto* event : m_display_change_events) m_context.CloseEvent(event);
+ m_context.CloseEvent(m_event);
+ }
+
+ Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override {
+ if (!ctx.IsTipc() || handlers_tipc.contains(ctx.GetCommand())) {
+ return ServiceFrameworkBase::HandleSyncRequest(session, ctx);
+ }
+ const auto guard = LockService();
+ LOG_DEBUG(Service_AM, "VI2 unsupported command={}, words={:08x}", ctx.GetCommand(),
+ fmt::join(std::span<const u32>(ctx.CommandBuffer() + 2, std::min<u32>(ctx.CommandBuffer()[1], 32)), ","));
+ InvokeRequestTipc(ctx);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(Kernel::ResultNotImplemented.raw);
+ return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess;
+ }
+private:
+ void SetDisplayCapability(HLERequestContext& ctx) {
+ const auto* p = ctx.CommandBuffer();
+ const u64 name = u64{p[2]} | (u64{p[3]} << 32);
+ Result result = Kernel::ResultInvalidCombination;
+ if (p[0] == ctx.GetCommand() + 16 && p[1] == 3 && p[4] <= 1) {
+ result = name == 0x00746c7561666564ULL && m_default_display
+ ? ResultSuccess : Kernel::ResultNotFound;
+ if (result.IsSuccess()) {
+ // Per-client display mode capabilities do not change the host video mode.
+ m_display_capabilities[(ctx.GetCommand() - 261) / 2] = p[4] != 0;
+ }
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void DuplicateVsyncEvent(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ Kernel::KReadableEvent* readable{};
+ if (packet[0] == 0x2f && packet[1] == 1 && packet[2] < 5) {
+ result = m_display_service && m_default_display
+ ? ResultSuccess : Kernel::ResultInvalidState;
+ if (result.IsSuccess() && !m_vsync_event) {
+ result = m_display_service->GetDisplayVsyncEvent(&m_vsync_event, *m_default_display);
+ }
+ if (result.IsSuccess()) {
+ if (packet[2] == 0) {
+ readable = m_vsync_event;
+ } else {
+ // Fixed host display properties have no change to report until changed.
+ auto*& event = m_display_change_events[packet[2] - 1];
+ if (!event) event = m_context.CreateEvent("DisplayPropertyChange");
+ if (event) readable = &event->GetReadableEvent();
+ else result = Kernel::ResultOutOfResource;
+ }
+ if (readable) readable->Open(kernel);
+ }
+ }
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(result.raw);
+ ctx.AddMoveObject(readable);
+ LOG_DEBUG(Service_AM, "VI2 vsync event result={:#x}", result.raw);
+ }
+ void DuplicateDisplayEvent(HLERequestContext& ctx) {
+ const auto* p = ctx.CommandBuffer();
+ auto source = ctx.GetObjectFromHandle<Kernel::KReadableEvent>(ctx.GetCopyHandle(0));
+ const Result result = p[0] == 0x30 && p[1] == 0x80000001 && p[4] <= 1 && source.IsNotNull()
+ ? ResultSuccess : Kernel::ResultInvalidCombination;
+ Kernel::KReadableEvent* readable{};
+ if (result.IsSuccess()) {
+ readable = source.GetPointerUnsafe();
+ readable->Open(kernel);
+ }
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(result.raw);
+ ctx.AddMoveObject(readable);
+ }
+ static std::optional<size_t> GetPlayerDisplayIndex(u64 name) {
+ if ((name & 0xffffffffffffULL) != 0x726579616c70ULL || (name >> 56) != 0) {
+ return std::nullopt;
+ }
+ const u32 player = (name >> 48) & 0xff;
+ if (player < '1' || player > '3') return std::nullopt;
+ return player - '1';
+ }
+ Result OpenDefaultDisplay(u64 name, u32 flags) {
+ if (const auto player = GetPlayerDisplayIndex(name)) {
+ R_UNLESS(flags < m_notifications.size() * 32, Kernel::ResultInvalidCombination);
+ R_UNLESS(m_player_display_references[*player] < 1024, Kernel::ResultOutOfResource);
+ // GameShare's detached player displays retain their own lifetime;
+ // they do not target the host's default display compositor.
+ m_player_display_notifications[*player] = flags;
+ ++m_player_display_references[*player];
+ R_SUCCEED();
+ }
+ // The native display token is an eight-byte name; command 101 returns
+ // only a result. Retain the real VI display ID for later operations.
+ R_UNLESS(name == 0x00746c7561666564ULL, Kernel::ResultNotFound);
+ R_UNLESS(flags == 1, Kernel::ResultNotImplemented);
+ R_UNLESS(m_display_references < 1024, Kernel::ResultOutOfResource);
+ if (!m_display_service) {
+ const auto root = system.ServiceManager().GetService<VI::IManagerRootService>("vi:m", true);
+ R_UNLESS(root, Kernel::ResultInvalidState);
+ R_TRY(root->GetDisplayService(&m_display_service, VI::Policy::Compositor));
+ }
+ if (!m_default_display) {
+ u64 id{};
+ R_TRY(m_display_service->OpenDefaultDisplay(&id));
+ m_default_display = id;
+ }
+ m_display_notification = flags;
+ ++m_display_references;
+ R_SUCCEED();
+ }
+ void OpenDisplay(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 name = u64{packet[2]} | (u64{packet[3]} << 32);
+ const auto result = packet[0] == 0x75 && packet[1] == 3
+ ? OpenDefaultDisplay(name, packet[4]) : Kernel::ResultInvalidCombination;
+ LOG_DEBUG(Service_AM, "VI2 open display name={:#x}, flags={:#x}, references={}, result={:#x}",
+ name, packet[4], m_display_references, result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void CloseDisplay(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ const u64 name = u64{packet[2]} | (u64{packet[3]} << 32);
+ if (packet[0] == 0x76 && packet[1] == 2 && GetPlayerDisplayIndex(name)) {
+ const auto player = *GetPlayerDisplayIndex(name);
+ result = Kernel::ResultNotFound;
+ if (m_player_display_references[player]) {
+ if (--m_player_display_references[player] == 0) {
+ const auto notification = m_player_display_notifications[player];
+ m_notifications[notification / 32] |= 1U << (notification % 32);
+ if (m_event) m_event->Signal(kernel);
+ }
+ result = ResultSuccess;
+ }
+ } else if (packet[0] == 0x76 && packet[1] == 2 && name == 0x00746c7561666564ULL) {
+ result = Kernel::ResultNotFound;
+ if (m_default_display && m_display_references) {
+ result = ResultSuccess;
+ if (m_display_references == 1) {
+ result = m_display_service->CloseDisplay(*m_default_display);
+ if (result.IsSuccess()) { m_default_display.reset(); }
+ }
+ if (result.IsSuccess()) {
+ --m_display_references;
+ if (!m_display_references) {
+ m_notifications[m_display_notification / 32] |= 1U << (m_display_notification % 32);
+ if (m_event) m_event->Signal(kernel);
+ LOG_DEBUG(Service_AM, "VI2 display retired notification={}", m_display_notification);
+ }
+ }
+ }
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void RegisterClientState(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 token = u64{packet[2]} | (u64{packet[3]} << 32);
+ const u32 first = packet[4];
+ const u32 second = packet[5];
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0xd9 && packet[1] == 4 && token != 0 &&
+ first < 32 && second < 32 && first != second) {
+ const u32 slots = (1U << first) | (1U << second);
+ const auto existing = std::ranges::find(m_client_states, token, &ClientState::token);
+ const auto free = std::ranges::find(m_client_states, u64{0}, &ClientState::token);
+ result = Kernel::ResultInvalidState;
+ if (existing == m_client_states.end() && !(m_client_slots & slots)) {
+ result = Kernel::ResultOutOfResource;
+ if (free != m_client_states.end()) {
+ // The SDK sends its object address as an opaque identity, not a
+ // service-accessible buffer. Keep the associated slots session-local.
+ *free = {token, slots};
+ free->retire_notification = first;
+ free->available_notification = second;
+ m_client_slots |= slots;
+ result = ResultSuccess;
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "VI2 register client state token={:#x}, slots={}/{}, result={:#x}",
+ token, first, second, result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void UnregisterClientState(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 token = u64{packet[2]} | (u64{packet[3]} << 32);
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0xda && packet[1] == 2 && token != 0) {
+ const auto state = std::ranges::find(m_client_states, token, &ClientState::token);
+ result = Kernel::ResultNotFound;
+ if (state != m_client_states.end()) {
+ if (m_default_endpoint == token) { m_default_endpoint.reset(); }
+ if (state->layer) m_display_service->GetContainer()->DestroyManagedLayer(*state->layer);
+ m_notifications[state->retire_notification / 32] |= 1U << (state->retire_notification % 32);
+ if (m_event) m_event->Signal(kernel);
+ LOG_DEBUG(Service_AM, "VI2 client retired token={:#x}, notification={}", token, state->retire_notification);
+ m_client_slots &= ~state->slots;
+ *state = {};
+ result = ResultSuccess;
+ }
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void GetClientAvailability(HLERequestContext& ctx) {
+ RefreshReleasedBuffers();
+ const auto* packet = ctx.CommandBuffer();
+ const u64 token = u64{packet[2]} | (u64{packet[3]} << 32);
+ Result result = Kernel::ResultInvalidCombination;
+ u64 id{};
+ if (packet[0] == 0xe4 && packet[1] == 2 && token) {
+ auto state = std::ranges::find(m_client_states, token, &ClientState::token);
+ result = Kernel::ResultNotFound;
+ if (state != m_client_states.end() && m_default_display) {
+ result = ResultSuccess;
+ if (state->player_display) {
+ id = 0;
+ } else if (!state->layer) {
+ result = m_display_service->GetContainer()->CreateManagedLayer(&id, *m_default_display, m_aruid);
+ if (result.IsSuccess()) state->layer = id;
+ } else {
+ id = *state->layer;
+ }
+ }
+ }
+ const auto state = std::ranges::find(m_client_states, token, &ClientState::token);
+ const u64 available = result.IsSuccess() ? state->available : 0;
+ LOG_DEBUG(Service_AM, "VI2 client availability token={:#x}, layer={}, mask={:#x}, result={:#x}", token, id, available, result.raw);
+ IPC::ResponseBuilder rb{ctx, 4};
+ rb.Push(result.raw);
+ rb.Push(available);
+ }
+ void CancelClientBuffer(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 token = u64{packet[2]} | (u64{packet[3]} << 32);
+ const u32 slot = GraphicsSupervisorProtocol::DecodeBufferSlot(packet[4]);
+ Result result = Kernel::ResultInvalidCombination;
+ u64 available{};
+ if (packet[0] == 0xf4 && packet[1] == 19 && slot < 15) {
+ auto state = std::ranges::find(m_client_states, token, &ClientState::token);
+ result = Kernel::ResultNotFound;
+ if (token && state != m_client_states.end()) {
+ // Unlike present (229), cancel has no image or input GPU fence.
+ state->available |= u64{1} << slot;
+ available = state->available;
+ result = ResultSuccess;
+ }
+ }
+ LOG_DEBUG(Service_AM, "VI2 cancel buffer token={:#x}, slot={}, mask={:#x}, result={:#x}", token, slot, available, result.raw);
+ IPC::ResponseBuilder rb{ctx, 8};
+ rb.Push(result.raw);
+ rb.Push(available);
+ for (u32 i = 0; i < 4; ++i) rb.Push<u32>(0);
+ }
+ void RefreshReleasedBuffers() {
+ for (auto& state : m_client_states) {
+ if (!state.producer || !m_images->module) continue;
+ bool changed{};
+ for (size_t slot = 0; slot < state.queued.size(); ++slot) {
+ if (!state.queued[slot]) continue;
+ const auto fence = state.producer->GetReleasedBufferFence(*state.queued[slot]);
+ if (!fence) continue;
+ bool complete = true;
+ for (u32 i = 0; i < fence->num_fences; ++i) {
+ const auto& point = fence->fences[i];
+ if (point.id != ~u32{0}) {
+ const auto counter = m_images->module->GetContainer().GetSyncpointManager().ReadAllocatedSyncpoint(point.id);
+ if (!counter || static_cast<s32>(*counter - point.value) < 0) {
+ complete = false;
+ }
+ }
+ }
+ if (!complete) continue;
+ state.queued[slot].reset();
+ state.available |= u64{1} << slot;
+ LOG_DEBUG(Service_AM, "Native display buffer retired: slot={}, mask={:#x}", slot, state.available);
+ changed = true;
+ }
+ if (changed) {
+ m_notifications[state.available_notification / 32] |= 1U << (state.available_notification % 32);
+ if (m_event) m_event->Signal(kernel);
+ }
+ }
+ }
+ Result QueueNativeImage(ClientState& state, const u32* packet) {
+ const u32 slot = GraphicsSupervisorProtocol::DecodeBufferSlot(packet[4]);
+ R_UNLESS(!state.player_display, Kernel::ResultNotImplemented);
+ R_UNLESS(state.layer && state.display && slot < 15 && !state.queued[slot], Kernel::ResultInvalidState);
+ const std::scoped_lock images_lock{m_images->mutex};
+ const auto image = std::ranges::find(m_images->images, packet[5], &NativeImage::id);
+ R_UNLESS(image != m_images->images.end() && image->window_pool && m_images->module, Kernel::ResultNotFound);
+ // The observed presentation is a four-byte color image with the
+ // backend's 16-GOB block height and no byte offset within its pool.
+ R_UNLESS(packet[6] == 0 && packet[7] == image->preference[2] && packet[8] == 3 &&
+ image->preference[4] == 7 &&
+ (state.output_encoding == 0x14 || state.output_encoding == 0x5),
+ Kernel::ResultNotImplemented);
+ const u32 fence_id = ~packet[22];
+ R_UNLESS(packet[23] == 0 && packet[25] == 0 &&
+ (fence_id == ~u32{0} || m_images->module->GetContainer().GetSyncpointManager().IsSyncpointAllocated(fence_id)),
+ Kernel::ResultInvalidCombination);
+ if (!state.producer) {
+ R_TRY(m_display_service->GetContainer()->GetLayerProducerHandle(&state.producer, *state.layer));
+ android::QueueBufferOutput output{};
+ R_UNLESS(state.producer->Connect({}, android::NativeWindowApi::Egl, false, &output) == android::Status::NoError,
+ Kernel::ResultInvalidState);
+ R_UNLESS(state.producer->SetBufferCount(16) == android::Status::NoError, Kernel::ResultInvalidState);
+ s32 binder{};
+ R_TRY(m_display_service->GetContainer()->OpenLayer(&binder, *state.layer, m_aruid));
+ R_TRY(m_display_service->GetContainer()->SetLayerVisibility(*state.layer, true));
+ }
+ auto native = std::make_shared<android::NvGraphicBuffer>();
+ native->width = static_cast<s32>((image->preference[2] & 0xffffU) + 1);
+ native->height = static_cast<s32>((image->preference[2] >> 16) + 1);
+ native->stride = static_cast<s32>(image->preference[5] * 16);
+ native->format = android::PixelFormat::Rgba8888;
+ native->external_format = native->format;
+ native->buffer_id = image->memory_id;
+ auto buffer = std::make_shared<android::GraphicBuffer>(m_images->module->GetContainer().GetNvMapFile(), native);
+ s32 queue_slot{};
+ R_UNLESS(state.producer->AttachBuffer(&queue_slot, buffer) == android::Status::NoError, Kernel::ResultOutOfResource);
+ android::QueueBufferInput input{};
+ android::QueueBufferOutput output{};
+ input.crop = {0, 0, native->width, native->height};
+ input.swap_interval = 1;
+ input.fence = android::Fence::NoFence();
+ if (fence_id != ~u32{0}) {
+ input.fence.num_fences = 1;
+ input.fence.fences[0] = {static_cast<s32>(fence_id), packet[24]};
+ }
+ if (state.producer->QueueBuffer(queue_slot, input, &output) != android::Status::NoError) {
+ state.producer->CancelBuffer(queue_slot, input.fence);
+ return Kernel::ResultInvalidState;
+ }
+ state.queued[slot] = queue_slot;
+ state.available &= ~(u64{1} << slot);
+ if (!m_release_poll) {
+ m_release_poll = Core::Timing::CreateEvent("NativeViBufferRelease", [this](s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> {
+ const auto guard = LockService();
+ RefreshReleasedBuffers();
+ return std::nullopt;
+ });
+ system.CoreTiming().ScheduleLoopingEvent(std::chrono::milliseconds{1}, std::chrono::milliseconds{1}, m_release_poll);
+ }
+ LOG_INFO(Service_AM, "Native image queued for display: image={}, slot={}, fence={}/{}, dimensions={}x{}",
+ image->id, slot, fence_id, packet[24], native->width, native->height);
+ R_SUCCEED();
+ }
+ void PresentClientBuffer(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 token = u64{packet[2]} | (u64{packet[3]} << 32);
+ RefreshReleasedBuffers();
+ Result result = Kernel::ResultInvalidCombination;
+ u64 available{};
+ if (packet[0] == 0xf5 && packet[1] == 24) {
+ const auto state = std::ranges::find(m_client_states, token, &ClientState::token);
+ result = Kernel::ResultNotFound;
+ if (token && state != m_client_states.end()) {
+ result = QueueNativeImage(*state, packet);
+ available = state->available;
+ }
+ }
+ LOG_DEBUG(Service_AM, "VI2 present token={:#x}, image={}, slot={}, mask={:#x}, result={:#x}", token, packet[5], packet[4], available, result.raw);
+ IPC::ResponseBuilder rb{ctx, 8};
+ rb.Push(result.raw);
+ rb.Push(available);
+ for (u32 i = 0; i < 4; ++i) rb.Push<u32>(0);
+ }
+ void SetClientOutputEncoding(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 token = u64{packet[2]} | (u64{packet[3]} << 32);
+ const u64 encoding = u64{packet[4]} | (u64{packet[5]} << 32);
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x16f && packet[1] == 4) {
+ const auto state = std::ranges::find(m_client_states, token, &ClientState::token);
+ result = Kernel::ResultNotFound;
+ if (token && state != m_client_states.end()) {
+ result = Kernel::ResultNotImplemented;
+ // Only the SDK's observed default output encoding is supported.
+ if (encoding == 0x14 || encoding == 0x5) {
+ state->output_encoding = encoding;
+ result = ResultSuccess;
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "VI2 output encoding token={:#x}, encoding={:#x}, result={:#x}", token, encoding, result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void BindClientDisplay(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 token = u64{packet[2]} | (u64{packet[3]} << 32);
+ const u64 name = u64{packet[4]} | (u64{packet[5]} << 32);
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x10b && packet[1] == 4) {
+ const auto state = std::ranges::find(m_client_states, token, &ClientState::token);
+ result = Kernel::ResultNotFound;
+ if (token && state != m_client_states.end() && GetPlayerDisplayIndex(name)) {
+ const auto player = *GetPlayerDisplayIndex(name);
+ result = Kernel::ResultInvalidState;
+ if (m_player_display_references[player] && !state->display) {
+ result = ResultSuccess;
+ if (state->layer) {
+ result = m_display_service->GetContainer()->DestroyManagedLayer(*state->layer);
+ }
+ if (result.IsSuccess()) {
+ state->layer.reset();
+ state->display = name;
+ state->player_display = player;
+ }
+ }
+ } else if (token && state != m_client_states.end() && name == 0x00746c7561666564ULL &&
+ m_default_display && m_display_references) {
+ result = Kernel::ResultInvalidState;
+ if (!state->display) {
+ state->display = m_default_display;
+ result = ResultSuccess;
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "VI2 bind client token={:#x}, display={:#x}, result={:#x}",
+ token, name, result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void UnbindClientDisplay(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 token = u64{packet[2]} | (u64{packet[3]} << 32);
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x10c && packet[1] == 2) {
+ const auto state = std::ranges::find(m_client_states, token, &ClientState::token);
+ result = Kernel::ResultNotFound;
+ if (token && state != m_client_states.end() && state->display) {
+ state->display.reset();
+ state->player_display.reset();
+ result = ResultSuccess;
+ }
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void OpenEndpoint(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 name = u64{packet[2]} | (u64{packet[3]} << 32);
+ const u64 display_name = u64{packet[4]} | (u64{packet[5]} << 32);
+ const u64 token = u64{packet[6]} | (u64{packet[7]} << 32);
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x7f && packet[1] == 6) {
+ const auto state = std::ranges::find(m_client_states, token, &ClientState::token);
+ result = Kernel::ResultNotFound;
+ if (token && state != m_client_states.end() && state->display &&
+ m_default_display == state->display && name == 0x00746c7561666564ULL &&
+ display_name == name) {
+ result = Kernel::ResultInvalidState;
+ if (!m_default_endpoint) {
+ m_default_endpoint = token;
+ result = ResultSuccess;
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "VI2 open endpoint name={:#x}, display={:#x}, client={:#x}, result={:#x}",
+ name, display_name, token, result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void CloseEndpoint(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 name = u64{packet[2]} | (u64{packet[3]} << 32);
+ const u64 display_name = u64{packet[4]} | (u64{packet[5]} << 32);
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x80 && packet[1] == 4 && name == 0x00746c7561666564ULL &&
+ display_name == name) {
+ result = Kernel::ResultNotFound;
+ if (m_default_endpoint) {
+ m_default_endpoint.reset();
+ result = ResultSuccess;
+ }
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void ConfigureRequestGroups(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x43 && packet[1] == 1 && packet[2] < 8) {
+ m_request_group_limit = packet[2];
+ m_request_groups_initialized = true;
+ result = ResultSuccess;
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void GetSessionLimit(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const Result result = packet[0] == 16 + ctx.GetCommand() && packet[1] == 0
+ ? ResultSuccess : Kernel::ResultInvalidCombination;
+ // The SDK converts this inclusive upper group index into 32 client slots.
+ IPC::ResponseBuilder rb{ctx, 3};
+ rb.Push(result.raw);
+ rb.Push<u32>(ctx.GetCommand() == 5 ? 7 : 0);
+ }
+
+ void GetNotifications(HLERequestContext& ctx) {
+ Result result = Kernel::ResultInvalidSize;
+ if (ctx.GetWriteBufferSize() >= sizeof(m_notifications)) {
+ ctx.WriteBuffer(m_notifications);
+ LOG_DEBUG(Service_AM, "VI2 notifications={:08x}", fmt::join(m_notifications, ","));
+ m_notifications.fill(0);
+ if (m_event) m_event->Clear(kernel);
+ result = ResultSuccess;
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void ReleaseNotificationEvent(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const auto result = packet[0] == 0x1d && packet[1] == 1 && packet[2] == 0
+ ? ResultSuccess : Kernel::ResultInvalidCombination;
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ }
+ void GetNotificationEvent(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ Kernel::KReadableEvent* readable{};
+ if (packet[0] == 0x1b && packet[1] == 1 && packet[2] == 0) {
+ if (!m_event) {
+ m_event = m_context.CreateEvent("Vi2Notification");
+ }
+ result = m_event ? ResultSuccess : Kernel::ResultOutOfResource;
+ if (m_event) {
+ if (std::ranges::any_of(m_notifications, [](u32 value) { return value != 0; })) m_event->Signal(kernel);
+ readable = &m_event->GetReadableEvent();
+ readable->Open(kernel);
+ }
+ }
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(result.raw);
+ ctx.AddMoveObject(readable);
+ }
+
+ std::shared_ptr<NativeImageRegistry> m_images;
+ std::shared_ptr<Core::Timing::EventType> m_release_poll;
+ std::array<ClientState, 16> m_client_states{};
+ u32 m_client_slots{};
+ std::shared_ptr<VI::IApplicationDisplayService> m_display_service;
+ Kernel::KReadableEvent* m_vsync_event{};
+ std::array<Kernel::KEvent*, 4> m_display_change_events{};
+ std::optional<u64> m_default_display;
+ std::array<bool, 6> m_display_capabilities{};
+ u32 m_display_references{};
+ u32 m_display_notification{};
+ std::array<u32, 3> m_player_display_references{};
+ std::array<u32, 3> m_player_display_notifications{};
+ bool m_request_groups_initialized{};
+ u32 m_request_group_limit{};
+ std::optional<u64> m_default_endpoint;
+ std::array<u32, 32> m_notifications{};
+ const u64 m_aruid;
+ KernelHelpers::ServiceContext m_context;
+ Kernel::KEvent* m_event{};
+};
+
+} // namespace
+
+
+IGraphicsSupervisor::IGraphicsSupervisor(Core::System& system, Kernel::KProcess* process, u64 aruid)
+ : ServiceFramework{system, "IGraphicsSupervisor"}, m_context{system, "GraphicsSupervisor"},
+ m_syncpoint_cache_event{m_context}, m_process{process}, m_aruid{aruid} {
+ m_process->Open(kernel);
+ m_image_registry = std::make_shared<NativeImageRegistry>(kernel, m_process);
+ static const FunctionInfo functions[] = {
+ {4, &IGraphicsSupervisor::Initialize, "Initialize"},
+ {11, &IGraphicsSupervisor::BindProcess, "BindProcess"},
+ {101, &IGraphicsSupervisor::CompleteBootstrap, "CompleteBootstrap"},
+ {115, &IGraphicsSupervisor::ConfigureNativeMemory, "ConfigureNativeMemory"},
+ {111, &IGraphicsSupervisor::CreateNativeMemory, "CreateNativeMemory"},
+ {112, &IGraphicsSupervisor::DestroyNativeMemory, "DestroyNativeMemory"},
+ {114, &IGraphicsSupervisor::BindNativeMemory, "BindNativeMemory"},
+ {201, &IGraphicsSupervisor::InitializeSyncpointContext, "InitializeSyncpointContext"},
+ {202, &IGraphicsSupervisor::GetSyncpointSession, "GetSyncpointSession"},
+ {204, &IGraphicsSupervisor::BindSyncpointMemory, "BindSyncpointMemory"},
+ {301, &IGraphicsSupervisor::InitializeDisplayContext, "InitializeDisplayContext"},
+ {305, &IGraphicsSupervisor::OpenDisplaySession, "OpenDisplaySession"},
+ {314, &IGraphicsSupervisor::ConfigureNativeWindowImagePool, "DestroyNativeWindowAliasImagePool"},
+ {315, &IGraphicsSupervisor::ConfigureNativeWindowImagePool, "CreateNativeWindowAliasImagePool"},
+ {401, &IGraphicsSupervisor::InitializeGpuContext, "InitializeGpuContext"},
+ {402, &IGraphicsSupervisor::InitializeGpuContext, "ActivateGpuContext"},
+ {423, &IGraphicsSupervisor::BindNativeGpuMemory, "BindNativeGpuMemory"},
+ {429, &IGraphicsSupervisor::CreateNativeImage, "CreateNativeImage"},
+ {430, &IGraphicsSupervisor::DestroyNativeImage, "DestroyNativeImage"},
+ {434, &IGraphicsSupervisor::RegisterGpuAddressSpace, "RegisterGpuAddressSpace"},
+ {497, &IGraphicsSupervisor::OpenGpuSession, "OpenGpuSession"},
+ {1101, &IGraphicsSupervisor::InitializeFrameContext, "InitializeFrameContext"},
+ {1102, &IGraphicsSupervisor::InitializeFrameContext, "ActivateFrameContext"},
+ {1402, &IGraphicsSupervisor::GetNvidiaDriver, "GetNvidiaDriver"},
+ {1501, &IGraphicsSupervisor::InitializeImageContext, "InitializeImageContext1501"},
+ {435, &IGraphicsSupervisor::CalculateImageRequirements, "CalculateNativeImageSize"},
+ {1510, &IGraphicsSupervisor::CalculateImageRequirements, "CalculateImageRequirements"},
+ {1502, &IGraphicsSupervisor::InitializeImageContext, "InitializeImageContext1502"},
+ };
+ RegisterHandlersTipc(functions);
+}
+
+IGraphicsSupervisor::~IGraphicsSupervisor() {
+ if (m_cache_poll) {
+ system.CoreTiming().UnscheduleEvent(m_cache_poll);
+ }
+ if (m_syncpoint_rw) {
+ m_syncpoint_rw->Close(kernel);
+ m_syncpoint_ro->Close(kernel);
+ }
+ if (m_nv_module) {
+ auto& nvmap = m_nv_module->GetContainer().GetNvMapFile();
+ for (const auto id : m_native_memories) {
+ if (const auto info = nvmap.FreeHandle(id, false); info && info->can_unlock && info->address != 0) {
+ m_process->GetPageTable().UnlockForDeviceAddressSpace(info->address, info->size);
+ }
+ }
+ if (m_native_memory_session) {
+ m_nv_module->GetContainer().CloseSession(Nvidia::NvCore::SessionId{*m_native_memory_session});
+ }
+ }
+ m_context.CloseEvent(m_syncpoint_request_event);
+ m_process->Close(kernel);
+}
+
+Result IGraphicsSupervisor::HandleSyncRequest(Kernel::KServerSession& session,
+ HLERequestContext& ctx) {
+ if (!ctx.IsTipc() || handlers_tipc.contains(ctx.GetCommand())) {
+ return ServiceFrameworkBase::HandleSyncRequest(session, ctx);
+ }
+ const auto guard = LockService();
+ InvokeRequestTipc(ctx);
+ // Unsupported TIPC must return an error instead of leaving request data as a reply.
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(Kernel::ResultNotImplemented.raw);
+ return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess;
+}
+
+void IGraphicsSupervisor::Initialize(HLERequestContext& ctx) {
+ const auto request = GraphicsSupervisorProtocol::DecodeBootstrap(
+ std::span<const u32>{ctx.CommandBuffer(), IPC::COMMAND_BUFFER_LENGTH});
+ Result result = Kernel::ResultInvalidCombination;
+ if (request) {
+ auto process = ctx.GetObjectFromHandle<Kernel::KProcess>(request->process_handle);
+ const bool empty_remote_memory = std::ranges::all_of(
+ request->parameters, [](u64 value) { return value == 0; });
+ if (process.IsNull() || process.GetPointerUnsafe() != m_process) {
+ result = Kernel::ResultInvalidHandle;
+ } else if (m_phase != Phase::Created) {
+ result = Kernel::ResultInvalidState;
+ } else if (request->applet_resource_user_id != m_aruid) {
+ result = Kernel::ResultInvalidArgument;
+ } else if ((request->flags != 0x8000000000000001ULL && request->flags != 0x8000000000000005ULL && request->flags != 0x8000000000000006ULL && request->flags != 0x8000000000000007ULL) || !empty_remote_memory) {
+ result = Kernel::ResultNotImplemented;
+ } else {
+ m_phase = Phase::Initialized;
+ result = ResultSuccess;
+ }
+ LOG_DEBUG(Service_AM, "graphics bootstrap aruid={:#x}, flags={:#x}, result={:#x}",
+ request->applet_resource_user_id, request->flags, result.raw);
+ }
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::BindProcess(HLERequestContext& ctx) {
+ Result result = Kernel::ResultInvalidCombination;
+ const auto* packet = ctx.CommandBuffer();
+ if (packet[0] == 0x1b && packet[1] == 0x80000000 && packet[2] == 2) {
+ auto process = ctx.GetObjectFromHandle<Kernel::KProcess>(ctx.GetCopyHandle(0));
+ if (process.IsNull() || process.GetPointerUnsafe() != m_process) {
+ result = Kernel::ResultInvalidHandle;
+ } else if (m_phase != Phase::Initialized) {
+ result = Kernel::ResultInvalidState;
+ } else {
+ m_phase = Phase::ProcessBound;
+ result = ResultSuccess;
+ }
+ }
+ LOG_DEBUG(Service_AM, "graphics process binding result={:#x}", result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::CompleteBootstrap(HLERequestContext& ctx) {
+ Result result = Kernel::ResultInvalidCombination;
+ const auto* packet = ctx.CommandBuffer();
+ if (packet[0] == 0x75 && packet[1] == 0) {
+ if (m_phase != Phase::ProcessBound) {
+ result = Kernel::ResultInvalidState;
+ } else {
+ m_phase = Phase::Ready;
+ result = ResultSuccess;
+ }
+ }
+ LOG_DEBUG(Service_AM, "graphics bootstrap completion result={:#x}", result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::InitializeImageContext(HLERequestContext& ctx) {
+ Result result = Kernel::ResultInvalidCombination;
+ const auto* packet = ctx.CommandBuffer();
+ if (packet[0] == 16 + ctx.GetCommand() && packet[1] == 0) {
+ if (m_phase != Phase::Ready) {
+ result = Kernel::ResultInvalidState;
+ } else if (ctx.GetCommand() == 1501) {
+ m_image_context_initialized = true;
+ result = ResultSuccess;
+ } else if (m_image_context_initialized) {
+ m_image_formats_initialized = true;
+ result = ResultSuccess;
+ } else {
+ result = Kernel::ResultInvalidState;
+ }
+ }
+ LOG_DEBUG(Service_AM, "empty image context command={}, result={:#x}",
+ ctx.GetCommand(), result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::OpenGpuSession(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ Kernel::KClientSession* client{};
+ const u64 flags = u64{packet[2]} | (u64{packet[3]} << 32);
+ if (packet[0] == 0x201 && packet[1] == 2 && (flags == 3 || flags == 4)) {
+ if (m_phase != Phase::Ready || !m_nv_module) {
+ result = Kernel::ResultInvalidState;
+ } else {
+ auto handler = std::make_shared<IGpuSession>(system, m_nv_module, m_process, flags == 3);
+ result = CreateGraphicsSession(system, ctx, handler, &client);
+ if (result.IsSuccess()) {
+ if (flags == 3) { m_image_formats_initialized = true; }
+ std::erase_if(m_gpu_sessions, [](const auto& item) { return item.expired(); });
+ m_gpu_sessions.emplace_back(handler);
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "GPU control session flags={:#x}, result={:#x}", flags, result.raw);
+ // This SDK reads the move-handle slot before the result, including on failure.
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(result.raw);
+ ctx.AddMoveObject(client);
+}
+
+void IGraphicsSupervisor::RegisterGpuAddressSpace(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x1c2 && packet[1] == 1) {
+ result = m_gpu_context_ready ? Kernel::ResultNotFound : Kernel::ResultInvalidState;
+ if (m_gpu_context_ready) {
+ for (const auto& item : m_gpu_sessions) {
+ if (auto handler = item.lock()) {
+ result = handler->RegisterAddressSpace(packet[2]);
+ if (result.IsSuccess()) {
+ break;
+ }
+ }
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "register GPU address space id={}, result={:#x}", packet[2], result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::InitializeDisplayContext(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x13d && packet[1] == 0) {
+ result = m_phase == Phase::Ready ? ResultSuccess : Kernel::ResultInvalidState;
+ if (result.IsSuccess()) {
+ m_display_context_initialized = true;
+ }
+ }
+ LOG_DEBUG(Service_AM, "display context initialization result={:#x}", result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::OpenDisplaySession(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Kernel::KClientSession* client{};
+ const u64 flags = u64{packet[2]} | (u64{packet[3]} << 32);
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x141 && packet[1] == 2 && (flags == 0x300010000ULL || flags == 0x300020000ULL)) {
+ result = m_display_context_initialized ? ResultSuccess : Kernel::ResultInvalidState;
+ if (result.IsSuccess()) {
+ result = CreateGraphicsSession(system, ctx, std::make_shared<IVi2Session>(system, m_aruid, m_image_registry),
+ &client);
+ }
+ }
+ LOG_DEBUG(Service_AM, "VI2 session flags={:#x}, result={:#x}", flags, result.raw);
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(result.raw);
+ ctx.AddMoveObject(client);
+}
+
+void IGraphicsSupervisor::InitializeGpuContext(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 16 + ctx.GetCommand() && packet[1] == 0) {
+ if (m_phase != Phase::Ready) {
+ result = Kernel::ResultInvalidState;
+ } else if (ctx.GetCommand() == 401) {
+ m_gpu_context_initialized = true;
+ result = ResultSuccess;
+ } else if (m_gpu_context_initialized) {
+ m_gpu_context_ready = true;
+ result = ResultSuccess;
+ } else {
+ result = Kernel::ResultInvalidState;
+ }
+ }
+ LOG_DEBUG(Service_AM, "GPU context command={}, result={:#x}", ctx.GetCommand(), result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::InitializeFrameContext(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 16 + ctx.GetCommand() && packet[1] == 0) {
+ if (m_phase != Phase::Ready) {
+ result = Kernel::ResultInvalidState;
+ } else if (ctx.GetCommand() == 1101) {
+ m_frame_context_initialized = true;
+ result = ResultSuccess;
+ } else if (m_frame_context_initialized) {
+ m_frame_context_ready = true;
+ result = ResultSuccess;
+ } else {
+ result = Kernel::ResultInvalidState;
+ }
+ }
+ LOG_DEBUG(Service_AM, "empty frame context command={}, result={:#x}", ctx.GetCommand(), result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::ConfigureNativeMemory(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x83 && packet[1] == 2 && packet[3] == 0 &&
+ (packet[2] == 0 || packet[2] == 2)) {
+ result = m_phase == Phase::Ready ? ResultSuccess : Kernel::ResultInvalidState;
+ }
+ // Clear the optional Gpuk-memory mode: Eden currently provides the NVDRV path.
+ IPC::ResponseBuilder rb{ctx, 4};
+ rb.Push(result.raw);
+ rb.Push<u64>(0);
+ LOG_DEBUG(Service_AM, "native memory uses NVDRV-compatible mode, result={:#x}", result.raw);
+}
+
+void IGraphicsSupervisor::InitializeSyncpointContext(HLERequestContext& ctx) {
+ Result result = Kernel::ResultInvalidCombination;
+ const auto* packet = ctx.CommandBuffer();
+ if (packet[0] == 0xd9 && packet[1] == 0) {
+ if (m_phase != Phase::Ready) {
+ result = Kernel::ResultInvalidState;
+ } else {
+ m_syncpoint_context_initialized = true;
+ result = ResultSuccess;
+ }
+ }
+ LOG_DEBUG(Service_AM, "syncpoint context initialization result={:#x}", result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::GetSyncpointSession(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidState;
+ Kernel::KClientSession* client{};
+ if (packet[0] == 0xda && packet[1] == 0 && m_syncpoint_context_initialized && m_nv_module) {
+ result = CreateGraphicsSession(system, ctx,
+ std::make_shared<ISyncpointSupervisor>(system, m_nv_module), &client);
+ }
+ if (result.IsError()) {
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ return;
+ }
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(ResultSuccess.raw);
+ rb.PushMoveObjects(ctx, client);
+ LOG_DEBUG(Service_AM, "opened syncpoint-supervisor session");
+}
+
+void IGraphicsSupervisor::GetNvidiaDriver(HLERequestContext& ctx) {
+ const auto Fail = [&](Result result) {
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+ };
+ const auto* packet = ctx.CommandBuffer();
+ if (packet[0] != 0x58a || packet[1] != 0) {
+ Fail(Kernel::ResultInvalidCombination);
+ return;
+ }
+ if (m_phase != Phase::Ready) {
+ Fail(Kernel::ResultInvalidState);
+ return;
+ }
+ auto source = system.ServiceManager().GetService<Nvidia::NVDRV>("nvdrv:s", true);
+ if (!source) {
+ Fail(Kernel::ResultNotFound);
+ return;
+ }
+ m_nv_module = source->GetModule();
+ Kernel::KClientSession* client{};
+ const auto result = CreateGraphicsSession(system, ctx,
+ std::make_shared<Nvidia::NVDRV>(system, m_nv_module, "nvdrv:gsv", m_process, m_aruid),
+ &client);
+ if (result.IsError()) {
+ Fail(result);
+ return;
+ }
+ LOG_DEBUG(Service_AM, "opened process-bound NVDRV session for aruid={:#x}", m_aruid);
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(ResultSuccess.raw);
+ rb.PushMoveObjects(ctx, client);
+}
+
+void IGraphicsSupervisor::BindSyncpointMemory(HLERequestContext& ctx) {
+ using namespace GraphicsSyncpointCache;
+ const auto Fail = [&](Result result) {
+ IPC::ResponseBuilder rb{ctx, 2, 0, 2};
+ rb.Push(result.raw);
+ ctx.AddMoveObject(nullptr);
+ ctx.AddMoveObject(nullptr);
+ };
+ const auto* packet = ctx.CommandBuffer();
+ if (packet[0] != 0xdc || packet[1] != 0x80000006 || packet[2] != 4 ||
+ packet[5] != 1 || packet[6] != 0x1ff || packet[7] != 0x1000 || packet[8] != 0 ||
+ packet[9] != 0x2000 || packet[10] != 0) {
+ Fail(Kernel::ResultInvalidCombination);
+ return;
+ }
+ if (!m_syncpoint_context_initialized || !m_nv_module || m_syncpoint_rw) {
+ Fail(Kernel::ResultInvalidState);
+ return;
+ }
+ auto rw = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(ctx.GetCopyHandle(0));
+ auto ro = ctx.GetObjectFromHandle<Kernel::KTransferMemory>(ctx.GetCopyHandle(1));
+ if (rw.IsNull() || ro.IsNull() || rw->GetOwner() != m_process || ro->GetOwner() != m_process ||
+ rw.GetPointerUnsafe() == ro.GetPointerUnsafe()) {
+ Fail(Kernel::ResultInvalidHandle);
+ return;
+ }
+ if (rw->GetSize() != 0x1000 || ro->GetSize() != 0x2000) {
+ Fail(Kernel::ResultInvalidSize);
+ return;
+ }
+ auto* request_event = m_context.CreateEvent("SyncpointCacheRequest");
+ if (request_event == nullptr) {
+ Fail(Kernel::ResultOutOfResource);
+ return;
+ }
+ m_syncpoint_request_event = request_event;
+ m_syncpoint_rw = rw.GetPointerUnsafe();
+ m_syncpoint_ro = ro.GetPointerUnsafe();
+ m_syncpoint_rw->Open(kernel);
+ m_syncpoint_ro->Open(kernel);
+ auto& memory = m_process->GetMemory();
+ const auto rw_base = rw->GetSourceAddress();
+ const auto ro_base = ro->GetSourceAddress();
+ const std::array<u8, 0x2000> zeros{};
+ memory.WriteBlock(rw_base, zeros.data(), 0x1000);
+ memory.WriteBlock(ro_base, zeros.data(), 0x2000);
+ memory.WriteBlock(ro_base, Header.data(), sizeof(Header));
+ for (u32 id = 0; id < CounterCount; ++id) {
+ memory.Write64(ro_base + ValuesOffset + id * sizeof(u64), Unallocated);
+ }
+ m_cache_poll = Core::Timing::CreateEvent("GraphicsSyncpointCache",
+ [this](s64, std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> {
+ const auto guard = LockService();
+ this->RefreshSyncpointCache();
+ return std::nullopt;
+ });
+ system.CoreTiming().ScheduleLoopingEvent(std::chrono::milliseconds{1},
+ std::chrono::milliseconds{1}, m_cache_poll);
+ request_event->Open(kernel);
+ auto* completed = m_syncpoint_cache_event.GetHandle();
+ completed->Open(kernel);
+ IPC::ResponseBuilder rb{ctx, 2, 0, 2};
+ rb.Push(ResultSuccess.raw);
+ ctx.AddMoveObject(request_event);
+ ctx.AddMoveObject(completed);
+ LOG_DEBUG(Service_AM, "bound syncpoint cache: {} real counters, rw={:#x}, ro={:#x}",
+ CounterCount, GetInteger(rw_base), GetInteger(ro_base));
+}
+
+void IGraphicsSupervisor::RefreshSyncpointCache() {
+ using namespace GraphicsSyncpointCache;
+ if (!m_syncpoint_rw || !system.IsPoweredOn()) {
+ return;
+ }
+ auto& memory = m_process->GetMemory();
+ const auto rw = m_syncpoint_rw->GetSourceAddress();
+ const auto ro = m_syncpoint_ro->GetSourceAddress();
+ const u64 requests = memory.Read64(rw + GroupRequestOffset);
+ const u64 acknowledgements = memory.Read64(ro + GroupAckOffset);
+ u64 groups = (requests ^ acknowledgements) & ((1ULL << GroupCount) - 1);
+ if (groups == 0) {
+ return;
+ }
+ auto& syncpoints = m_nv_module->GetContainer().GetSyncpointManager();
+ while (groups) {
+ const u32 group = static_cast<u32>(std::countr_zero(groups));
+ groups &= groups - 1;
+ const u64 requested_slots = memory.Read64(rw + SlotRequestOffset + group * sizeof(u64));
+ u64 slots = requested_slots ^ memory.Read64(ro + SlotAckOffset + group * sizeof(u64));
+ while (slots) {
+ const u32 bit = static_cast<u32>(std::countr_zero(slots));
+ slots &= slots - 1;
+ const u32 id = group * 64 + bit;
+ const auto address = ro + ValuesOffset + id * sizeof(u64);
+ const auto value = syncpoints.ReadAllocatedSyncpoint(id);
+ const u64 updated = m_counter_values[id].Observe(value);
+ memory.Write64(address, updated);
+ }
+ std::atomic_thread_fence(std::memory_order_release);
+ memory.Write64(ro + SlotAckOffset + group * sizeof(u64), requested_slots);
+ }
+ std::atomic_thread_fence(std::memory_order_release);
+ memory.Write64(ro + GroupAckOffset, requests);
+ m_syncpoint_request_event->Clear(kernel);
+ m_syncpoint_cache_event.Signal(kernel);
+}
+
+void IGraphicsSupervisor::CalculateImageRequirements(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ std::optional<NativeImageRequirements> requirements;
+ const bool gpu_image = ctx.GetCommand() == 435;
+ if (packet[0] == (gpu_image ? 0x1c3U : 0x5f6U) && packet[1] == 17 &&
+ packet[3] == (gpu_image ? 1U : 2U)) {
+ result = Kernel::ResultInvalidState;
+ if (m_phase == Phase::Ready && m_image_formats_initialized) {
+ requirements = CalculateNativeImageRequirements(std::span{packet + 2, 16}, packet[18]);
+ result = requirements ? ResultSuccess : Kernel::ResultNotImplemented;
+ }
+ }
+ LOG_DEBUG(Service_AM, "native image requirements dimensions={:#x}, block={}, pitch={}, size={:#x}, alignment={:#x}, result={:#x}",
+ packet[4], packet[6], packet[7], requirements ? requirements->size : 0,
+ requirements ? requirements->alignment : 0, result.raw);
+ IPC::ResponseBuilder rb{ctx, gpu_image ? 4U : 6U};
+ rb.Push(result.raw);
+ rb.Push<u64>(requirements ? requirements->size : 0);
+ if (!gpu_image) { rb.Push<u64>(requirements ? requirements->alignment : 0); }
+}
+void IGraphicsSupervisor::CreateNativeMemory(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 size = u64{packet[3]} | (u64{packet[4]} << 32);
+ Result result = Kernel::ResultInvalidCombination;
+ u32 id{};
+ if (packet[0] == 0x7f && packet[1] == 3 && (packet[2] & 0xff) == 3) {
+ result = Kernel::ResultInvalidState;
+ if (m_phase == Phase::Ready && m_image_formats_initialized && m_nv_module) {
+ result = Kernel::ResultInvalidSize;
+ if (size && size <= (1ULL << 32) && Common::IsAligned(size, Kernel::PageSize)) {
+ result = Kernel::ResultOutOfResource;
+ if (m_native_memories.size() < 1024) {
+ auto& nvmap = m_nv_module->GetContainer().GetNvMapFile();
+ std::shared_ptr<Nvidia::NvCore::NvMap::Handle> handle;
+ if (nvmap.CreateHandle(size, handle) == Nvidia::NvResult::Success) {
+ id = handle->id;
+ m_native_memories.push_back(id);
+ result = ResultSuccess;
+ }
+ }
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "native memory create size={:#x}, id={}, result={:#x}", size, id, result.raw);
+ IPC::ResponseBuilder rb{ctx, 3};
+ rb.Push(result.raw);
+ rb.Push(id);
+}
+
+Result IGraphicsSupervisor::BindNativeMemoryBacking(u32 id, u64 address, u64 size, u64 flags, u64 offset) {
+ R_UNLESS(m_nv_module && std::ranges::find(m_native_memories, id) != m_native_memories.end(),
+ Kernel::ResultNotFound);
+ R_UNLESS((flags == 0 || flags == 2) && offset == 0, Kernel::ResultNotImplemented);
+ auto handle = m_nv_module->GetContainer().GetNvMapFile().GetHandle(id);
+ R_UNLESS(handle && !handle->allocated, Kernel::ResultInvalidState);
+ R_UNLESS(size && size == handle->size && address != 0 && address + size > address,
+ Kernel::ResultInvalidSize);
+ R_UNLESS(Common::IsAligned(address, Kernel::PageSize), Kernel::ResultInvalidAddress);
+ auto& page_table = m_process->GetPageTable();
+ R_UNLESS(page_table.Contains(address, size), Kernel::ResultInvalidCurrentMemory);
+ if (flags == 2) {
+ for (u64 current = address; current < address + size;) {
+ Kernel::KMemoryInfo info{};
+ Kernel::Svc::PageInfo page{};
+ R_TRY(page_table.QueryInfo(&info, &page, current));
+ R_UNLESS(True(info.m_attribute & Kernel::KMemoryAttribute::GpuAccessible),
+ Kernel::ResultInvalidCurrentMemory);
+ const u64 next = info.m_address + info.m_size;
+ R_UNLESS(next > current, Kernel::ResultInvalidCurrentMemory);
+ current = next;
+ }
+ }
+ bool is_io{};
+ R_TRY(page_table.LockForMapDeviceAddressSpace(&is_io, address, size,
+ Kernel::KMemoryPermission::None, true, true));
+ if (is_io) {
+ page_table.UnlockForDeviceAddressSpace(address, size);
+ R_RETURN(Kernel::ResultNotImplemented);
+ }
+ if (!m_native_memory_session) {
+ m_native_memory_session = m_nv_module->GetContainer().OpenSession(m_process).id;
+ }
+ const auto result = handle->Alloc({}, Kernel::PageSize, 0, address,
+ Nvidia::NvCore::SessionId{*m_native_memory_session});
+ if (result != Nvidia::NvResult::Success) {
+ page_table.UnlockForDeviceAddressSpace(address, size);
+ R_RETURN(Kernel::ResultOutOfResource);
+ }
+ R_SUCCEED();
+}
+
+void IGraphicsSupervisor::BindNativeMemory(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const auto pair = [&](size_t i) { return u64{packet[i]} | (u64{packet[i + 1]} << 32); };
+ const auto result = packet[0] == 0x82 && packet[1] == 9
+ ? BindNativeMemoryBacking(packet[2], pair(3), pair(5), pair(7), pair(9))
+ : Kernel::ResultInvalidCombination;
+ LOG_DEBUG(Service_AM, "native memory bind id={}, address={:#x}, size={:#x}, flags={:#x}, result={:#x}",
+ packet[2], pair(3), pair(5), pair(7), result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+Result IGraphicsSupervisor::ReleaseNativeMemory(u32 id) {
+ const auto it = std::ranges::find(m_native_memories, id);
+ R_UNLESS(m_nv_module && it != m_native_memories.end(), Kernel::ResultNotFound);
+ auto& nvmap = m_nv_module->GetContainer().GetNvMapFile();
+ auto handle = nvmap.GetHandle(id);
+ R_UNLESS(handle, Kernel::ResultNotFound);
+ {
+ const std::scoped_lock lock{handle->mutex};
+ R_UNLESS(handle->pins == 0 && handle->internal_dupes == 0 && handle->dupes == 1,
+ Kernel::ResultBusy);
+ }
+ const bool allocated = handle->allocated;
+ handle.reset();
+ const auto info = nvmap.FreeHandle(id, false);
+ m_native_memories.erase(it);
+ if (allocated) {
+ R_UNLESS(info && info->can_unlock, Kernel::ResultInvalidState);
+ R_TRY(m_process->GetPageTable().UnlockForDeviceAddressSpace(info->address, info->size));
+ }
+ R_SUCCEED();
+}
+
+void IGraphicsSupervisor::DestroyNativeMemory(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const auto result = packet[0] == 0x80 && packet[1] == 1
+ ? ReleaseNativeMemory(packet[2]) : Kernel::ResultInvalidCombination;
+ LOG_DEBUG(Service_AM, "native memory destroy id={}, result={:#x}", packet[2], result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::BindNativeGpuMemory(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const auto pair = [&](size_t i) { return u64{packet[i]} | (u64{packet[i + 1]} << 32); };
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x1b7 && packet[1] == 12 && pair(10) == 0 && pair(12) == 0) {
+ result = Kernel::ResultNotFound;
+ if (m_nv_module) {
+ auto& container = m_nv_module->GetContainer();
+ const auto parent = container.GetNvMapFile().GetHandle(packet[7]);
+ if (parent && parent->allocated) {
+ const auto* session = container.GetSession(parent->session_id);
+ const u64 offset = pair(8);
+ const u64 address = pair(3);
+ const u64 size = pair(5);
+ if (session && session->process == m_process && offset <= parent->size &&
+ size <= parent->size - offset && parent->address + offset >= parent->address &&
+ address == parent->address + offset) {
+ result = BindNativeMemoryBacking(packet[2], address, size, 2, 0);
+ }
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "native GPU memory bind id={}, parent={}, address={:#x}, size={:#x}, offset={:#x}, result={:#x}",
+ packet[2], packet[7], pair(3), pair(5), pair(8), result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::CreateNativeImage(HLERequestContext& ctx) {
+ const std::scoped_lock images_lock{m_image_registry->mutex};
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ u32 id{};
+ if (packet[0] == 0x1bd && packet[1] == 18 && packet[3] == 1) {
+ result = Kernel::ResultInvalidState;
+ if (m_phase == Phase::Ready && m_gpu_context_ready && m_image_formats_initialized && m_nv_module) {
+ const auto requirements = CalculateNativeImageRequirements(std::span{packet + 2, 16}, packet[18]);
+ result = Kernel::ResultNotImplemented;
+ if (requirements) {
+ result = Kernel::ResultNotFound;
+ auto& nvmap = m_nv_module->GetContainer().GetNvMapFile();
+ const auto memory = nvmap.GetHandle(packet[19]);
+ if (std::ranges::find(m_native_memories, packet[19]) != m_native_memories.end() &&
+ memory && memory->allocated) {
+ result = Kernel::ResultInvalidSize;
+ if (memory->size >= requirements->size) {
+ result = Kernel::ResultOutOfResource;
+ if (m_image_registry->images.size() < 1024 && m_next_native_image_id != 0 &&
+ memory->Duplicate(true) == Nvidia::NvResult::Success) {
+ NativeImage image{m_next_native_image_id++, packet[19], {}, requirements->size};
+ std::copy_n(packet + 2, image.preference.size(), image.preference.begin());
+ id = image.id;
+ if (!m_image_registry->module) {
+ m_image_registry->module = m_nv_module;
+ m_image_registry->memory_session = m_nv_module->GetContainer().OpenSession(m_process);
+ }
+ m_image_registry->images.push_back(image);
+ result = ResultSuccess;
+ }
+ }
+ }
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "native image create memory={}, count={}, id={}, result={:#x}",
+ packet[19], packet[18], id, result.raw);
+ IPC::ResponseBuilder rb{ctx, 3};
+ rb.Push(result.raw);
+ rb.Push(id);
+}
+
+void IGraphicsSupervisor::DestroyNativeImage(HLERequestContext& ctx) {
+ const std::scoped_lock images_lock{m_image_registry->mutex};
+ const auto* packet = ctx.CommandBuffer();
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == 0x1be && packet[1] == 1) {
+ const auto image = std::ranges::find(m_image_registry->images, packet[2], &NativeImage::id);
+ result = Kernel::ResultNotFound;
+ if (m_nv_module && image != m_image_registry->images.end()) {
+ result = Kernel::ResultBusy;
+ if (!image->window_pool) {
+ m_image_registry->ReleaseMemory(image->memory_id);
+ m_image_registry->images.erase(image);
+ result = ResultSuccess;
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "native image destroy id={}, result={:#x}", packet[2], result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+void IGraphicsSupervisor::ConfigureNativeWindowImagePool(HLERequestContext& ctx) {
+ const std::scoped_lock images_lock{m_image_registry->mutex};
+ const auto* packet = ctx.CommandBuffer();
+ const bool create = ctx.GetCommand() == 315;
+ Result result = Kernel::ResultInvalidCombination;
+ if (packet[0] == (create ? 0x14bU : 0x14aU) && packet[1] == (create ? 2U : 1U)) {
+ const auto image = std::ranges::find(m_image_registry->images, packet[2], &NativeImage::id);
+ result = Kernel::ResultNotFound;
+ if (image != m_image_registry->images.end()) {
+ result = Kernel::ResultNotImplemented;
+ if (!create || packet[3] == 1) {
+ result = Kernel::ResultInvalidState;
+ if (image->window_pool != create) {
+ image->window_pool = create;
+ result = ResultSuccess;
+ }
+ }
+ }
+ }
+ LOG_DEBUG(Service_AM, "native window image pool id={}, create={}, result={:#x}",
+ packet[2], create, result.raw);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(result.raw);
+}
+
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/graphics_supervisor.h b/src/core/hle/service/am/service/graphics_supervisor.h
new file mode 100644
index 0000000000..4163c2c150
--- /dev/null
+++ b/src/core/hle/service/am/service/graphics_supervisor.h
@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <vector>
+
+#include "core/hle/service/service.h"
+#include "core/hle/service/am/service/graphics_syncpoint_cache.h"
+#include "core/hle/service/kernel_helpers.h"
+#include "core/hle/service/os/event.h"
+
+namespace Kernel {
+class KProcess;
+class KTransferMemory;
+class KEvent;
+}
+
+namespace Core::Timing {
+struct EventType;
+}
+namespace Service::Nvidia {
+class Module;
+}
+
+namespace Service::AM {
+struct NativeImageRegistry;
+
+class IGpuSession;
+
+class IGraphicsSupervisor final : public ServiceFramework<IGraphicsSupervisor> {
+public:
+ explicit IGraphicsSupervisor(Core::System& system, Kernel::KProcess* process, u64 aruid);
+ ~IGraphicsSupervisor() override;
+ Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override;
+
+private:
+ enum class Phase { Created, Initialized, ProcessBound, Ready };
+ void Initialize(HLERequestContext& ctx);
+ void BindProcess(HLERequestContext& ctx);
+ void CompleteBootstrap(HLERequestContext& ctx);
+ void InitializeImageContext(HLERequestContext& ctx);
+ void GetNvidiaDriver(HLERequestContext& ctx);
+ void CalculateImageRequirements(HLERequestContext& ctx);
+ void CreateNativeMemory(HLERequestContext& ctx);
+ void CreateNativeImage(HLERequestContext& ctx);
+ void DestroyNativeImage(HLERequestContext& ctx);
+ void ConfigureNativeWindowImagePool(HLERequestContext& ctx);
+ void BindNativeMemory(HLERequestContext& ctx);
+ void BindNativeGpuMemory(HLERequestContext& ctx);
+ void DestroyNativeMemory(HLERequestContext& ctx);
+ Result BindNativeMemoryBacking(u32 id, u64 address, u64 size, u64 flags, u64 offset);
+ Result ReleaseNativeMemory(u32 id);
+ void ConfigureNativeMemory(HLERequestContext& ctx);
+ void InitializeFrameContext(HLERequestContext& ctx);
+ void RegisterGpuAddressSpace(HLERequestContext& ctx);
+ void OpenGpuSession(HLERequestContext& ctx);
+ void OpenDisplaySession(HLERequestContext& ctx);
+ void InitializeDisplayContext(HLERequestContext& ctx);
+ void InitializeGpuContext(HLERequestContext& ctx);
+ void InitializeSyncpointContext(HLERequestContext& ctx);
+ void GetSyncpointSession(HLERequestContext& ctx);
+ void BindSyncpointMemory(HLERequestContext& ctx);
+ void RefreshSyncpointCache();
+
+ std::vector<std::weak_ptr<IGpuSession>> m_gpu_sessions;
+ std::shared_ptr<Nvidia::Module> m_nv_module;
+ std::optional<size_t> m_native_memory_session;
+ std::vector<u32> m_native_memories;
+ std::shared_ptr<NativeImageRegistry> m_image_registry;
+ u32 m_next_native_image_id{1};
+ std::shared_ptr<Core::Timing::EventType> m_cache_poll;
+ std::array<GraphicsSyncpointCache::CounterValue, GraphicsSyncpointCache::CounterCount>
+ m_counter_values{};
+ Kernel::KTransferMemory* m_syncpoint_rw{};
+ Kernel::KTransferMemory* m_syncpoint_ro{};
+ Kernel::KEvent* m_syncpoint_request_event{};
+ KernelHelpers::ServiceContext m_context;
+ Event m_syncpoint_cache_event;
+ bool m_syncpoint_context_initialized{};
+ Kernel::KProcess* const m_process;
+ const u64 m_aruid;
+ Phase m_phase{Phase::Created};
+ bool m_display_context_initialized{};
+ bool m_gpu_context_initialized{};
+ bool m_gpu_context_ready{};
+ bool m_frame_context_initialized{};
+ bool m_frame_context_ready{};
+ bool m_image_context_initialized{};
+ bool m_image_formats_initialized{};
+};
+
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/graphics_supervisor_protocol.h b/src/core/hle/service/am/service/graphics_supervisor_protocol.h
new file mode 100644
index 0000000000..7762ced531
--- /dev/null
+++ b/src/core/hle/service/am/service/graphics_supervisor_protocol.h
@@ -0,0 +1,37 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <array>
+#include <optional>
+#include <span>
+#include "common/common_types.h"
+
+namespace Service::AM::GraphicsSupervisorProtocol {
+
+constexpr u8 DecodeBufferSlot(u32 packed) {
+ return static_cast<u8>(packed);
+}
+
+struct BootstrapRequest {
+ u32 process_handle;
+ u64 applet_resource_user_id;
+ u64 flags;
+ std::array<u64, 4> parameters;
+};
+
+inline std::optional<BootstrapRequest> DecodeBootstrap(std::span<const u32> packet) {
+ // Captured remote-memory bootstrap: TIPC 4, one copied handle, 48 data bytes.
+ if (packet.size() < 16 || packet[0] != 0x14 || packet[1] != 0x8000000c ||
+ packet[2] != 2) {
+ return std::nullopt;
+ }
+ const auto Read64 = [&](size_t index) {
+ return u64{packet[index]} | (u64{packet[index + 1]} << 32);
+ };
+ return BootstrapRequest{packet[3], Read64(4), Read64(6),
+ {Read64(8), Read64(10), Read64(12), Read64(14)}};
+}
+
+} // namespace Service::AM::GraphicsSupervisorProtocol
diff --git a/src/core/hle/service/am/service/graphics_syncpoint_cache.h b/src/core/hle/service/am/service/graphics_syncpoint_cache.h
new file mode 100644
index 0000000000..23361b006c
--- /dev/null
+++ b/src/core/hle/service/am/service/graphics_syncpoint_cache.h
@@ -0,0 +1,49 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <array>
+#include <limits>
+#include <optional>
+#include "common/common_types.h"
+
+namespace Service::AM::GraphicsSyncpointCache {
+constexpr u32 ValuesOffset = 0x40;
+constexpr u32 GroupAckOffset = 0x640;
+constexpr u32 SlotAckOffset = 0x648;
+constexpr u32 GroupRequestOffset = 0;
+constexpr u32 SlotRequestOffset = 8;
+constexpr u32 CounterCount = 192;
+constexpr u32 GroupCount = (CounterCount + 63) / 64;
+constexpr u64 Unallocated = (std::numeric_limits<u64>::max)();
+
+constexpr std::array<u32, 8> Header{
+ 1, 0, CounterCount - 1, ValuesOffset, GroupRequestOffset,
+ GroupAckOffset, SlotRequestOffset, SlotAckOffset};
+static_assert(ValuesOffset + CounterCount * sizeof(u64) <= GroupAckOffset);
+static_assert(SlotAckOffset + GroupCount * sizeof(u64) <= 0x2000);
+static_assert(SlotRequestOffset + GroupCount * sizeof(u64) <= 0x1000);
+
+constexpr u64 ExtendCounter(u64 previous, u32 current) {
+ if (previous == Unallocated) {
+ return current;
+ }
+ u64 high = previous & 0xffffffff00000000ULL;
+ if (current < static_cast<u32>(previous)) {
+ high += 1ULL << 32;
+ }
+ return high | current;
+}
+struct CounterValue {
+ u64 previous{Unallocated};
+
+ u64 Observe(std::optional<u32> current) {
+ if (!current) {
+ return Unallocated;
+ }
+ previous = ExtendCounter(previous, *current);
+ return previous;
+ }
+};
+} // namespace Service::AM::GraphicsSyncpointCache
diff --git a/src/core/hle/service/am/service/native_image_layout.h b/src/core/hle/service/am/service/native_image_layout.h
new file mode 100644
index 0000000000..eb45fc470d
--- /dev/null
+++ b/src/core/hle/service/am/service/native_image_layout.h
@@ -0,0 +1,40 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <optional>
+#include <span>
+#include "common/common_types.h"
+
+namespace Service::AM {
+
+struct NativeImageRequirements {
+ u64 size;
+ u64 alignment;
+};
+
+// Experimental inter-module block image layout, restricted to the observed
+// four-byte color format. Row pitch is in 64-byte GOBs; block height is in pixels.
+inline std::optional<NativeImageRequirements> CalculateNativeImageRequirements(
+ std::span<const u32> preference, u32 count) {
+ if (preference.size() != 16 || count != 1 || preference[0] != 0 ||
+ (preference[1] != 1 && preference[1] != 2) || preference[3] != 0 || preference[6] != 1 ||
+ preference[7] != 1 || preference[4] < 3 || preference[4] > 8) {
+ return std::nullopt;
+ }
+ for (size_t i = 8; i < preference.size(); ++i) {
+ if (preference[i] != 0) { return std::nullopt; }
+ }
+ const u64 width = (preference[2] & 0xffffU) + 1;
+ const u64 height = (preference[2] >> 16) + 1;
+ const u64 pitch_gobs = preference[5];
+ if (pitch_gobs < (width * 4 + 63) / 64 || pitch_gobs > 4096) {
+ return std::nullopt;
+ }
+ const u64 block_height = 1ULL << preference[4];
+ const u64 aligned_height = (height + block_height - 1) & ~(block_height - 1);
+ return NativeImageRequirements{pitch_gobs * 64 * aligned_height, 64 * block_height};
+}
+
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/performance_functions.cpp b/src/core/hle/service/am/service/performance_functions.cpp
new file mode 100644
index 0000000000..71a46cc5b5
--- /dev/null
+++ b/src/core/hle/service/am/service/performance_functions.cpp
@@ -0,0 +1,19 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "core/hle/service/am/service/performance_functions.h"
+
+namespace Service::AM {
+
+IPerformanceFunctions::IPerformanceFunctions(Core::System& system)
+ : ServiceFramework{system, "IPerformanceFunctions"} {
+ static const FunctionInfo functions[] = {
+ {0, nullptr, "IsCpuOverclockEnabled"},
+ {1, nullptr, "SetCpuOverclockEnabled"},
+ };
+ RegisterHandlers(functions);
+}
+
+IPerformanceFunctions::~IPerformanceFunctions() = default;
+
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/performance_functions.h b/src/core/hle/service/am/service/performance_functions.h
new file mode 100644
index 0000000000..12195d1217
--- /dev/null
+++ b/src/core/hle/service/am/service/performance_functions.h
@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "core/hle/service/service.h"
+
+namespace Service::AM {
+
+class IPerformanceFunctions final : public ServiceFramework<IPerformanceFunctions> {
+public:
+ explicit IPerformanceFunctions(Core::System& system);
+ ~IPerformanceFunctions() override;
+};
+
+} // namespace Service::AM
diff --git a/src/core/hle/service/am/service/self_controller.cpp b/src/core/hle/service/am/service/self_controller.cpp
index 3a654b9daa..45b9120093 100644
--- a/src/core/hle/service/am/service/self_controller.cpp
+++ b/src/core/hle/service/am/service/self_controller.cpp
@@ -5,10 +5,17 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include "common/logging.h"
+#include "core/hle/kernel/k_scoped_resource_reservation.h"
+#include "core/hle/kernel/k_session.h"
+#include "core/hle/kernel/k_client_session.h"
+#include "core/hle/kernel/k_thread.h"
#include "core/hle/result.h"
#include "core/hle/service/am/am_results.h"
#include "core/hle/service/am/frontend/applets.h"
#include "core/hle/service/am/service/self_controller.h"
+#include "core/hle/service/am/service/graphics_supervisor.h"
+#include "core/hle/service/ipc_helpers.h"
+#include "core/hle/service/server_manager.h"
#include "core/hle/service/caps/caps_su.h"
#include "core/hle/service/cmif_serialization.h"
#include "core/hle/service/nvnflinger/nvnflinger.h"
@@ -73,6 +80,7 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet>
{230, D<&ISelfController::Unknown230>, "Unknown230"},
{240, D<&ISelfController::Unknown240>, "Unknown240"},
{1000, nullptr, "GetDebugStorageChannel"},
+ {2000, &ISelfController::GetGraphicsSupervisor, "GetGraphicsSupervisor"},
};
// clang-format on
@@ -84,10 +92,47 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet>
}
ISelfController::~ISelfController() {
+ if (m_graphics_supervisor_session != nullptr) {
+ m_graphics_supervisor_session->Close(kernel);
+ }
std::scoped_lock lk{m_applet->lock};
m_applet->display_layer_manager.Finalize();
}
+void ISelfController::GetGraphicsSupervisor(HLERequestContext& ctx) {
+ if (m_graphics_supervisor_session != nullptr) {
+ IPC::ResponseBuilder rb{ctx, 2, 1};
+ rb.Push(ResultSuccess);
+ rb.PushCopyObjects(ctx, m_graphics_supervisor_session);
+ return;
+ }
+ Kernel::KScopedResourceReservation reservation(
+ kernel, Kernel::GetCurrentProcessPointer(kernel), Kernel::LimitableResource::SessionCountMax);
+ if (!reservation.Succeeded()) {
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(Kernel::ResultLimitReached);
+ return;
+ }
+ auto* session = Kernel::KSession::Create(kernel);
+ if (session == nullptr) {
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(Kernel::ResultOutOfResource);
+ return;
+ }
+ session->Initialize(kernel, nullptr, 0);
+ reservation.Commit();
+ Kernel::KSession::Register(kernel, session);
+ auto& server = ctx.GetManager()->GetServerManager();
+ auto manager = std::make_shared<SessionRequestManager>(kernel, server);
+ manager->SetSessionHandler(std::make_shared<IGraphicsSupervisor>(system, m_process, m_applet->aruid.pid));
+ server.RegisterSession(&session->GetServerSession(), std::move(manager));
+ LOG_DEBUG(Service_AM, "created graphics-supervisor session");
+ m_graphics_supervisor_session = &session->GetClientSession();
+ IPC::ResponseBuilder rb{ctx, 2, 1};
+ rb.Push(ResultSuccess);
+ rb.PushCopyObjects(ctx, m_graphics_supervisor_session);
+}
+
Result ISelfController::Exit() {
LOG_DEBUG(Service_AM, "called");
diff --git a/src/core/hle/service/am/service/self_controller.h b/src/core/hle/service/am/service/self_controller.h
index c9f480f575..52970ed4a8 100644
--- a/src/core/hle/service/am/service/self_controller.h
+++ b/src/core/hle/service/am/service/self_controller.h
@@ -12,6 +12,7 @@
namespace Kernel {
class KReadableEvent;
+class KClientSession;
}
namespace Service::Capture {
@@ -30,6 +31,7 @@ public:
~ISelfController() override;
private:
+ void GetGraphicsSupervisor(HLERequestContext& ctx);
Result Exit();
Result LockExit();
Result UnlockExit();
@@ -70,6 +72,7 @@ private:
Result Unknown230(u32 in_val, Out<u16> out_val);
Result Unknown240(u32 in_val);
+ Kernel::KClientSession* m_graphics_supervisor_session{};
Kernel::KProcess* const m_process;
const std::shared_ptr<Applet> m_applet;
};
diff --git a/src/core/hle/service/audio/audio.cpp b/src/core/hle/service/audio/audio.cpp
index fde766a275..63d578462f 100644
--- a/src/core/hle/service/audio/audio.cpp
+++ b/src/core/hle/service/audio/audio.cpp
@@ -5,6 +5,8 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include "core/core.h"
+#include "core/hle/kernel/svc_results.h"
+#include "core/hle/service/ipc_helpers.h"
#include "core/hle/service/audio/audio.h"
#include "core/hle/service/audio/audio_controller.h"
#include "core/hle/service/audio/audio_in_manager.h"
@@ -18,6 +20,57 @@
namespace Service::Audio {
+class INativeAudioManager final : public ServiceFramework<INativeAudioManager> {
+public:
+ INativeAudioManager(Core::System& system_, std::shared_ptr<IAudioInManager> input,
+ std::shared_ptr<IAudioOutManager> output,
+ std::shared_ptr<IAudioRendererManager> renderer)
+ : ServiceFramework{system_, "aud:u"}, m_input{std::move(input)},
+ m_output{std::move(output)}, m_renderer{std::move(renderer)} {
+ static const FunctionInfo functions[] = {
+ {0, &INativeAudioManager::GetAudioInManager, "GetAudioInManager"},
+ {1, &INativeAudioManager::GetAudioOutManager, "GetAudioOutManager"},
+ {2, &INativeAudioManager::GetAudioRendererManager, "GetAudioRendererManager"},
+ };
+ RegisterHandlers(functions);
+ }
+
+ Result HandleSyncRequest(Kernel::KServerSession& session, HLERequestContext& ctx) override {
+ if (!ctx.IsTipc()) {
+ return ServiceFrameworkBase::HandleSyncRequest(session, ctx);
+ }
+ const auto guard = LockService();
+ LOG_WARNING(Service_Audio, "Native audio unsupported command={}, words={:08x}", ctx.GetCommand(),
+ fmt::join(std::span<const u32>(ctx.CommandBuffer() + 2,
+ std::min<u32>(ctx.CommandBuffer()[1], 32)), ","));
+ InvokeRequestTipc(ctx);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(Kernel::ResultNotImplemented.raw);
+ return system.IsPoweredOn() ? ctx.WriteToOutgoingCommandBuffer() : ResultSuccess;
+ }
+
+private:
+ void GetAudioInManager(HLERequestContext& ctx) {
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(ResultSuccess);
+ rb.PushIpcInterface(ctx, m_input);
+ }
+ void GetAudioOutManager(HLERequestContext& ctx) {
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(ResultSuccess);
+ rb.PushIpcInterface(ctx, m_output);
+ }
+ void GetAudioRendererManager(HLERequestContext& ctx) {
+ LOG_DEBUG(Service_Audio, "Native user client opens existing audio renderer manager");
+ IPC::ResponseBuilder rb{ctx, 2, 0, 1};
+ rb.Push(ResultSuccess);
+ rb.PushIpcInterface(ctx, m_renderer);
+ }
+ std::shared_ptr<IAudioInManager> m_input;
+ std::shared_ptr<IAudioOutManager> m_output;
+ std::shared_ptr<IAudioRendererManager> m_renderer;
+};
+
class IAudioOutManagerForApplet final : public ServiceFramework<IAudioOutManagerForApplet> {
public:
explicit IAudioOutManagerForApplet(Core::System& system_)
@@ -193,18 +246,23 @@ void LoopProcess(Core::System& system) {
server_manager->RegisterNamedService("audrec:d", std::make_shared<IFinalOutputRecorderManagerForDebugger>(system), 30);
server_manager->RegisterNamedService("audren:d", std::make_shared<IAudioRendererManagerForDebugger>(system), 30);
- server_manager->RegisterNamedService("audin:u", std::make_shared<IAudioInManager>(system), 30);
+ auto audio_input = std::make_shared<IAudioInManager>(system);
+ server_manager->RegisterNamedService("audin:u", audio_input, 30);
server_manager->RegisterNamedService("audin:a", std::make_shared<IAudioInManagerForApplet>(system), 30);
- server_manager->RegisterNamedService("audout:u", std::make_shared<IAudioOutManager>(system), 30);
+ auto audio_output = std::make_shared<IAudioOutManager>(system);
+ server_manager->RegisterNamedService("audout:u", audio_output, 30);
server_manager->RegisterNamedService("audout:a", std::make_shared<IAudioOutManagerForApplet>(system), 30);
server_manager->RegisterNamedService("auddev", std::make_shared<IAudioSnoopManager>(system), 30);
// Depends on audout:u and audin:u on ctor!
server_manager->RegisterNamedService("audctl", std::make_shared<IAudioController>(system), 30);
server_manager->RegisterNamedService("audrec:a", std::make_shared<IFinalOutputRecorderManagerForApplet>(system), 30);
server_manager->RegisterNamedService("audrec:u", std::make_shared<IFinalOutputRecorderManager>(system), 30);
- server_manager->RegisterNamedService("audren:u", std::make_shared<IAudioRendererManager>(system), 30);
+ auto audio_renderer = std::make_shared<IAudioRendererManager>(system);
+ server_manager->RegisterNamedService("audren:u", audio_renderer, 30);
server_manager->RegisterNamedService("audren:a", std::make_shared<IAudioRendererManagerForApplet>(system), 30);
server_manager->RegisterNamedService("hwopus", std::make_shared<IHardwareOpusDecoderManager>(system), 25);
+ server_manager->RegisterNamedService("aud:u",
+ std::make_shared<INativeAudioManager>(system, audio_input, audio_output, audio_renderer), 30);
ServerManager::RunServer(std::move(server_manager));
}
diff --git a/src/core/hle/service/hid/hid_server.cpp b/src/core/hle/service/hid/hid_server.cpp
index 9689bce8dd..7ddb944abe 100644
--- a/src/core/hle/service/hid/hid_server.cpp
+++ b/src/core/hle/service/hid/hid_server.cpp
@@ -132,6 +132,7 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r
{210, C<&IHidServer::EndPermitVibrationSession>, "EndPermitVibrationSession"},
{211, C<&IHidServer::IsVibrationDeviceMounted>, "IsVibrationDeviceMounted"},
{212, C<&IHidServer::SendVibrationValueInBool>, "SendVibrationValueInBool"},
+ {214, C<&IHidServer::SendNativeVibrationValues>, "SendNativeVibrationValues"},
{300, C<&IHidServer::ActivateConsoleSixAxisSensor>, "ActivateConsoleSixAxisSensor"},
{301, C<&IHidServer::StartConsoleSixAxisSensor>, "StartConsoleSixAxisSensor"},
{302, C<&IHidServer::StopConsoleSixAxisSensor>, "StopConsoleSixAxisSensor"},
@@ -237,6 +238,11 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r
{3013, nullptr, "SetDebugPadGenericPadMap"}, //21.0.0+
{3014, nullptr, "GetDebugPadKeyboardMap"}, //21.0.0+
{3015, nullptr, "SetDebugPadKeyboardMap"}, //21.0.0+
+ {3110, C<&IHidServer::RegisterJcmsMemory>, "RegisterJcmsMemory"},
+ {3136, C<&IHidServer::SetNmouseButtonConfigJoyLeftJcms>, "SetNmouseButtonConfigJoyLeftJcms"},
+ {3137, C<&IHidServer::SetNmouseButtonConfigJoyRightJcms>, "SetNmouseButtonConfigJoyRightJcms"},
+ {3138, C<&IHidServer::SetMergedNmouseSourceTypes>, "SetMergedNmouseSourceTypes"},
+ {3139, C<&IHidServer::RegisterNativeInputMemory>, "RegisterNativeInputMemory"},
{3150, C<&IHidServer::SetMouseLibraryVersion>, "SetMouseLibraryVersion"}, //21.0.0+
// What? -- {12010, nullptr, "SetButtonConfigLeft"},
};
@@ -245,7 +251,11 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r
RegisterHandlers(functions);
}
-IHidServer::~IHidServer() = default;
+IHidServer::~IHidServer() {
+ for (const auto& registration : native_input_memories) {
+ registration.memory->Close(kernel);
+ }
+}
Result IHidServer::CreateAppletResource(OutInterface<IAppletResource> out_applet_resource,
ClientAppletResourceUserId aruid) {
@@ -984,6 +994,17 @@ Result IHidServer::SendVibrationValues(
R_SUCCEED();
}
+Result IHidServer::SendNativeVibrationValues(
+ u16 flags, AppletResourceUserId aruid,
+ InArray<Core::HID::VibrationDeviceHandle, BufferAttr_HipcPointer> vibration_handles,
+ InArray<Core::HID::VibrationValue, BufferAttr_HipcPointer> vibration_values) {
+ static u32 diagnostic_batches{};
+ if (diagnostic_batches++ < 8) LOG_INFO(Service_HID, "Native vibration batch flags={:#x}, aruid={}, handles={}, values={}",
+ flags, aruid.pid, vibration_handles.size(), vibration_values.size());
+ R_UNLESS(flags == 0 || flags == 2, IPC::ResultNotSupported);
+ R_RETURN(SendVibrationValues(ClientAppletResourceUserId{aruid.pid}, vibration_handles, vibration_values));
+}
+
Result IHidServer::SendVibrationGcErmCommand(
Core::HID::VibrationDeviceHandle vibration_device_handle, ClientAppletResourceUserId aruid,
Core::HID::VibrationGcErmCommand gc_erm_command) {
@@ -1471,6 +1492,63 @@ Result IHidServer::SetTouchScreenResolution(u32 width, u32 height,
R_SUCCEED();
}
+Result IHidServer::RegisterNativeInputMemory(
+ u8 mode, u32 parameter0, u32 parameter1, u32 parameter2,
+ ClientAppletResourceUserId aruid, u64 size,
+ InCopyHandle<Kernel::KTransferMemory> memory) {
+ LOG_INFO(Service_HID, "Native input memory mode={}, parameters={:#x}/{:#x}/{:#x}, aruid={}, size={:#x}",
+ mode, parameter0, parameter1, parameter2, aruid.pid, size);
+ R_UNLESS(memory.Get() && memory->GetOwner()->UsesPAuthReturns() &&
+ memory->GetOwner()->GetProcessId() == aruid.pid, Kernel::ResultInvalidHandle);
+ R_UNLESS(mode <= 1 && size && size == memory->GetSize() && size <= 0x1000000,
+ Kernel::ResultInvalidSize);
+ R_UNLESS(native_input_memories.size() < 64, Kernel::ResultOutOfResource);
+ const auto existing = std::ranges::find(native_input_memories, memory.Get(), &NativeInputMemory::memory);
+ R_UNLESS(existing == native_input_memories.end(), Kernel::ResultInvalidState);
+ memory->Open(kernel);
+ native_input_memories.push_back({memory.Get(), aruid.pid, size, {mode, parameter0, parameter1, parameter2}});
+ // Retain the real guest-owned input buffer. No Joy-Con mouse samples are
+ // produced while those devices are disconnected.
+ R_SUCCEED();
+}
+
+Result IHidServer::RegisterJcmsMemory(ClientAppletResourceUserId aruid, u64 size,
+ InCopyHandle<Kernel::KTransferMemory> memory) {
+ LOG_INFO(Service_HID, "Native Joy-Con mouse memory: aruid={}, size={:#x}", aruid.pid, size);
+ return RegisterNativeInputMemory(1, 0, 0, 0, aruid, size, memory);
+}
+
+Result IHidServer::SetMergedNmouseSourceTypes(u32 source_types, ClientAppletResourceUserId aruid) {
+ LOG_INFO(Service_HID, "Native merged mouse sources: types={:#x}, aruid={}", source_types, aruid.pid);
+ R_UNLESS((source_types & ~6U) == 0, Kernel::ResultInvalidCombination);
+ const auto memory = std::ranges::find_if(native_input_memories, [&](const auto& registration) {
+ return registration.aruid == aruid.pid && registration.parameters[0] == 0;
+ });
+ R_UNLESS(memory != native_input_memories.end(), Kernel::ResultNotFound);
+ memory->merged_source_types = source_types;
+ R_SUCCEED();
+}
+
+Result IHidServer::SetNmouseButtonConfiguration(ClientAppletResourceUserId aruid, u64 configuration,
+ size_t side) {
+ LOG_INFO(Service_HID, "Native Joy-Con mouse buttons: side={}, config={:#x}, aruid={}",
+ side, configuration, aruid.pid);
+ const auto memory = std::ranges::find_if(native_input_memories, [&](const auto& registration) {
+ return registration.aruid == aruid.pid && registration.parameters[0] == 0;
+ });
+ R_UNLESS(memory != native_input_memories.end(), Kernel::ResultNotFound);
+ memory->jcms_button_configurations[side] = configuration;
+ R_SUCCEED();
+}
+
+Result IHidServer::SetNmouseButtonConfigJoyLeftJcms(ClientAppletResourceUserId aruid, u64 configuration) {
+ return SetNmouseButtonConfiguration(aruid, configuration, 0);
+}
+
+Result IHidServer::SetNmouseButtonConfigJoyRightJcms(ClientAppletResourceUserId aruid, u64 configuration) {
+ return SetNmouseButtonConfiguration(aruid, configuration, 1);
+}
+
Result IHidServer::SetMouseLibraryVersion(ClientAppletResourceUserId aruid) {
LOG_INFO(Service_HID, "(STUBBED) called, applet_resource_user_id={}", aruid.pid);
diff --git a/src/core/hle/service/hid/hid_server.h b/src/core/hle/service/hid/hid_server.h
index 57a1a04b9c..45097f361d 100644
--- a/src/core/hle/service/hid/hid_server.h
+++ b/src/core/hle/service/hid/hid_server.h
@@ -6,6 +6,8 @@
#pragma once
+#include <array>
+#include <vector>
#include "core/hle/service/cmif_types.h"
#include "core/hle/service/service.h"
#include "hid_core/hid_types.h"
@@ -175,6 +177,10 @@ private:
ClientAppletResourceUserId aruid,
InArray<Core::HID::VibrationDeviceHandle, BufferAttr_HipcPointer> vibration_handles,
InArray<Core::HID::VibrationValue, BufferAttr_HipcPointer> vibration_values);
+ Result SendNativeVibrationValues(
+ u16 flags, AppletResourceUserId aruid,
+ InArray<Core::HID::VibrationDeviceHandle, BufferAttr_HipcPointer> vibration_handles,
+ InArray<Core::HID::VibrationValue, BufferAttr_HipcPointer> vibration_values);
Result SendVibrationGcErmCommand(Core::HID::VibrationDeviceHandle vibration_device_handle,
ClientAppletResourceUserId aruid,
Core::HID::VibrationGcErmCommand gc_erm_command);
@@ -265,6 +271,24 @@ private:
Result SetTouchScreenResolution(u32 width, u32 height, ClientAppletResourceUserId aruid);
Result SetMouseLibraryVersion(ClientAppletResourceUserId aruid);
+ Result RegisterNativeInputMemory(u8 mode, u32 parameter0, u32 parameter1, u32 parameter2,
+ ClientAppletResourceUserId aruid, u64 size,
+ InCopyHandle<Kernel::KTransferMemory> memory);
+ Result RegisterJcmsMemory(ClientAppletResourceUserId aruid, u64 size,
+ InCopyHandle<Kernel::KTransferMemory> memory);
+ Result SetMergedNmouseSourceTypes(u32 source_types, ClientAppletResourceUserId aruid);
+ Result SetNmouseButtonConfigJoyLeftJcms(ClientAppletResourceUserId aruid, u64 configuration);
+ Result SetNmouseButtonConfigJoyRightJcms(ClientAppletResourceUserId aruid, u64 configuration);
+ Result SetNmouseButtonConfiguration(ClientAppletResourceUserId aruid, u64 configuration, size_t side);
+ struct NativeInputMemory {
+ Kernel::KTransferMemory* memory;
+ u64 aruid;
+ u64 size;
+ std::array<u32, 4> parameters;
+ u32 merged_source_types{};
+ std::array<u64, 2> jcms_button_configurations{};
+ };
+ std::vector<NativeInputMemory> native_input_memories;
std::shared_ptr<ResourceManager> resource_manager;
std::shared_ptr<HidFirmwareSettings> firmware_settings;
};
diff --git a/src/core/hle/service/ngc/ngc.cpp b/src/core/hle/service/ngc/ngc.cpp
index e79108a1eb..2c79336d93 100644
--- a/src/core/hle/service/ngc/ngc.cpp
+++ b/src/core/hle/service/ngc/ngc.cpp
@@ -7,6 +7,7 @@
#include "common/string_util.h"
#include "core/core.h"
#include "core/hle/kernel/k_client_session.h"
+#include "core/hle/kernel/k_shared_memory.h"
#include "core/hle/result.h"
#include "core/hle/service/acc/profile_manager.h"
#include "core/hle/service/cmif_types.h"
@@ -182,6 +183,8 @@ public:
explicit IUserShimScopedObject(Core::System& system_) : ServiceFramework(system_, "IUserShimScopedObject") {
// clang-format off
static const FunctionInfo functions[] = {
+ {1, D<&IUserShimScopedObject::GetRequiredMemorySize>, "GetRequiredMemorySize"},
+ {300, D<&IUserShimScopedObject::GetGuestHidSharedMemory>, "GetGuestHidSharedMemory"},
{450, nullptr, "InitializeForSaveData"},
{451, nullptr, "FinalizeForSaveData"},
{452, D<&IUserShimScopedObject::OpenSaveData>, "OpenSaveData"},
@@ -195,6 +198,50 @@ public:
RegisterHandlers(functions);
}
+ ~IUserShimScopedObject() override {
+ if (m_guest_hid_memory) {
+ m_guest_hid_memory->Close(kernel);
+ }
+ }
+
+ struct GuestHidMemoryConfig {
+ u64 size;
+ u64 offset;
+ };
+
+ Result GetRequiredMemorySize(Out<u32> out_size, u8 type, u8 mode, s32 guest_count) {
+ LOG_INFO(Service_NGC, "GameShare work memory query type={}, mode={}, guests={}", type, mode, guest_count);
+ R_UNLESS(type <= 1 && mode == 1 && guest_count >= 0 && guest_count <= 3,
+ IPC::ResultNotSupported);
+ // The disconnected implementation reserves one page for its session
+ // control state. Streaming/encoding work is not implemented.
+ *out_size = 0x1000;
+ R_SUCCEED();
+ }
+
+ Result GetGuestHidSharedMemory(Out<GuestHidMemoryConfig> out_config,
+ OutCopyHandle<Kernel::KSharedMemory> out_memory) {
+ // Three guest pad blocks, touch rings, and JCMS rings. Empty rings and
+ // zero style sets describe no connected GameShare guests.
+ constexpr u64 GuestHidMemorySize = 0x84000;
+ if (!m_guest_hid_memory) {
+ auto* memory = Kernel::KSharedMemory::Create(kernel);
+ const auto result = memory->Initialize(kernel, system.DeviceMemory(), nullptr,
+ Kernel::Svc::MemoryPermission::ReadWrite, Kernel::Svc::MemoryPermission::Read,
+ GuestHidMemorySize);
+ if (R_FAILED(result)) {
+ memory->Close(kernel);
+ R_RETURN(result);
+ }
+ Kernel::KSharedMemory::Register(kernel, memory);
+ m_guest_hid_memory = memory;
+ }
+ *out_config = GuestHidMemoryConfig{GuestHidMemorySize, 0};
+ *out_memory = m_guest_hid_memory;
+ LOG_DEBUG(Service_NGC, "GameShare guest HID memory size={:#x}, no guests connected", GuestHidMemorySize);
+ R_SUCCEED();
+ }
+
Result OpenSaveData(Account::Uid unk0, Out<SaveDataHandle> unk1) {
LOG_WARNING(Service_NGC, "stubbed");
R_THROW(IPC::ResultNotSupported);
@@ -210,6 +257,9 @@ public:
// to implement
R_SUCCEED();
}
+
+private:
+ Kernel::KSharedMemory* m_guest_hid_memory{};
};
class IUserService final : public ServiceFramework<IUserService> {
diff --git a/src/core/hle/service/nvdrv/core/syncpoint_manager.cpp b/src/core/hle/service/nvdrv/core/syncpoint_manager.cpp
index c4c4c25932..8b929a7ef3 100644
--- a/src/core/hle/service/nvdrv/core/syncpoint_manager.cpp
+++ b/src/core/hle/service/nvdrv/core/syncpoint_manager.cpp
@@ -63,6 +63,14 @@ void SyncpointManager::FreeSyncpoint(u32 id) {
syncpoint.reserved = false;
}
+std::optional<u32> SyncpointManager::ReadAllocatedSyncpoint(u32 id) {
+ std::lock_guard lock(reservation_lock);
+ if (id >= SyncpointCount || !syncpoints[id].reserved) {
+ return std::nullopt;
+ }
+ return UpdateMin(id);
+}
+
bool SyncpointManager::IsSyncpointAllocated(u32 id) const {
return (id < SyncpointCount) && syncpoints[id].reserved;
}
diff --git a/src/core/hle/service/nvdrv/core/syncpoint_manager.h b/src/core/hle/service/nvdrv/core/syncpoint_manager.h
index 7728ff5967..42e1d547b8 100644
--- a/src/core/hle/service/nvdrv/core/syncpoint_manager.h
+++ b/src/core/hle/service/nvdrv/core/syncpoint_manager.h
@@ -7,6 +7,7 @@
#include <array>
#include <atomic>
#include <mutex>
+#include <optional>
#include "common/common_types.h"
#include "core/hle/service/nvdrv/nvdata.h"
@@ -38,6 +39,8 @@ enum class ChannelType : u32 {
*/
class SyncpointManager final {
public:
+ std::optional<u32> ReadAllocatedSyncpoint(u32 id);
+
explicit SyncpointManager(Tegra::Host1x::Host1x& host1x);
~SyncpointManager();
diff --git a/src/core/hle/service/nvdrv/devices/gpu_address_space_layout.h b/src/core/hle/service/nvdrv/devices/gpu_address_space_layout.h
new file mode 100644
index 0000000000..a54ebb18d7
--- /dev/null
+++ b/src/core/hle/service/nvdrv/devices/gpu_address_space_layout.h
@@ -0,0 +1,34 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "common/common_types.h"
+
+namespace Service::Nvidia::Devices {
+
+struct GpuAddressSpaceLayout {
+ u64 start;
+ u64 split;
+ u64 end;
+};
+
+constexpr GpuAddressSpaceLayout GetGpuAddressSpaceLayout(bool extended) {
+ // Leave room for other 4 KiB allocations beyond the native bootstrap
+ // reservation of 0x7ff8000000 bytes.
+ // Both layouts fit within the GPU's reported 40-bit virtual address width.
+ return {0x8000000, extended ? 3ULL << 38 : 1ULL << 34,
+ extended ? 1ULL << 40 : 1ULL << 37};
+}
+
+constexpr u32 GetGpuAddressSpaceBigPageSize(bool extended, u32 requested) {
+ // The native SDK passes its preferred allocation size here. A preference for
+ // ordinary 4 KiB mappings still needs a separate large-page pool in NVDRV.
+ return extended && requested == 0x1000 ? 0x10000 : requested;
+}
+
+constexpr u64 GpuPageAddress(u32 page, u32 page_bits) {
+ return static_cast<u64>(page) << page_bits;
+}
+
+} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/native_gpu_characteristics.h b/src/core/hle/service/nvdrv/devices/native_gpu_characteristics.h
new file mode 100644
index 0000000000..44f45cab75
--- /dev/null
+++ b/src/core/hle/service/nvdrv/devices/native_gpu_characteristics.h
@@ -0,0 +1,67 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <array>
+#include <cstring>
+#include <span>
+#include "common/common_types.h"
+#include "common/swap.h"
+
+namespace Service::Nvidia::Devices {
+
+inline constexpr size_t NativeGpuCharacteristicsSize = 0xd0;
+
+inline std::array<u8, NativeGpuCharacteristicsSize> MakeNativeGpuCharacteristics(
+ std::span<const u8, 0xa0> legacy) {
+ std::array<u8, NativeGpuCharacteristicsSize> result{};
+ // The GPU description and engine classes share the legacy prefix. The native
+ // SDK consumes signed ioctl interface levels where the legacy ABI has topology.
+ std::memcpy(result.data(), legacy.data(), 0x70);
+ // Select the SDK's native control ABI, whose normal mappings support
+ // 4 KiB CPU/GPU aliases. Engine execution remains a separate implementation.
+ const u32_le native_arch{0x170};
+ const u32_le native_impl{0xe};
+ std::memcpy(result.data(), &native_arch, sizeof(native_arch));
+ std::memcpy(result.data() + 4, &native_impl, sizeof(native_impl));
+ // Native NVN's built-in shader table accepts Ampere SM 8.x. Carrying the
+ // inherited Maxwell SM 5.3 value here rejects its initialization shaders.
+ const u32_le native_sm_version{0x807};
+ std::memcpy(result.data() + 0x3c, &native_sm_version, sizeof(native_sm_version));
+ std::memcpy(result.data() + 0x40, &native_sm_version, sizeof(native_sm_version));
+ // Native NVN maps CPU buffers into its shared VA reservation. Those buffers
+ // are 4 KiB aligned, so large pages must remain optional for allocation.
+ const u32_le preferred_page_size{0x1000};
+ std::memcpy(result.data() + 0x28, &preferred_page_size, sizeof(preferred_page_size));
+ u32_le available_page_sizes;
+ std::memcpy(&available_page_sizes, legacy.data() + 0x34, sizeof(available_page_sizes));
+ available_page_sizes = (static_cast<u32>(available_page_sizes) & 0x10000) | 0x1000;
+ std::memcpy(result.data() + 0x34, &available_page_sizes, sizeof(available_page_sizes));
+ u64_le flags;
+ std::memcpy(&flags, legacy.data() + 0x50, sizeof(flags));
+ // Native SDK class enumeration filters 2D/3D/compute/inline families
+ // with bits 40..43 before selecting context identities from its profile.
+ // The native SDK gates ModifyKind (AS ioctl 12) on capability bit 34.
+ flags = static_cast<u64>(flags) | (1ULL << 8) | (1ULL << 22) | (1ULL << 26) | (1ULL << 34) | (0xfULL << 40);
+ std::memcpy(result.data() + 0x50, &flags, sizeof(flags));
+ constexpr std::array<s16, 9> levels{
+ 0x1c, // GPU control: GetGpuTime
+ 11, // TSG: bind/unbind channel with native subcontext IDs
+ -1, // GPU debugger device is unavailable
+ 0x1d, // Channel: SetTimeslice
+ 0x14, // Address space: Remap
+ 0, 0, 0, 0, // Additional interfaces have not been implemented.
+ };
+ for (size_t i = 0; i < levels.size(); ++i) {
+ const u16_le value{static_cast<u16>(levels[i])};
+ std::memcpy(result.data() + 0x70 + i * sizeof(value), &value, sizeof(value));
+ }
+ // The native SDK allocates shared queue subcontexts locally, then binds
+ // each channel with TSG ioctl 11. Each emulated channel retains its own state.
+ const u32_le max_subcontexts{64};
+ std::memcpy(result.data() + 0xb4, &max_subcontexts, sizeof(max_subcontexts));
+ return result;
+}
+
+} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/native_gpu_context.h b/src/core/hle/service/nvdrv/devices/native_gpu_context.h
new file mode 100644
index 0000000000..7c7131f211
--- /dev/null
+++ b/src/core/hle/service/nvdrv/devices/native_gpu_context.h
@@ -0,0 +1,23 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <optional>
+#include "common/common_types.h"
+
+namespace Service::Nvidia::Devices {
+
+// Native profile context identities. This classifies allocation metadata;
+// engine method execution is handled separately by the GPU frontend.
+inline std::optional<u32> GetNativeGpuContextSlot(u32 identity) {
+ switch (identity) {
+ case 0xc797: return 1;
+ case 0xc7c0: return 2;
+ case 0xc7b5: return 4;
+ case 0xc56f: return 5;
+ default: return std::nullopt;
+ }
+}
+
+} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/native_gpu_kind.h b/src/core/hle/service/nvdrv/devices/native_gpu_kind.h
new file mode 100644
index 0000000000..ecf3781162
--- /dev/null
+++ b/src/core/hle/service/nvdrv/devices/native_gpu_kind.h
@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <optional>
+#include "common/common_types.h"
+#include "video_core/pte_kind.h"
+
+namespace Service::Nvidia::Devices {
+
+inline std::optional<u8> TranslateNativeUncompressedKind(u8 kind, u8 fallback) {
+ // NVIDIA's Turing MMU uses 6 for GENERIC_MEMORY and 7 for INVALID:
+ // open-gpu-doc/manuals/turing/tu104/dev_mmu.ref.txt. Eden's older kind
+ // numbering uses 0xfe for generic memory. Equal kinds need no compression.
+ if (kind != fallback) { return std::nullopt; }
+ switch (kind) {
+ case 0: return static_cast<u8>(Tegra::PTEKind::PITCH);
+ case 1: return static_cast<u8>(Tegra::PTEKind::Z16);
+ case 3: return static_cast<u8>(Tegra::PTEKind::S8Z24);
+ case 6: return static_cast<u8>(Tegra::PTEKind::GENERIC_16BX2);
+ default: return std::nullopt;
+ }
+}
+
+struct NativeMemoryKind {
+ u8 backend;
+ u8 selected;
+};
+
+inline std::optional<NativeMemoryKind> SelectNativeMemoryKind(u8 requested, u8 fallback) {
+ if (const auto kind = TranslateNativeUncompressedKind(requested, fallback)) {
+ return NativeMemoryKind{*kind, requested};
+ }
+ // The native caller supplies an uncompressed alternative explicitly. Return
+ // that selected native kind so the SDK invalidates its compressed preference.
+ if (requested == 8 && fallback == 6) {
+ return NativeMemoryKind{static_cast<u8>(Tegra::PTEKind::GENERIC_16BX2), 6};
+ }
+ // TU104 kind 0x0c is the compressible S8Z24 variant; kind 3 is
+ // the caller's uncompressed S8Z24 alternative (dev_mmu.ref.txt).
+ if (requested == 0xb && fallback == 1) {
+ return NativeMemoryKind{static_cast<u8>(Tegra::PTEKind::Z16), 1};
+ }
+ if (requested == 0xc && fallback == 3) {
+ return NativeMemoryKind{static_cast<u8>(Tegra::PTEKind::S8Z24), 3};
+ }
+ return std::nullopt;
+}
+
+} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/nvdevice.h b/src/core/hle/service/nvdrv/devices/nvdevice.h
index d23ec63bae..cbcee2a938 100644
--- a/src/core/hle/service/nvdrv/devices/nvdevice.h
+++ b/src/core/hle/service/nvdrv/devices/nvdevice.h
@@ -27,7 +27,9 @@ namespace Service::Nvidia::Devices {
/// implement the ioctl interface.
class nvdevice {
public:
- explicit nvdevice(Core::System& system_) : system{system_} {}
+ enum class Type { Other, GpuChannel, GpuTsg, GpuAddressSpace };
+ explicit nvdevice(Core::System& system_, Type type_ = Type::Other) : system{system_}, type{type_} {}
+ Type GetType() const { return type; }
virtual ~nvdevice() = default;
/**
@@ -80,6 +82,8 @@ public:
protected:
Core::System& system;
+private:
+ const Type type;
};
} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/nvdisp_disp0.cpp b/src/core/hle/service/nvdrv/devices/nvdisp_disp0.cpp
index b8a8979315..0db83215ca 100644
--- a/src/core/hle/service/nvdrv/devices/nvdisp_disp0.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvdisp_disp0.cpp
@@ -5,17 +5,24 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <boost/container/small_vector.hpp>
+#include <cstdlib>
+#include <fstream>
+#include <filesystem>
+#include "core/frontend/framebuffer_layout.h"
+#include "video_core/renderer_base.h"
#include "common/adpf.h"
#include "common/assert.h"
#include "common/logging.h"
#include "core/core.h"
+#include "core/hle/kernel/k_process.h"
#include "core/core_timing.h"
#include "core/hle/service/nvdrv/core/container.h"
#include "core/hle/service/nvdrv/core/nvmap.h"
#include "core/hle/service/nvdrv/devices/nvdisp_disp0.h"
#include "core/perf_stats.h"
#include "video_core/gpu.h"
+#include "video_core/host1x/host1x.h"
namespace Service::Nvidia::Devices {
@@ -67,6 +74,9 @@ void nvdisp_disp0::Composite(std::span<const Nvnflinger::HwcLayer> sorted_layers
output_fences.reserve(sorted_layers.size());
for (auto& layer : sorted_layers) {
+ const auto handle = nvmap.GetHandle(layer.buffer_handle);
+ const auto* session = handle ? container.GetSession(handle->session_id) : nullptr;
+ const bool native_tiling = session && session->process->UsesPAuthReturns();
output_layers.emplace_back(Tegra::FramebufferConfig{
.address = nvmap.GetHandleAddress(layer.buffer_handle),
.offset = layer.offset,
@@ -77,6 +87,7 @@ void nvdisp_disp0::Composite(std::span<const Nvnflinger::HwcLayer> sorted_layers
.transform_flags = layer.transform,
.crop_rect = layer.crop_rect,
.blending = ConvertBlending(layer.blending),
+ .native_tiling = native_tiling,
.layer_stack_mask = layer.layer_stack_mask,
});
@@ -87,6 +98,56 @@ void nvdisp_disp0::Composite(std::span<const Nvnflinger::HwcLayer> sorted_layers
}
}
+ const char* capture = std::getenv("EDEN_NATIVE_FRAME_CAPTURE");
+ const bool sequence = std::getenv("EDEN_NATIVE_FRAME_SEQUENCE") != nullptr;
+ const bool on_demand = std::getenv("EDEN_NATIVE_FRAME_ON_DEMAND") != nullptr;
+ const bool requested = on_demand && !output_layers.empty() &&
+ !diagnostic_frame_pending->load() && std::filesystem::exists("native-frame.request");
+ if (requested) std::filesystem::remove("native-frame.request");
+ const bool sample_frame = requested || (capture && std::string_view{capture} == "1" && !output_layers.empty() &&
+ diagnostic_frame_count < (sequence ? 36000U : 3840U) && ++diagnostic_frame_count != 0 &&
+ (sequence ? (diagnostic_frame_count == 1 || diagnostic_frame_count % 120 == 0)
+ : !diagnostic_visible_frame->load()));
+ if (sample_frame && !diagnostic_frame_pending->exchange(true)) {
+ for (const auto& fence : output_fences) {
+ LOG_INFO(Service_NVDRV, "Native composite waits for syncpoint={}/{}, current={}", fence.id, fence.value,
+ system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(fence.id));
+ }
+ LOG_INFO(Service_NVDRV, "Native composite submitted: layers={}, address={:#x}", output_layers.size(), output_layers.front().address);
+ const auto layout = Layout::DefaultFrameLayout(1920, 1080);
+ auto pixels = std::make_shared<std::vector<u8>>(layout.width * layout.height * 4);
+ const bool first = diagnostic_frame_count == 1;
+ auto pending = diagnostic_frame_pending;
+ auto visible = diagnostic_visible_frame;
+ system.GPU().Renderer().RequestScreenshot(pixels->data(), [pixels, layout, first, pending, visible, sequence, on_demand](bool invert_y) {
+ std::array<u8, 18> header{};
+ header[2] = 2;
+ header[12] = static_cast<u8>(layout.width & 255);
+ header[13] = static_cast<u8>(layout.width >> 8);
+ header[14] = static_cast<u8>(layout.height & 255);
+ header[15] = static_cast<u8>(layout.height >> 8);
+ header[16] = 32;
+ header[17] = invert_y ? 8 : 0x28;
+ bool nonblack{};
+ for (size_t i = 0; i < pixels->size(); i += 4) {
+ if ((*pixels)[i] || (*pixels)[i + 1] || (*pixels)[i + 2]) {
+ nonblack = true;
+ break;
+ }
+ }
+ const auto save = [&](const char* path) {
+ std::ofstream out(path, std::ios::binary);
+ out.write(reinterpret_cast<const char*>(header.data()), header.size());
+ out.write(reinterpret_cast<const char*>(pixels->data()), pixels->size());
+ return out.good();
+ };
+ if (first) save("native-first-frame.tga");
+ const bool saved = (first || nonblack || sequence || on_demand) ? save("native-latest-frame.tga") : true;
+ if (nonblack && !visible->load() && save("native-visible-frame.tga")) visible->store(true);
+ LOG_INFO(Service_NVDRV, "Captured composited frame: {}x{}, saved={}, nonblack={}", layout.width, layout.height, saved, nonblack);
+ pending->store(false);
+ }, layout);
+ }
system.GPU().RequestComposite(std::move(output_layers), std::move(output_fences));
Common::ADPF::ReportFrameInterval();
system.SpeedLimiter().DoSpeedLimiting(system.CoreTiming().GetGlobalTimeUs());
diff --git a/src/core/hle/service/nvdrv/devices/nvdisp_disp0.h b/src/core/hle/service/nvdrv/devices/nvdisp_disp0.h
index ff0bfb2ec9..f795144ebe 100644
--- a/src/core/hle/service/nvdrv/devices/nvdisp_disp0.h
+++ b/src/core/hle/service/nvdrv/devices/nvdisp_disp0.h
@@ -6,6 +6,7 @@
#pragma once
+#include <atomic>
#include <memory>
#include <vector>
#include "common/common_types.h"
@@ -45,6 +46,9 @@ public:
Kernel::KEvent* QueryEvent(u32 event_id) override;
private:
+ u32 diagnostic_frame_count{};
+ std::shared_ptr<std::atomic_bool> diagnostic_frame_pending{std::make_shared<std::atomic_bool>(false)};
+ std::shared_ptr<std::atomic_bool> diagnostic_visible_frame{std::make_shared<std::atomic_bool>(false)};
NvCore::Container& container;
NvCore::NvMap& nvmap;
};
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
index 3c9e42f602..947ea6088f 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
@@ -6,6 +6,7 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include <cstring>
+#include <limits>
#include <utility>
#include "common/alignment.h"
@@ -13,10 +14,13 @@
#include "common/common_types.h"
#include "common/logging.h"
#include "core/core.h"
+#include "core/hle/kernel/k_process.h"
+#include "core/hle/service/nvdrv/devices/gpu_address_space_layout.h"
#include "core/hle/service/nvdrv/core/container.h"
#include "core/hle/service/nvdrv/core/nvmap.h"
#include "core/hle/service/nvdrv/devices/ioctl_serialization.h"
#include "core/hle/service/nvdrv/devices/nvhost_as_gpu.h"
+#include "core/hle/service/nvdrv/devices/native_gpu_kind.h"
#include "core/hle/service/nvdrv/devices/nvhost_gpu.h"
#include "core/hle/service/nvdrv/nvdrv.h"
#include "video_core/control/channel_state.h"
@@ -27,10 +31,12 @@
namespace Service::Nvidia::Devices {
nvhost_as_gpu::nvhost_as_gpu(Core::System& system_, Module& module_, NvCore::Container& core)
- : nvdevice{system_}, module{module_}, container{core}, nvmap{core.GetNvMapFile()}, vm{},
+ : nvdevice{system_, Type::GpuAddressSpace}, module{module_}, container{core}, nvmap{core.GetNvMapFile()}, vm{},
gmmu{} {}
-nvhost_as_gpu::~nvhost_as_gpu() = default;
+nvhost_as_gpu::~nvhost_as_gpu() {
+ if (m_owner_process) { m_owner_process->Close(system.Kernel()); }
+}
NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input,
std::span<u8> output) {
@@ -51,6 +57,10 @@ NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> i
return WrapFixed(this, &nvhost_as_gpu::GetVARegions1, input, output);
case 0x9:
return WrapFixed(this, &nvhost_as_gpu::AllocAsEx, input, output);
+ case 0xb:
+ return WrapFixed(this, &nvhost_as_gpu::GetSyncpointShimInfo, input, output);
+ case 0xc:
+ return WrapFixed(this, &nvhost_as_gpu::ModifyKind, input, output);
case 0x14:
return WrapVariable(this, &nvhost_as_gpu::Remap, input, output);
default:
@@ -90,11 +100,139 @@ NvResult nvhost_as_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> i
return NvResult::NotImplemented;
}
-void nvhost_as_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) {}
+void nvhost_as_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) {
+ m_fd = fd;
+ if (const auto* session = container.GetSession(session_id)) {
+ m_owner_process = session->process;
+ m_owner_process->Open(system.Kernel());
+ m_extended_abi = m_owner_process->UsesPAuthReturns();
+ }
+}
void nvhost_as_gpu::OnClose(DeviceFD fd) {}
+NvResult nvhost_as_gpu::CreateNativeSparseRegion(u64 address, u64 size) {
+ if (!m_extended_abi || !size || !Common::IsAligned(address, VM::YUZU_PAGESIZE) ||
+ !Common::IsAligned(size, VM::YUZU_PAGESIZE) || (size >> VM::PAGE_SIZE_BITS) > 0xffffffffULL) {
+ return NvResult::BadParameter;
+ }
+ {
+ const std::scoped_lock lock{mutex};
+ if (!vm.initialised || address < vm.va_range_start || address >= vm.va_range_split ||
+ size > vm.va_range_split - address) { return NvResult::BadParameter; }
+ const auto next = allocation_map.lower_bound(address);
+ if ((next != allocation_map.end() && next->first < address + size) ||
+ (next != allocation_map.begin() && std::prev(next)->first + std::prev(next)->second.size > address)) {
+ return NvResult::BadParameter;
+ }
+ }
+ IoctlAllocSpace reservation{};
+ reservation.pages = static_cast<u32>(size >> VM::PAGE_SIZE_BITS);
+ reservation.page_size = VM::YUZU_PAGESIZE;
+ reservation.flags = MappingFlags::Fixed | MappingFlags::Sparse;
+ reservation.offset = address;
+ return AllocateSpace(reservation);
+}
+
+NvResult nvhost_as_gpu::DestroyNativeSparseRegion(u64 address, u64 size) {
+ {
+ const std::scoped_lock lock{mutex};
+ const auto it = allocation_map.find(address);
+ if (!m_extended_abi || it == allocation_map.end() || !it->second.sparse || it->second.size != size) {
+ return NvResult::BadParameter;
+ }
+ }
+ IoctlFreeSpace release{};
+ release.offset = address;
+ release.pages = static_cast<u32>(size >> VM::PAGE_SIZE_BITS);
+ release.page_size = VM::YUZU_PAGESIZE;
+ return FreeSpace(release);
+}
+
+NvResult nvhost_as_gpu::MapNative4K(u32 handle_id, u64 address, u64 buffer_offset, u64 size, u8 kind) {
+ if (!m_extended_abi || !size || !Common::IsAligned(address, VM::YUZU_PAGESIZE) ||
+ !Common::IsAligned(size, VM::YUZU_PAGESIZE) ||
+ !Common::IsAligned(buffer_offset, VM::YUZU_PAGESIZE)) {
+ return NvResult::BadParameter;
+ }
+ const auto handle = nvmap.GetHandle(handle_id);
+ if (!handle || buffer_offset > handle->orig_size || size > handle->orig_size - buffer_offset) {
+ return NvResult::BadParameter;
+ }
+ bool sparse_reservation = false;
+ {
+ const std::scoped_lock lock{mutex};
+ if (!vm.initialised || address < vm.va_range_start || address >= vm.va_range_split ||
+ size > vm.va_range_split - address) {
+ return NvResult::BadParameter;
+ }
+ const auto upper = allocation_map.upper_bound(address);
+ if (upper != allocation_map.begin()) {
+ const auto owner = std::prev(upper);
+ sparse_reservation = owner->second.sparse && address >= owner->first &&
+ address - owner->first < owner->second.size && size <= owner->second.size - (address - owner->first);
+ }
+ const auto next = allocation_map.lower_bound(address);
+ if (!sparse_reservation && ((next != allocation_map.end() && next->first < address + size) ||
+ (next != allocation_map.begin() &&
+ std::prev(next)->first + std::prev(next)->second.size > address))) {
+ return NvResult::BadParameter;
+ }
+ }
+ IoctlAllocSpace reservation{};
+ reservation.pages = static_cast<u32>(size >> VM::PAGE_SIZE_BITS);
+ reservation.page_size = VM::YUZU_PAGESIZE;
+ reservation.flags = MappingFlags::Fixed;
+ reservation.offset = address;
+ const auto reserved = sparse_reservation ? NvResult::Success : AllocateSpace(reservation);
+ if (reserved != NvResult::Success) { return reserved; }
+ IoctlMapBufferEx mapping{};
+ mapping.flags = MappingFlags::Fixed;
+ mapping.kind = kind;
+ mapping.handle = handle_id;
+ mapping.page_size = VM::YUZU_PAGESIZE;
+ mapping.buffer_offset = static_cast<s64>(buffer_offset);
+ mapping.mapping_size = size;
+ mapping.offset = static_cast<s64>(address);
+ const auto result = MapBufferEx(mapping);
+ if (result != NvResult::Success && !sparse_reservation) {
+ nvmap.UnpinHandle(handle_id);
+ IoctlFreeSpace release{};
+ release.offset = address;
+ release.pages = reservation.pages;
+ release.page_size = VM::YUZU_PAGESIZE;
+ FreeSpace(release);
+ }
+ return result;
+}
+
+NvResult nvhost_as_gpu::UnmapNative4K(u64 address, u64 size) {
+ if (!m_extended_abi || !size || !Common::IsAligned(address, VM::YUZU_PAGESIZE) ||
+ !Common::IsAligned(size, VM::YUZU_PAGESIZE)) {
+ return NvResult::BadParameter;
+ }
+ bool sparse_mapping = false;
+ {
+ const std::scoped_lock lock{mutex};
+ const auto it = mapping_map.find(address);
+ sparse_mapping = it != mapping_map.end() && it->second.sparse_alloc && it->second.size == size;
+ }
+ if (sparse_mapping) {
+ IoctlUnmapBuffer unmap{};
+ unmap.offset = address;
+ return UnmapBuffer(unmap);
+ }
+ IoctlFreeSpace release{};
+ release.offset = address;
+ release.pages = static_cast<u32>(size >> VM::PAGE_SIZE_BITS);
+ release.page_size = VM::YUZU_PAGESIZE;
+ return FreeSpace(release);
+}
+
+
NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) {
- LOG_DEBUG(Service_NVDRV, "called, big_page_size={:#x}", params.big_page_size);
+ LOG_DEBUG(Service_NVDRV, "called, flags={:#x}, as_fd={}, big_page_size={:#x}, reserved={:#x}, VA={:#x}/{:#x}/{:#x}",
+ params.flags, params.as_fd, params.big_page_size, params.reserved,
+ params.va_range_start, params.va_range_split, params.va_range_end);
std::scoped_lock lock(mutex);
@@ -103,21 +241,43 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) {
return NvResult::InvalidState;
}
- if (params.big_page_size) {
- if (!std::has_single_bit(params.big_page_size)) {
- LOG_ERROR(Service_NVDRV, "Non power-of-2 big page size: {:#x}!", params.big_page_size);
+ std::shared_ptr<nvhost_as_gpu> shared_parent;
+ if (m_extended_abi && params.as_fd != 0) {
+ if (params.as_fd == m_fd || params.flags != 5 || params.big_page_size != 0x1000 ||
+ params.reserved != 0 || params.va_range_split != 0) { return NvResult::BadParameter; }
+ const auto device = module.GetDevice<nvdevice>(params.as_fd);
+ if (!device || device->GetType() != Type::GpuAddressSpace) { return NvResult::BadParameter; }
+ shared_parent = std::static_pointer_cast<nvhost_as_gpu>(device);
+ if (!shared_parent->m_extended_abi || !shared_parent->vm.initialised ||
+ shared_parent->m_owner_process != m_owner_process ||
+ params.va_range_start < shared_parent->vm.va_range_start ||
+ params.va_range_start >= params.va_range_end ||
+ params.va_range_end > shared_parent->vm.va_range_split) { return NvResult::BadParameter; }
+ }
+
+ const u32 big_page_size = GetGpuAddressSpaceBigPageSize(m_extended_abi, params.big_page_size);
+ if (big_page_size) {
+ if (!std::has_single_bit(big_page_size)) {
+ LOG_ERROR(Service_NVDRV, "Non power-of-2 big page size: {:#x}!", big_page_size);
return NvResult::BadValue;
}
- if ((params.big_page_size & VM::SUPPORTED_BIG_PAGE_SIZES) == 0) {
- LOG_ERROR(Service_NVDRV, "Unsupported big page size: {:#x}!", params.big_page_size);
+ if ((big_page_size & VM::SUPPORTED_BIG_PAGE_SIZES) == 0) {
+ LOG_ERROR(Service_NVDRV, "Unsupported big page size: {:#x}!", big_page_size);
return NvResult::BadValue;
}
- vm.big_page_size = params.big_page_size;
- vm.big_page_size_bits = static_cast<u32>(std::countr_zero(params.big_page_size));
+ vm.big_page_size = big_page_size;
+ vm.big_page_size_bits = static_cast<u32>(std::countr_zero(big_page_size));
+
+ vm.va_range_start = big_page_size << VM::VA_START_SHIFT;
+ }
- vm.va_range_start = params.big_page_size << VM::VA_START_SHIFT;
+ if (m_extended_abi && !params.va_range_start) {
+ const auto layout = GetGpuAddressSpaceLayout(true);
+ vm.va_range_start = layout.start;
+ vm.va_range_split = layout.split;
+ vm.va_range_end = layout.end;
}
// If this is unspecified then default values should be used
@@ -127,6 +287,12 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) {
vm.va_range_end = params.va_range_end;
}
+ if (shared_parent) {
+ // Native flags 5 expose the parent's small-page region as a separate
+ // reservation view. GPU mappings must resolve through the same page table.
+ vm.va_range_split = params.va_range_end;
+ }
+
const u64 max_big_page_bits = Common::Log2Ceil(vm.va_range_end);
const auto start_pages{static_cast<u32>(vm.va_range_start >> VM::PAGE_SIZE_BITS)};
@@ -135,11 +301,17 @@ NvResult nvhost_as_gpu::AllocAsEx(IoctlAllocAsEx& params) {
const auto start_big_pages{static_cast<u32>(vm.va_range_split >> vm.big_page_size_bits)};
const auto end_big_pages{
- static_cast<u32>((vm.va_range_end - vm.va_range_split) >> vm.big_page_size_bits)};
+ static_cast<u32>(vm.va_range_end >> vm.big_page_size_bits)};
vm.big_page_allocator.emplace(start_big_pages, end_big_pages);
- gmmu = std::make_unique<Tegra::MemoryManager>(system, max_big_page_bits, vm.va_range_split, vm.big_page_size_bits, VM::PAGE_SIZE_BITS);
- system.GPU().InitAddressSpace(*gmmu);
+ if (shared_parent) {
+ m_shared_parent = std::move(shared_parent);
+ gmmu = m_shared_parent->gmmu;
+ LOG_DEBUG(Service_NVDRV, "native AS view fd={} shares page table with fd={}", m_fd, params.as_fd);
+ } else {
+ gmmu = std::make_shared<Tegra::MemoryManager>(system, max_big_page_bits, vm.va_range_split, vm.big_page_size_bits, VM::PAGE_SIZE_BITS);
+ system.GPU().InitAddressSpace(*gmmu);
+ }
vm.initialised = true;
return NvResult::Success;
@@ -159,7 +331,7 @@ NvResult nvhost_as_gpu::AllocateSpace(IoctlAllocSpace& params) {
return NvResult::BadValue;
}
- if (params.page_size != vm.big_page_size &&
+ if (!m_extended_abi && params.page_size != vm.big_page_size &&
((params.flags & MappingFlags::Sparse) != MappingFlags::None)) {
UNIMPLEMENTED_MSG("Sparse small pages are not implemented!");
return NvResult::NotImplemented;
@@ -184,7 +356,7 @@ NvResult nvhost_as_gpu::AllocateSpace(IoctlAllocSpace& params) {
u64 size{static_cast<u64>(params.pages) * params.page_size};
if ((params.flags & MappingFlags::Sparse) != MappingFlags::None) {
- gmmu->MapSparse(params.offset, size);
+ gmmu->MapSparse(params.offset, size, params.page_size != VM::YUZU_PAGESIZE);
}
allocation_map[params.offset] = {
@@ -217,6 +389,7 @@ bool nvhost_as_gpu::FreeMappingLocked(u64 offset) noexcept {
gmmu->Unmap(offset, mapping.size);
}
mapping_map.erase(offset);
+ map_buffer_offsets.erase(static_cast<s64>(offset));
return true;
}
return false;
@@ -251,6 +424,77 @@ NvResult nvhost_as_gpu::FreeSpace(IoctlFreeSpace& params) {
return NvResult::BadValue;
}
+NvResult nvhost_as_gpu::GetNativeSyncpointShim(u64& address, u32& stride, u32& count) {
+ if (!m_extended_abi || !vm.initialised) { return NvResult::NotSupported; }
+ address = vm.va_range_split - VM::YUZU_PAGESIZE;
+ if (!m_syncpoint_shim_reserved) {
+ IoctlAllocSpace allocation{};
+ allocation.pages = 1;
+ allocation.page_size = VM::YUZU_PAGESIZE;
+ allocation.flags = MappingFlags::Fixed;
+ allocation.offset = address;
+ const auto result = AllocateSpace(allocation);
+ if (result != NvResult::Success) { return result; }
+ m_syncpoint_shim_reserved = true;
+ gmmu->SetNativeSyncpointShim(address);
+ }
+ stride = sizeof(u32);
+ count = 192;
+ return NvResult::Success;
+}
+
+NvResult nvhost_as_gpu::GetSyncpointShimInfo(IoctlSyncpointShimInfo& params) {
+ u64 address{};
+ u32 stride{}, count{};
+ const auto result = GetNativeSyncpointShim(address, stride, count);
+ params.address = address;
+ params.stride = stride;
+ params.count = count;
+ LOG_DEBUG(Service_NVDRV, "native syncpoint shim address={:#x}, stride={}, count={}, result={}",
+ address, stride, count, static_cast<u32>(result));
+ return result;
+}
+
+NvResult nvhost_as_gpu::ModifyNativeKind(u64 address, u64 size, u8 requested, u8 fallback) {
+ IoctlModifyKind params{};
+ params.compressed_kind = requested;
+ params.uncompressed_kind = fallback;
+ params.mapping_base = address;
+ params.size = size;
+ return ModifyKind(params);
+}
+
+NvResult nvhost_as_gpu::ModifyKind(IoctlModifyKind& params) {
+ const u32 requested = params.compressed_kind;
+ const u32 fallback = params.uncompressed_kind;
+ const u64 base = params.mapping_base;
+ const u64 offset = params.buffer_offset;
+ const u64 size = params.size;
+ LOG_DEBUG(Service_NVDRV, "native modify kind base={:#x}, offset={:#x}, size={:#x}, kinds={:#x}/{:#x}",
+ base, offset, size, requested, fallback);
+ if (!m_extended_abi || params.reserved != 0 || fallback > 255 ||
+ (requested > 255 && requested != 0xffff) || !size ||
+ offset > std::numeric_limits<u64>::max() - base ||
+ !Common::IsAligned(base + offset, VM::YUZU_PAGESIZE) ||
+ !Common::IsAligned(size, VM::YUZU_PAGESIZE)) {
+ return NvResult::BadParameter;
+ }
+ const auto kind = SelectNativeMemoryKind(static_cast<u8>(requested == 0xffff ? fallback : requested),
+ static_cast<u8>(fallback));
+ if (!kind) { return NvResult::NotSupported; }
+ const std::scoped_lock lock{mutex};
+ const u64 address = base + offset;
+ auto it = mapping_map.upper_bound(address);
+ if (!vm.initialised || it == mapping_map.begin()) { return NvResult::BadParameter; }
+ --it;
+ const auto& mapping = it->second;
+ const u64 delta = address - it->first;
+ if (delta >= mapping.size || size > mapping.size - delta) { return NvResult::BadParameter; }
+ gmmu->Map(address, mapping.ptr + delta, size, static_cast<Tegra::PTEKind>(kind->backend),
+ mapping.big_page);
+ return NvResult::Success;
+}
+
NvResult nvhost_as_gpu::Remap(std::span<IoctlRemapEntry> entries) {
LOG_DEBUG(Service_NVDRV, "called, num_entries={:#x}", entries.size());
@@ -414,7 +658,13 @@ NvResult nvhost_as_gpu::UnmapBuffer(IoctlUnmapBuffer& params) {
NvResult nvhost_as_gpu::BindChannel(IoctlBindChannel& params) {
LOG_DEBUG(Service_NVDRV, "called, fd={:X}", params.fd);
- auto gpu_channel_device = module.GetDevice<nvhost_gpu>(params.fd);
+ const auto device = module.GetDevice<nvdevice>(params.fd);
+ if (!device || device->GetType() != nvdevice::Type::GpuChannel) { return NvResult::BadParameter; }
+ const auto gpu_channel_device = std::static_pointer_cast<nvhost_gpu>(device);
+ if (!vm.initialised || !gmmu || (gpu_channel_device->channel_state->initialized &&
+ gpu_channel_device->channel_state->memory_manager != gmmu.get())) { return NvResult::InvalidState; }
+ gpu_channel_device->channel_state->memory_manager_owner =
+ std::shared_ptr<Tegra::MemoryManager>(module.GetDevice<nvhost_as_gpu>(m_fd), gmmu.get());
gpu_channel_device->channel_state->memory_manager = gmmu.get();
return NvResult::Success;
}
@@ -424,13 +674,13 @@ void nvhost_as_gpu::GetVARegionsImpl(IoctlGetVaRegions& params) {
params.regions = std::array<VaRegion, 2>{
VaRegion{
- .offset = vm.small_page_allocator->GetVAStart() << VM::PAGE_SIZE_BITS,
+ .offset = GpuPageAddress(vm.small_page_allocator->GetVAStart(), VM::PAGE_SIZE_BITS),
.page_size = VM::YUZU_PAGESIZE,
._pad0_{},
.pages = vm.small_page_allocator->GetVALimit() - vm.small_page_allocator->GetVAStart(),
},
VaRegion{
- .offset = vm.big_page_allocator->GetVAStart() << vm.big_page_size_bits,
+ .offset = GpuPageAddress(vm.big_page_allocator->GetVAStart(), vm.big_page_size_bits),
.page_size = vm.big_page_size,
._pad0_{},
.pages = vm.big_page_allocator->GetVALimit() - vm.big_page_allocator->GetVAStart(),
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
index b13b1c9efa..8c7306e42a 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
+++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
@@ -64,6 +64,13 @@ public:
Kernel::KEvent* QueryEvent(u32 event_id) override;
+ NvResult CreateNativeSparseRegion(u64 address, u64 size);
+ NvResult DestroyNativeSparseRegion(u64 address, u64 size);
+ NvResult MapNative4K(u32 handle, u64 address, u64 buffer_offset, u64 size, u8 kind);
+ NvResult UnmapNative4K(u64 address, u64 size);
+ NvResult GetNativeSyncpointShim(u64& address, u32& stride, u32& count);
+ NvResult ModifyNativeKind(u64 address, u64 size, u8 requested, u8 fallback);
+
struct VaRegion {
u64 offset;
u32 page_size;
@@ -145,6 +152,25 @@ private:
static_assert(sizeof(IoctlGetVaRegions) == 16 + sizeof(VaRegion) * 2,
"IoctlGetVaRegions is incorrect size");
+ struct IoctlSyncpointShimInfo {
+ u64_le address;
+ u32_le stride;
+ u32_le count;
+ };
+ static_assert(sizeof(IoctlSyncpointShimInfo) == 16);
+ NvResult GetSyncpointShimInfo(IoctlSyncpointShimInfo& params);
+
+ struct IoctlModifyKind {
+ u16_le compressed_kind;
+ u16_le uncompressed_kind;
+ u32_le reserved;
+ u64_le buffer_offset;
+ u64_le size;
+ u64_le mapping_base;
+ };
+ static_assert(sizeof(IoctlModifyKind) == 32);
+ NvResult ModifyKind(IoctlModifyKind& params);
+
NvResult AllocAsEx(IoctlAllocAsEx& params);
NvResult AllocateSpace(IoctlAllocSpace& params);
NvResult Remap(std::span<IoctlRemapEntry> params);
@@ -196,6 +222,12 @@ private:
std::map<u64, Allocation> allocation_map;
std::mutex mutex; //!< Locks all AS operations
+ bool m_extended_abi{};
+ bool m_syncpoint_shim_reserved{};
+ DeviceFD m_fd{-1};
+ Kernel::KProcess* m_owner_process{};
+ std::shared_ptr<nvhost_as_gpu> m_shared_parent;
+
struct VM {
static constexpr u32 YUZU_PAGESIZE{0x1000};
static constexpr u32 PAGE_SIZE_BITS{static_cast<u32>(std::countr_zero<u32>(YUZU_PAGESIZE))};
@@ -219,7 +251,7 @@ private:
bool initialised{};
} vm;
- std::unique_ptr<Tegra::MemoryManager> gmmu;
+ std::shared_ptr<Tegra::MemoryManager> gmmu;
};
} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
index 14cdd8fda4..6684352b00 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
@@ -4,6 +4,7 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <algorithm>
#include <cstring>
#include "common/assert.h"
#include "common/logging.h"
@@ -11,6 +12,7 @@
#include "core/core_timing.h"
#include "core/hle/service/nvdrv/devices/ioctl_serialization.h"
#include "core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h"
+#include "core/hle/service/nvdrv/devices/native_gpu_characteristics.h"
#include "core/hle/service/nvdrv/nvdrv.h"
namespace Service::Nvidia::Devices {
@@ -94,6 +96,25 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8>
case 'G':
switch (command.cmd) {
case 0x5:
+ if (command.raw == 0xc0e04705) {
+ constexpr size_t packet_size = 0x10 + NativeGpuCharacteristicsSize;
+ if (input.size() < packet_size || output.size() < packet_size ||
+ inline_output.size() < NativeGpuCharacteristicsSize) {
+ return NvResult::BadParameter;
+ }
+ IoctlCharacteristics legacy{};
+ GetCharacteristics3(legacy, {});
+ const auto native = MakeNativeGpuCharacteristics(
+ std::span<const u8, 0xa0>{reinterpret_cast<const u8*>(&legacy.gc), 0xa0});
+ std::fill(output.begin(), output.end(), u8{});
+ std::fill(inline_output.begin(), inline_output.end(), u8{});
+ std::memcpy(output.data(), input.data(), 0x10);
+ const u64_le size{NativeGpuCharacteristicsSize};
+ std::memcpy(output.data(), &size, sizeof(size));
+ std::memcpy(output.data() + 0x10, native.data(), native.size());
+ std::memcpy(inline_output.data(), native.data(), native.size());
+ return NvResult::Success;
+ }
return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetCharacteristics3, input, output,
inline_output);
case 0x6:
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp
index 1e571be4c9..6b9678e440 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp
@@ -4,6 +4,8 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <cstdlib>
+#include <fstream>
#include <cstring>
#include "common/assert.h"
#include "common/logging.h"
@@ -14,11 +16,13 @@
#include "core/hle/service/nvdrv/core/syncpoint_manager.h"
#include "core/hle/service/nvdrv/devices/ioctl_serialization.h"
#include "core/hle/service/nvdrv/devices/nvhost_gpu.h"
+#include "core/hle/service/nvdrv/devices/native_gpu_context.h"
#include "core/hle/service/nvdrv/nvdrv.h"
#include "core/memory.h"
#include "video_core/control/channel_state.h"
#include "video_core/engines/puller.h"
#include "video_core/gpu.h"
+#include "video_core/memory_manager.h"
#include "video_core/host1x/host1x.h"
namespace Service::Nvidia::Devices {
@@ -33,7 +37,7 @@ Tegra::CommandHeader BuildFenceAction(Tegra::Engines::Puller::FenceOperation op,
nvhost_gpu::nvhost_gpu(Core::System& system_, EventInterface& events_interface_,
NvCore::Container& core_)
- : nvdevice{system_}, events_interface{events_interface_}, core{core_},
+ : nvdevice{system_, Type::GpuChannel}, events_interface{events_interface_}, core{core_},
syncpoint_manager{core_.GetSyncpointManager()}, nvmap{core.GetNvMapFile()},
channel_state{system.GPU().AllocateChannel()} {
channel_syncpoint = syncpoint_manager.AllocateSyncpoint(false);
@@ -128,6 +132,7 @@ NvResult nvhost_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> inpu
void nvhost_gpu::OnOpen(NvCore::SessionId session_id, DeviceFD fd) {
sessions[fd] = session_id;
+ channel_state->native_abi = UsesNativeAbi();
}
void nvhost_gpu::OnClose(DeviceFD fd) {
@@ -280,6 +285,9 @@ NvResult nvhost_gpu::AllocateObjectContext(IoctlAllocObjCtx& params) {
s32_le ctx_class_number_index =
GetObjectContextClassNumberIndex(static_cast<CtxClasses>(params.class_num));
+ if (const auto native_slot = GetNativeGpuContextSlot(params.class_num)) {
+ if (UsesNativeAbi()) { ctx_class_number_index = *native_slot; }
+ }
if (ctx_class_number_index < 0) {
LOG_ERROR(Service_NVDRV, "Invalid class number for object context: {:#x}",
params.class_num);
@@ -311,13 +319,13 @@ static boost::container::small_vector<Tegra::CommandHeader, 512> BuildWaitComman
}
static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementCommandList(
- NvFence fence) {
+ NvFence fence, u32 increment_count = 2) {
boost::container::small_vector<Tegra::CommandHeader, 512> result{
Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointPayload, 1,
Tegra::SubmissionMode::Increasing),
{}};
- for (u32 count = 0; count < 2; ++count) {
+ for (u32 count = 0; count < increment_count; ++count) {
result.push_back(Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointOperation, 1,
Tegra::SubmissionMode::Increasing));
result.push_back(
@@ -328,12 +336,12 @@ static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementC
}
static boost::container::small_vector<Tegra::CommandHeader, 512> BuildIncrementWithWfiCommandList(
- NvFence fence) {
+ NvFence fence, u32 increment_count = 2) {
boost::container::small_vector<Tegra::CommandHeader, 512> result{
Tegra::BuildCommandHeader(Tegra::BufferMethods::WaitForIdle, 1,
Tegra::SubmissionMode::Increasing),
{}};
- auto increment_list{BuildIncrementCommandList(fence)};
+ auto increment_list{BuildIncrementCommandList(fence, increment_count)};
result.insert(result.end(), increment_list.begin(), increment_list.end());
return result;
}
@@ -351,6 +359,41 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL
system.GPU().InitChannel(*channel_state, channel_state->program_id);
}
+ if (UsesNativeAbi() && native_submit_logs++ < 8) {
+ LOG_DEBUG(Service_NVDRV, "native submit channel={}, initialized={}, entries={}, flags={:#x}, fence={}:{}",
+ channel_state->bind_id, channel_state->initialized, entries.command_lists.size(),
+ params.flags.raw, params.fence.id, params.fence.value);
+ for (size_t i = 0; i < std::min<size_t>(entries.command_lists.size(), 4); ++i) {
+ const auto& entry = entries.command_lists[i];
+ std::array<u32, 16> words{};
+ const size_t bytes = std::min<size_t>(entry.size * sizeof(u32), sizeof(words));
+ const bool mapped = channel_state->memory_manager && bytes &&
+ channel_state->memory_manager->IsFullyMappedRange(entry.addr, bytes);
+ if (mapped) { channel_state->memory_manager->ReadBlockUnsafe(entry.addr, words.data(), bytes); }
+ if (mapped && entry.size > 1000 && std::getenv("EDEN_NATIVE_FRAME_CAPTURE")) {
+ std::vector<u32> contents(static_cast<size_t>(entry.size));
+ channel_state->memory_manager->ReadBlockUnsafe(entry.addr, contents.data(), contents.size() * sizeof(u32));
+ std::ofstream dump("native-init-channel-" + std::to_string(channel_state->bind_id) + "-" + std::to_string(i) + ".bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(contents.data()), contents.size() * sizeof(u32));
+ }
+ LOG_DEBUG(Service_NVDRV, "native pushbuffer entry={} raw={:#x}, address={:#x}, words={}, mapped={}, data={:08x}",
+ i, entry.raw, static_cast<u64>(entry.addr), static_cast<u64>(entry.size),
+ mapped, fmt::join(words, ","));
+ }
+ }
+
+ if (UsesNativeAbi() && std::getenv("EDEN_NATIVE_FRAME_CAPTURE") && native_submit_logs < 100) {
+ for (size_t i = 0; i < entries.command_lists.size(); ++i) {
+ const auto& entry = entries.command_lists[i];
+ const auto bytes = static_cast<size_t>(entry.size) * sizeof(u32);
+ if (!bytes || bytes > 0x100000 || !channel_state->memory_manager->IsFullyMappedRange(entry.addr, bytes)) continue;
+ std::vector<u32> contents(static_cast<size_t>(entry.size));
+ channel_state->memory_manager->ReadBlockUnsafe(entry.addr, contents.data(), bytes);
+ std::ofstream dump("native-submit-" + std::to_string(channel_state->bind_id) + "-" + std::to_string(native_submit_logs) + "-" + std::to_string(i) + ".bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(contents.data()), bytes);
+ }
+ }
+
const auto bind_id = channel_state->bind_id;
auto& flags = params.flags;
@@ -365,6 +408,8 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL
}
}
+ const u32 native_completion = UsesNativeAbi() && flags.increment_value.Value() && !flags.fence_increment.Value() ? params.fence.value : 0;
+ if (native_completion > 64) return NvResult::BadParameter;
params.fence.id = channel_syncpoint;
u32 increment{(flags.fence_increment.Value() != 0 ? 2 : 0) +
@@ -382,6 +427,14 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL
}
}
+ if (native_completion) {
+ // The native C56F stream reports its completion count without emitting
+ // host syncpoint methods. Order actual increments after that GPU work.
+ gpu.PushGPUEntries(bind_id, Tegra::CommandList{flags.suppress_wfi.Value()
+ ? BuildIncrementCommandList(params.fence, native_completion)
+ : BuildIncrementWithWfiCommandList(params.fence, native_completion)});
+ LOG_DEBUG(Service_NVDRV, "Native ordered completion: syncpoint={}, increments={}, target={}", channel_syncpoint, native_completion, params.fence.value);
+ }
flags.raw = 0;
return NvResult::Success;
@@ -463,10 +516,46 @@ Kernel::KEvent* nvhost_gpu::QueryEvent(u32 event_id) {
return sm_exception_breakpoint_pause_report_event;
case 3:
return error_notifier_event;
+ case 7:
+ // The native SDK requests this event immediately before configuring
+ // CHANNEL_SET_ERROR_NOTIFIER, then wraps it in its channel error wait.
+ return UsesNativeAbi() ? error_notifier_event : nullptr;
default:
LOG_CRITICAL(Service_NVDRV, "Unknown Ctrl GPU Event {}", event_id);
return nullptr;
}
}
+NvResult nvhost_gpu::BindToTsg(DeviceFD group, Kernel::KProcess* process, std::shared_ptr<Tegra::NativeMacroContext> macros) {
+ const std::scoped_lock lock{tsg_mutex};
+ if (tsg_group) { return NvResult::Busy; }
+ bool owned{};
+ for (const auto& [fd, session_id] : sessions) {
+ const auto* session = core.GetSession(session_id);
+ owned |= session && session->process == process;
+ }
+ if (!owned) { return NvResult::AccessDenied; }
+ if (channel_state->native_abi) {
+ channel_state->native_macro_context = std::move(macros);
+ if (channel_state->payload) {
+ channel_state->payload->maxwell_3d.SetNativeMacroContext(channel_state->native_macro_context);
+ }
+ }
+ tsg_group = group;
+ return NvResult::Success;
+}
+
+void nvhost_gpu::UnbindFromTsg(DeviceFD group) {
+ const std::scoped_lock lock{tsg_mutex};
+ if (tsg_group == group) { tsg_group.reset(); }
+}
+
+bool nvhost_gpu::UsesNativeAbi() const {
+ for (const auto& [fd, session_id] : sessions) {
+ const auto* session = core.GetSession(session_id);
+ if (session && session->process && session->process->UsesPAuthReturns()) { return true; }
+ }
+ return false;
+}
+
} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h
index 70a6b1bd68..8219e4afea 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h
+++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h
@@ -7,6 +7,8 @@
#pragma once
#include <memory>
+#include <mutex>
+#include <optional>
#include <vector>
#include "common/bit_field.h"
#include "common/common_funcs.h"
@@ -20,7 +22,10 @@ namespace Core::Memory {
class Memory;
}
+namespace Kernel { class KProcess; }
+
namespace Tegra {
+struct NativeMacroContext;
namespace Control {
struct ChannelState;
}
@@ -58,9 +63,12 @@ public:
void OnClose(DeviceFD fd) override;
Kernel::KEvent* QueryEvent(u32 event_id) override;
+ NvResult BindToTsg(DeviceFD group, Kernel::KProcess* process, std::shared_ptr<Tegra::NativeMacroContext> macros);
+ void UnbindFromTsg(DeviceFD group);
private:
friend class nvhost_as_gpu;
+ bool UsesNativeAbi() const;
enum class CtxClasses : u32_le {
Ctx2D = 0x902D,
Ctx3D = 0xB197,
@@ -218,6 +226,9 @@ private:
NvCore::NvMap& nvmap;
std::shared_ptr<Tegra::Control::ChannelState> channel_state;
::Common::unordered_map<DeviceFD, NvCore::SessionId> sessions;
+ std::mutex tsg_mutex;
+ std::optional<DeviceFD> tsg_group;
+ u32 native_submit_logs{};
u32 channel_syncpoint;
std::mutex channel_mutex;
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.cpp
new file mode 100644
index 0000000000..ae9d2a50f5
--- /dev/null
+++ b/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.cpp
@@ -0,0 +1,85 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <algorithm>
+#include <array>
+#include <cstring>
+#include "common/logging.h"
+#include "core/core.h"
+#include "video_core/macro.h"
+#include "core/hle/kernel/k_process.h"
+#include "core/hle/service/nvdrv/nvdrv.h"
+#include "core/hle/service/nvdrv/devices/nvhost_gpu.h"
+#include "core/hle/service/nvdrv/devices/nvhost_tsg_gpu.h"
+
+namespace Service::Nvidia::Devices {
+
+nvhost_tsg_gpu::nvhost_tsg_gpu(Core::System& system, Module& module_, NvCore::Container& core_)
+ : nvdevice{system, Type::GpuTsg}, native_macros{std::make_shared<Tegra::NativeMacroContext>()}, module{module_}, core{core_} { native_macros->engine.native_turing = true; native_macros->compute_engine.native_turing = true; }
+
+nvhost_tsg_gpu::~nvhost_tsg_gpu() { OnClose(0); }
+
+void nvhost_tsg_gpu::OnOpen(NvCore::SessionId id, DeviceFD fd) {
+ group_fd = fd;
+ if (const auto* session = core.GetSession(id)) {
+ process = session->process;
+ process->Open(system.Kernel());
+ }
+}
+
+void nvhost_tsg_gpu::OnClose(DeviceFD fd) {
+ for (const auto& [channel_fd, channel] : channels) { channel->UnbindFromTsg(group_fd); }
+ channels.clear();
+ if (process) {
+ process->Close(system.Kernel());
+ process = nullptr;
+ }
+}
+
+
+NvResult nvhost_tsg_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input,
+ std::span<u8> output) {
+ std::array<u32, 8> words{};
+ if (!input.empty()) { std::memcpy(words.data(), input.data(), std::min(input.size(), sizeof(words))); }
+ LOG_DEBUG(Service_NVDRV, "native TSG ioctl={:#x}, input={}, output={}, words={:#x},{:#x},{:#x},{:#x},{:#x},{:#x},{:#x},{:#x}",
+ command.raw, input.size(), output.size(), words[0], words[1], words[2], words[3],
+ words[4], words[5], words[6], words[7]);
+ if (command.raw != 0xc0045401 && command.raw != 0xc0045402 && command.raw != 0xc018540b) { return NvResult::NotImplemented; }
+ const size_t packet_size = command.raw == 0xc018540b ? 24 : 4;
+ if (!process || input.size() != packet_size || output.size() != packet_size) {
+ return NvResult::BadParameter;
+ }
+ if (packet_size == 24 && (words[1] >= 64 || words[2] || words[3] || words[4] || words[5])) {
+ return NvResult::BadParameter;
+ }
+ const DeviceFD channel_fd = static_cast<s32>(words[0]);
+ if (command.cmd == 1 || command.cmd == 11) {
+ if (channels.contains(channel_fd)) { return NvResult::Busy; }
+ if (channels.size() >= 64) { return NvResult::InsufficientMemory; }
+ const auto device = module.GetDevice<nvdevice>(channel_fd);
+ if (!device || device->GetType() != Type::GpuChannel) { return NvResult::BadParameter; }
+ const auto channel = std::static_pointer_cast<nvhost_gpu>(device);
+ const auto result = channel->BindToTsg(fd, process, native_macros);
+ if (result != NvResult::Success) { return result; }
+ channels.emplace(channel_fd, channel);
+ } else {
+ const auto it = channels.find(channel_fd);
+ if (it == channels.end()) { return NvResult::BadParameter; }
+ it->second->UnbindFromTsg(fd);
+ channels.erase(it);
+ }
+ std::copy(input.begin(), input.end(), output.begin());
+ return NvResult::Success;
+}
+
+NvResult nvhost_tsg_gpu::Ioctl2(DeviceFD fd, Ioctl command, std::span<const u8> input,
+ std::span<const u8> inline_input, std::span<u8> output) {
+ return NvResult::NotImplemented;
+}
+
+NvResult nvhost_tsg_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> input,
+ std::span<u8> output, std::span<u8> inline_output) {
+ return NvResult::NotImplemented;
+}
+
+} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.h
new file mode 100644
index 0000000000..d64c5df3e0
--- /dev/null
+++ b/src/core/hle/service/nvdrv/devices/nvhost_tsg_gpu.h
@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include "core/hle/service/nvdrv/devices/nvdevice.h"
+
+namespace Tegra { struct NativeMacroContext; }
+namespace Kernel { class KProcess; }
+namespace Service::Nvidia { class Module; }
+
+namespace Service::Nvidia::Devices {
+class nvhost_gpu;
+
+class nvhost_tsg_gpu final : public nvdevice {
+public:
+ nvhost_tsg_gpu(Core::System& system, Module& module, NvCore::Container& core);
+ ~nvhost_tsg_gpu() override;
+ NvResult Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input,
+ std::span<u8> output) override;
+ NvResult Ioctl2(DeviceFD fd, Ioctl command, std::span<const u8> input,
+ std::span<const u8> inline_input, std::span<u8> output) override;
+ NvResult Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> input,
+ std::span<u8> output, std::span<u8> inline_output) override;
+ void OnOpen(NvCore::SessionId session, DeviceFD fd) override;
+ void OnClose(DeviceFD fd) override;
+private:
+ std::shared_ptr<Tegra::NativeMacroContext> native_macros;
+ Module& module;
+ NvCore::Container& core;
+ Kernel::KProcess* process{};
+ DeviceFD group_fd{-1};
+ std::map<DeviceFD, std::shared_ptr<nvhost_gpu>> channels;
+};
+
+} // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/nvdrv.cpp b/src/core/hle/service/nvdrv/nvdrv.cpp
index b252a2efa5..9873d07c24 100644
--- a/src/core/hle/service/nvdrv/nvdrv.cpp
+++ b/src/core/hle/service/nvdrv/nvdrv.cpp
@@ -18,6 +18,8 @@
#include "core/hle/service/nvdrv/devices/nvhost_ctrl.h"
#include "core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h"
#include "core/hle/service/nvdrv/devices/nvhost_gpu.h"
+#include "core/hle/service/nvdrv/devices/nvhost_tsg_gpu.h"
+#include "core/hle/kernel/k_process.h"
#include "core/hle/service/nvdrv/devices/nvhost_nvdec.h"
#include "core/hle/service/nvdrv/devices/nvhost_nvdec_common.h"
#include "core/hle/service/nvdrv/devices/nvhost_nvjpg.h"
@@ -133,6 +135,10 @@ Module::Module(Core::System& system)
auto device = std::make_shared<Devices::nvhost_gpu>(system, events_interface, container);
return open_files.emplace(fd, std::move(device)).first;
};
+ builders["/dev/nvhost-tsg-gpu"] = [this, &system](DeviceFD fd) {
+ auto device = std::make_shared<Devices::nvhost_tsg_gpu>(system, *this, container);
+ return open_files.emplace(fd, std::move(device)).first;
+ };
builders["/dev/nvhost-ctrl-gpu"] = [this, &system](DeviceFD fd) {
auto device = std::make_shared<Devices::nvhost_ctrl_gpu>(system, events_interface);
return open_files.emplace(fd, std::move(device)).first;
@@ -163,7 +169,12 @@ Module::Module(Core::System& system)
};
}
-Module::~Module() {}
+Module::~Module() {
+ // Devices (including TSG-retained channels) release events and core state.
+ // Destroy them while those module members are still alive.
+ for (const auto& [fd, device] : open_files) { device->OnClose(fd); }
+ open_files.clear();
+}
NvResult Module::VerifyFD(DeviceFD fd) const {
if (fd < 0) {
@@ -180,6 +191,12 @@ NvResult Module::VerifyFD(DeviceFD fd) const {
}
DeviceFD Module::Open(const std::string& device_name, NvCore::SessionId session_id) {
+ if (device_name == "/dev/nvhost-tsg-gpu") {
+ const auto* session = container.GetSession(session_id);
+ if (!session || !session->process || !session->process->UsesPAuthReturns()) {
+ return INVALID_NVDRV_FD;
+ }
+ }
auto it = builders.find(device_name);
if (it == builders.end()) {
LOG_ERROR(Service_NVDRV, "Trying to open unknown device {}", device_name);
diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.cpp b/src/core/hle/service/nvdrv/nvdrv_interface.cpp
index 5fae68adbb..3f81720a39 100644
--- a/src/core/hle/service/nvdrv/nvdrv_interface.cpp
+++ b/src/core/hle/service/nvdrv/nvdrv_interface.cpp
@@ -72,6 +72,10 @@ void NVDRV::Ioctl1(HLERequestContext& ctx) {
const auto input_buffer = ctx.ReadBuffer(0);
const auto nv_result = nvdrv->Ioctl1(fd, command, input_buffer, output_buffer);
+ if (nv_result != NvResult::Success && ctx.GetThread().GetOwnerProcess()->UsesPAuthReturns()) {
+ LOG_DEBUG(Service_NVDRV, "native ioctl failure fd={}, ioctl={:#x}, result={}, caller={:#x}",
+ fd, command.raw, static_cast<u32>(nv_result), ctx.GetThread().GetContext().lr);
+ }
if (command.is_out != 0) {
ctx.WriteBuffer(output_buffer);
}
@@ -99,6 +103,10 @@ void NVDRV::Ioctl2(HLERequestContext& ctx) {
const auto nv_result =
nvdrv->Ioctl2(fd, command, input_buffer, input_inlined_buffer, output_buffer);
+ if (nv_result != NvResult::Success && ctx.GetThread().GetOwnerProcess()->UsesPAuthReturns()) {
+ LOG_DEBUG(Service_NVDRV, "native ioctl failure fd={}, ioctl={:#x}, result={}, caller={:#x}",
+ fd, command.raw, static_cast<u32>(nv_result), ctx.GetThread().GetContext().lr);
+ }
if (command.is_out != 0) {
ctx.WriteBuffer(output_buffer);
}
@@ -126,6 +134,11 @@ void NVDRV::Ioctl3(HLERequestContext& ctx) {
const auto nv_result =
nvdrv->Ioctl3(fd, command, input_buffer, output_buffer, inline_output_buffer);
+ if (command.raw == 0xc0e04705) {
+ LOG_DEBUG(Service_NVDRV, "native characteristics input={}, output={}, inline={}, result={}",
+ input_buffer.size(), output_buffer.size(), inline_output_buffer.size(),
+ static_cast<u32>(nv_result));
+ }
if (command.is_out != 0) {
ctx.WriteBuffer(output_buffer, 0);
ctx.WriteBuffer(inline_output_buffer, 1);
@@ -185,6 +198,10 @@ void NVDRV::QueryEvent(HLERequestContext& ctx) {
IPC::RequestParser rp{ctx};
const auto fd = rp.Pop<DeviceFD>();
const auto event_id = rp.Pop<u32>();
+ if (ctx.GetThread().GetOwnerProcess()->UsesPAuthReturns()) {
+ LOG_DEBUG(Service_NVDRV, "native QueryEvent fd={}, id={}, caller={:#x}", fd, event_id,
+ ctx.GetThread().GetContext().lr);
+ }
if (!is_initialized) {
ServiceError(ctx, NvResult::NotInitialized);
@@ -265,6 +282,14 @@ NVDRV::NVDRV(Core::System& system_, std::shared_ptr<Module> nvdrv_, const char*
RegisterHandlers(functions);
}
+NVDRV::NVDRV(Core::System& system, std::shared_ptr<Module> module, const char* name,
+ Kernel::KProcess* process, u64 aruid)
+ : NVDRV{system, std::move(module), name} {
+ session_id = nvdrv->GetContainer().OpenSession(process);
+ pid = aruid;
+ is_initialized = true;
+}
+
NVDRV::~NVDRV() {
if (is_initialized) {
auto& container = nvdrv->GetContainer();
diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.h b/src/core/hle/service/nvdrv/nvdrv_interface.h
index f628fca7df..d6fe4c18f5 100644
--- a/src/core/hle/service/nvdrv/nvdrv_interface.h
+++ b/src/core/hle/service/nvdrv/nvdrv_interface.h
@@ -17,6 +17,8 @@ namespace Service::Nvidia {
class NVDRV final : public ServiceFramework<NVDRV> {
public:
explicit NVDRV(Core::System& system_, std::shared_ptr<Module> nvdrv_, const char* name);
+ explicit NVDRV(Core::System& system, std::shared_ptr<Module> module, const char* name,
+ Kernel::KProcess* process, u64 aruid);
~NVDRV() override;
std::shared_ptr<Module> GetModule() const {
diff --git a/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp b/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp
index c3eb2796e3..9de8abaeb6 100644
--- a/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp
+++ b/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp
@@ -393,6 +393,15 @@ Status BufferQueueProducer::DetachNextBuffer(std::shared_ptr<GraphicBuffer>* out
return Status::NoError;
}
+std::optional<Fence> BufferQueueProducer::GetReleasedBufferFence(s32 slot) {
+ const std::scoped_lock lock{core->mutex};
+ if (slot < 0 || static_cast<size_t>(slot) >= slots.size() ||
+ slots[slot].buffer_state != BufferState::Free) {
+ return std::nullopt;
+ }
+ return slots[slot].fence;
+}
+
Status BufferQueueProducer::AttachBuffer(s32* out_slot,
const std::shared_ptr<GraphicBuffer>& buffer) {
if (out_slot == nullptr) {
diff --git a/src/core/hle/service/nvnflinger/buffer_queue_producer.h b/src/core/hle/service/nvnflinger/buffer_queue_producer.h
index 697de0ac9b..bf815eb8d8 100644
--- a/src/core/hle/service/nvnflinger/buffer_queue_producer.h
+++ b/src/core/hle/service/nvnflinger/buffer_queue_producer.h
@@ -12,6 +12,7 @@
#include <condition_variable>
#include <memory>
#include <mutex>
+#include <optional>
#include "common/common_funcs.h"
#include "common/cpu_features.h"
@@ -57,6 +58,7 @@ public:
Kernel::KReadableEvent* GetNativeHandle(u32 type_id) override;
public:
+ std::optional<Fence> GetReleasedBufferFence(s32 slot);
Status RequestBuffer(s32 slot, std::shared_ptr<GraphicBuffer>* buf);
Status SetBufferCount(s32 buffer_count);
Status DequeueBuffer(s32* out_slot, android::Fence* out_fence, bool async, u32 width,
diff --git a/src/core/hle/service/service.cpp b/src/core/hle/service/service.cpp
index b4fe2b3ed5..a7e7e10199 100644
--- a/src/core/hle/service/service.cpp
+++ b/src/core/hle/service/service.cpp
@@ -4,12 +4,16 @@
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <chrono>
+#include <limits>
#include <fmt/ranges.h>
#include "common/assert.h"
#include "common/logging.h"
#include "common/settings.h"
#include "core/core.h"
#include "core/hle/ipc.h"
+#include "core/hle/kernel/k_process.h"
+#include "core/hle/kernel/k_thread.h"
#include "core/hle/kernel/kernel.h"
#include "core/hle/service/ipc_helpers.h"
#include "core/hle/service/service.h"
@@ -67,6 +71,26 @@ void ServiceFrameworkBase::ReportUnimplementedFunction(HLERequestContext& ctx,
fmt::format_to(std::back_inserter(buf), ", [{}]={:#x}", i, cmd_buf[i]);
buf.push_back('}');
+ if (ctx.GetThread().GetOwnerProcess()->UsesPAuthReturns()) {
+ const auto& context = ctx.GetThread().GetContext();
+ LOG_DEBUG(Service, "Unsupported IPC caller pc={:#x}, lr={:#x}, sp={:#x}, fp={:#x}",
+ context.pc, context.lr, context.sp, context.fp);
+ u64 frame = context.fp;
+ for (u32 depth = 0; depth < 8 && frame != 0; ++depth) {
+ if (frame > (std::numeric_limits<u64>::max)() - 15 ||
+ !ctx.GetMemory().IsValidVirtualAddress(frame) ||
+ !ctx.GetMemory().IsValidVirtualAddress(frame + 15)) {
+ break;
+ }
+ const u64 next = ctx.GetMemory().Read64(frame);
+ LOG_DEBUG(Service, "Unsupported IPC frame {}: return={:#x}", depth,
+ ctx.GetMemory().Read64(frame + 8));
+ if (next <= frame) {
+ break;
+ }
+ frame = next;
+ }
+ }
system.GetReporter().SaveUnimplementedFunctionReport(ctx, ctx.GetCommand(), function_name, service_name);
UNIMPLEMENTED_MSG("Unknown / unimplemented {}", fmt::to_string(buf));
if (Settings::values.use_auto_stub) {
@@ -82,7 +106,7 @@ void ServiceFrameworkBase::InvokeRequest(HLERequestContext& ctx) {
if (info == nullptr || info->handler_callback == nullptr)
return ReportUnimplementedFunction(ctx, info);
- LOG_TRACE(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer()));
+ LOG_DEBUG(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer()));
handler_invoker(this, info->handler_callback, ctx);
}
@@ -92,7 +116,7 @@ void ServiceFrameworkBase::InvokeRequestTipc(HLERequestContext& ctx) {
if (info == nullptr || info->handler_callback == nullptr)
return ReportUnimplementedFunction(ctx, info);
- LOG_TRACE(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer()));
+ LOG_DEBUG(Service, "{}", MakeFunctionString(info->name, GetServiceName(), ctx.CommandBuffer()));
handler_invoker(this, info->handler_callback, ctx);
}
diff --git a/src/core/launch_params.cpp b/src/core/launch_params.cpp
index a0483226ec..814a2cb553 100644
--- a/src/core/launch_params.cpp
+++ b/src/core/launch_params.cpp
@@ -94,7 +94,7 @@ LaunchParams ParseLaunchParams(Core::System& system, int argc, char *argv[], wch
break;
case 'u': {
// Launch game with a specific user
- bool argument_ok = isdigit(optarg[0]);
+ bool argument_ok = isdigit(static_cast<unsigned char>(optarg[0])) != 0;
p.selected_user = atoi(optarg);
if (!argument_ok) {
// try to look it up by username, only finds the first username that matches.
diff --git a/src/core/loader/deconstructed_rom_directory.cpp b/src/core/loader/deconstructed_rom_directory.cpp
index 5e2ec7e0b9..1dce245f7f 100644
--- a/src/core/loader/deconstructed_rom_directory.cpp
+++ b/src/core/loader/deconstructed_rom_directory.cpp
@@ -13,6 +13,7 @@
#include "core/file_sys/control_metadata.h"
#include "core/file_sys/patch_manager.h"
#include "core/file_sys/romfs_factory.h"
+#include "core/file_sys/romfs.h"
#include "core/hle/kernel/k_page_table.h"
#include "core/hle/kernel/k_process.h"
#include "core/hle/service/filesystem/filesystem.h"
@@ -129,10 +130,14 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(
: AppLoader(directory->GetFile("main"))
, dir(std::move(directory))
, override_update(override_update_)
+ , standalone(false)
{}
FileType AppLoader_DeconstructedRomDirectory::IdentifyType(const FileSys::VirtualFile& dir_file) {
- if (FileSys::IsDirectoryExeFS(dir_file->GetContainingDirectory())) {
+ const auto directory = dir_file->GetContainingDirectory();
+ if (dir_file->GetName() == "main" && directory &&
+ AppLoader_NSO::IdentifyType(dir_file) == FileType::NSO &&
+ FileSys::IsDirectoryExeFS(directory)) {
return FileType::DeconstructedRomDirectory;
}
@@ -182,11 +187,11 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect
}
metadata.Print();
- // Enable NCE only for applications with 39-bit address space.
+ // Extended ACID programs currently require the Dynarmic compatibility path.
const bool is_39bit =
metadata.GetAddressSpaceType() == FileSys::ProgramAddressSpaceType::Is39Bit;
const bool is_application = metadata.GetPoolPartition() == FileSys::PoolPartition::Application;
- Settings::SetNceEnabled(is_39bit);
+ Settings::SetNceEnabled(is_39bit && !metadata.UsesExtendedAcid());
const std::array static_modules = {"rtld", "main", "subsdk0", "subsdk1", "subsdk2",
"subsdk3", "subsdk4", "subsdk5", "subsdk6", "subsdk7",
@@ -267,11 +272,39 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect
}
is_loaded = true;
+ if (standalone) {
+ title_id = metadata.GetTitleID();
+ system.GetFileSystemController().RegisterProcess(
+ process.GetProcessId(), {},
+ std::make_shared<FileSys::RomFSFactory>(*this, system.GetContentProvider(),
+ system.GetFileSystemController()));
+ }
return {ResultStatus::Success,
LoadParameters{metadata.GetMainThreadPriority(), metadata.GetMainThreadStackSize()}};
}
ResultStatus AppLoader_DeconstructedRomDirectory::ReadRomFS(FileSys::VirtualFile& out_dir) {
+ if (romfs == nullptr && standalone) {
+ const auto directory = file->GetContainingDirectory();
+ for (const auto& entry : directory->GetFiles()) {
+ if (entry->GetExtension() == "romfs") {
+ romfs = entry;
+ break;
+ }
+ }
+ if (romfs == nullptr) {
+ auto loose = directory->GetSubdirectory("romfs");
+ if (!loose && directory->GetName() == "exefs") {
+ const auto parent = directory->GetParentDirectory();
+ if (parent) {
+ loose = parent->GetSubdirectory("romfs");
+ }
+ }
+ if (loose) {
+ romfs = FileSys::CreateRomFS(std::move(loose));
+ }
+ }
+ }
if (romfs == nullptr) {
return ResultStatus::ErrorNoRomFS;
}
diff --git a/src/core/loader/deconstructed_rom_directory.h b/src/core/loader/deconstructed_rom_directory.h
index 7283db0662..aba89935a3 100644
--- a/src/core/loader/deconstructed_rom_directory.h
+++ b/src/core/loader/deconstructed_rom_directory.h
@@ -62,6 +62,7 @@ private:
std::string name;
u64 title_id{};
bool override_update;
+ bool standalone{true};
Modules modules;
};
diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp
index 92e2e2f602..d802d15e9f 100644
--- a/src/core/loader/loader.cpp
+++ b/src/core/loader/loader.cpp
@@ -125,6 +125,10 @@ bool HasApplicationProgramContent(const std::shared_ptr<FileSys::NSP>& nsp) {
} // namespace
FileType IdentifyFile(FileSys::VirtualFile file) {
+ if (const auto romdir_type =
+ IdentifyFileLoader<AppLoader_DeconstructedRomDirectory>(file)) {
+ return *romdir_type;
+ }
if (const auto nsp_type = IdentifyFileLoader<AppLoader_NSP>(file)) {
return *nsp_type;
} else if (const auto xci_type = IdentifyFileLoader<AppLoader_XCI>(file)) {
@@ -139,9 +143,6 @@ FileType IdentifyFile(FileSys::VirtualFile file) {
return *kip_type;
} else if (const auto nso_type = IdentifyFileLoader<AppLoader_NSO>(file)) {
return *nso_type;
- } else if (const auto romdir_type =
- IdentifyFileLoader<AppLoader_DeconstructedRomDirectory>(file)) {
- return *romdir_type;
} else {
return FileType::Unknown;
}
diff --git a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp
index 492242fa73..af1d202edc 100644
--- a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp
+++ b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp
@@ -367,7 +367,11 @@ A64AddressSpace::A64AddressSpace(const A64::UserConfig& conf)
void A64AddressSpace::GenerateIR(IR::Block& ir_block, IR::LocationDescriptor descriptor) const {
ir_block.Reset(descriptor);
const auto get_code = [this](u64 vaddr) { return conf.callbacks->MemoryReadCode(vaddr); };
- A64::Translate(ir_block, A64::LocationDescriptor{descriptor}, get_code, {conf.define_unpredictable_behaviour, conf.wall_clock_cntpct});
+ A64::Translate(ir_block, A64::LocationDescriptor{descriptor}, get_code,
+ {.define_unpredictable_behaviour = conf.define_unpredictable_behaviour,
+ .wall_clock_cntpct = conf.wall_clock_cntpct,
+ .enable_pauth_returns = conf.enable_pauth_returns,
+ .address_space_bits = conf.page_table_address_space_bits});
Optimization::Optimize(ir_block, conf, {});
}
diff --git a/src/dynarmic/src/dynarmic/backend/x64/a64_interface.cpp b/src/dynarmic/src/dynarmic/backend/x64/a64_interface.cpp
index bb072129e7..0d5a7d3498 100644
--- a/src/dynarmic/src/dynarmic/backend/x64/a64_interface.cpp
+++ b/src/dynarmic/src/dynarmic/backend/x64/a64_interface.cpp
@@ -258,7 +258,11 @@ private:
// LocationDescriptor ctor() does important ops (like tflags) do not skip
auto const arch_descriptor = A64::LocationDescriptor{descriptor};
ir_block.Reset(arch_descriptor);
- A64::Translate(ir_block, arch_descriptor, get_code, {conf.define_unpredictable_behaviour, conf.wall_clock_cntpct});
+ A64::Translate(ir_block, arch_descriptor, get_code,
+ {.define_unpredictable_behaviour = conf.define_unpredictable_behaviour,
+ .wall_clock_cntpct = conf.wall_clock_cntpct,
+ .enable_pauth_returns = conf.enable_pauth_returns,
+ .address_space_bits = conf.page_table_address_space_bits});
Optimization::Optimize(ir_block, conf, polyfill_options);
return emitter.Emit(ir_block).entrypoint;
}
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc b/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc
index 8a3fab7ac1..0301a007a1 100644
--- a/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc
+++ b/src/dynarmic/src/dynarmic/frontend/A64/decoder/a64.inc
@@ -34,6 +34,16 @@ INST(IC_IVAU, "IC IVAU", "110101010000101101110101001ttttt")
INST(BLR, "BLR", "1101011000111111000000nnnnn00000")
INST(BR, "BR", "1101011000011111000000nnnnn00000")
INST(RET, "RET", "1101011001011111000000nnnnn00000")
+INST(RETA, "RETA", "110101100101111100001M1111111111")
+INST(CAS, "CAS", "zz0010001L1ssssso11111nnnnnttttt")
+INST(LDAPR, "LDAPR", "ss11100010111111110000nnnnnttttt")
+INST(LDADD, "LDADD", "1z111000AR1sssss000000nnnnnttttt")
+INST(LDCLR, "LDCLR", "1z111000AR1sssss000100nnnnnttttt")
+INST(LDSET, "LDSET", "1z111000AR1sssss001100nnnnnttttt")
+INST(SWPB, "SWPB", "00111000AR1sssss100000nnnnnttttt")
+INST(SWPH, "SWPH", "01111000AR1sssss100000nnnnnttttt")
+INST(SWP, "SWP", "1z111000AR1sssss100000nnnnnttttt")
+INST(XPAC_1, "XPAC", "110110101100000101000D11111ddddd")
INST(HINT, "HINT", "11010101000000110010MMMMooo11111")
INST(SXTB_1, "SXTB (32-bit)", "0001001100000000000111nnnnnddddd")
INST(SXTB_2, "SXTB (64-bit)", "1001001101000000000111nnnnnddddd")
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.h b/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.h
index 61033eb530..5acb50df74 100644
--- a/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.h
+++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/a64_translate.h
@@ -38,6 +38,9 @@ struct TranslationOptions {
/// If this is false, we treat the instruction as a NOP.
/// If this is true, we emit an ExceptionRaised instruction.
bool hook_hint_instructions = true;
+
+ bool enable_pauth_returns = false;
+ u32 address_space_bits = 64;
};
/**
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/a64_branch.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/a64_branch.cpp
index 36b91f13e9..4cb89f5e86 100644
--- a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/a64_branch.cpp
+++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/a64_branch.cpp
@@ -70,6 +70,14 @@ bool TranslatorVisitor::RET(Reg Rn) {
return false;
}
+bool TranslatorVisitor::RETA([[maybe_unused]] bool M) {
+ if (!options.enable_pauth_returns) {
+ return UnallocatedEncoding();
+ }
+ // With SCTLR.EnIA/EnIB clear, authentication leaves LR unchanged.
+ return RET(Reg::R30);
+}
+
bool TranslatorVisitor::CBZ(bool sf, Imm<19> imm19, Reg Rt) {
const size_t datasize = sf ? 64 : 32;
const s64 offset = concatenate(imm19, Imm<2>{0}).SignExtend<s64>();
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h
index c8afef3283..fdc9fe6ac1 100644
--- a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h
+++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/impl.h
@@ -251,7 +251,7 @@ struct TranslatorVisitor final {
bool CASP(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt);
bool CASB(bool L, Reg Rs, bool o0, Reg Rn, Reg Rt);
bool CASH(bool L, Reg Rs, bool o0, Reg Rn, Reg Rt);
- bool CAS(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt);
+ bool CAS(Imm<2> sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt);
// Loads and stores - Load register (literal)
bool LDR_lit_gen(bool opc_0, Imm<19> imm19, Reg Rt);
@@ -315,16 +315,16 @@ struct TranslatorVisitor final {
bool LDUMINH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
bool SWPH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
bool LDAPRH(Reg Rn, Reg Rt);
- bool LDADD(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
- bool LDCLR(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
+ bool LDADD(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
+ bool LDCLR(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
bool LDEOR(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
- bool LDSET(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
+ bool LDSET(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
bool LDSMAX(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
bool LDSMIN(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
bool LDUMAX(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
bool LDUMIN(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
- bool SWP(bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
- bool LDAPR(Reg Rn, Reg Rt);
+ bool SWP(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt);
+ bool LDAPR(Imm<2> size, Reg Rn, Reg Rt);
// Loads and stores - Load/Store register (register offset)
bool STRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, Reg Rt);
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp
index 0e67e11b4b..20ab71623f 100644
--- a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp
+++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/load_store_exclusive.cpp
@@ -6,9 +6,117 @@
#include <optional>
#include "dynarmic/frontend/A64/translate/impl/impl.h"
+#include "dynarmic/ir/terminal.h"
namespace Dynarmic::A64 {
+bool TranslatorVisitor::CAS(Imm<2> sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) {
+ const size_t bits = 8U << sz.ZeroExtend<size_t>();
+ const size_t reg_bits = bits == 64 ? 64 : 32;
+ const IR::U64 address = Rn == Reg::SP ? IR::U64{SP(64)} : IR::U64{X(64, Rn)};
+ const IR::U64 original_expected = X(64, Rs);
+ const IR::U32U64 expected = ZeroExtend(X(bits, Rs), reg_bits);
+ const IR::U32U64 desired = ZeroExtend(X(bits, Rt), reg_bits);
+ const auto flags = ir.GetNZCVRaw();
+ const auto access = L || o0 ? IR::AccType::ORDERED : IR::AccType::ATOMIC;
+ const auto observed = ZeroExtend(ExclusiveMem(address, bits / 8, access), reg_bits);
+
+ ir.SetNZCV(ir.NZCVFrom(ir.SubWithCarry(observed, expected, ir.Imm1(1))));
+ const auto replacement = ir.ConditionalSelect(IR::Cond::EQ, desired, observed);
+ ir.SetNZCVRaw(flags);
+ // A failed comparison may atomically write back the observed value.
+ IR::UAny store_value = replacement;
+ if (bits == 8) {
+ store_value = ir.LeastSignificantByte(replacement);
+ } else if (bits == 16) {
+ store_value = ir.LeastSignificantHalf(replacement);
+ }
+ const auto status = ExclusiveMem(address, bits / 8, access, store_value);
+ ir.SetNZCV(ir.NZCVFrom(ir.SubWithCarry(status, ir.Imm32(0), ir.Imm1(1))));
+ X(64, Rs, ir.ConditionalSelect(IR::Cond::EQ, ir.ZeroExtendToLong(observed), original_expected));
+ ir.SetNZCVRaw(flags);
+ ir.ClearExclusive();
+
+ // A store-exclusive failure must retry without changing the expected value.
+ ir.SetCheckBit(ir.IsZero(status));
+ ir.SetTerm(IR::Term::CheckBit{
+ IR::Term::LinkBlock{ir.current_location->AdvancePC(4)},
+ IR::Term::LinkBlock{*ir.current_location}});
+ return false;
+}
+
+bool TranslatorVisitor::LDAPR(Imm<2> size, Reg Rn, Reg Rt) {
+ // LDAR provides ordering at least as strong as an RCpc acquire load.
+ return LDAR(size, Rn, Rt);
+}
+
+enum class AtomicRegisterOperation { Add, Set, Clear, Swap };
+
+static bool AtomicRegisterOperationImpl(TranslatorVisitor& v, size_t bits, bool A, bool R,
+ Reg Rs, Reg Rn, Reg Rt, AtomicRegisterOperation operation) {
+ const size_t reg_bits = bits == 64 ? 64 : 32;
+ const IR::U64 address = Rn == Reg::SP ? IR::U64{v.SP(64)} : IR::U64{v.X(64, Rn)};
+ const IR::U64 original_result = v.X(64, Rt);
+ const IR::U32U64 operand = v.ZeroExtend(v.X(bits, Rs), reg_bits);
+ const auto flags = v.ir.GetNZCVRaw();
+ const auto access = A || R ? IR::AccType::ORDERED : IR::AccType::ATOMIC;
+ const auto observed = v.ZeroExtend(v.ExclusiveMem(address, bits / 8, access), reg_bits);
+ auto replacement = operand;
+ switch (operation) {
+ case AtomicRegisterOperation::Add:
+ replacement = v.ir.Add(observed, operand);
+ break;
+ case AtomicRegisterOperation::Set:
+ replacement = v.ir.Or(observed, operand);
+ break;
+ case AtomicRegisterOperation::Clear:
+ replacement = v.ir.And(observed, v.ir.Not(operand));
+ break;
+ case AtomicRegisterOperation::Swap:
+ break;
+ }
+ IR::UAny store_value = replacement;
+ if (bits == 8) {
+ store_value = v.ir.LeastSignificantByte(replacement);
+ } else if (bits == 16) {
+ store_value = v.ir.LeastSignificantHalf(replacement);
+ }
+ const auto status = v.ExclusiveMem(address, bits / 8, access, store_value);
+ v.ir.SetNZCV(v.ir.NZCVFrom(v.ir.SubWithCarry(status, v.ir.Imm32(0), v.ir.Imm1(1))));
+ v.X(64, Rt, v.ir.ConditionalSelect(IR::Cond::EQ, v.ir.ZeroExtendToLong(observed), original_result));
+ v.ir.SetNZCVRaw(flags);
+ v.ir.ClearExclusive();
+ v.ir.SetCheckBit(v.ir.IsZero(status));
+ v.ir.SetTerm(IR::Term::CheckBit{
+ IR::Term::LinkBlock{v.ir.current_location->AdvancePC(4)},
+ IR::Term::LinkBlock{*v.ir.current_location}});
+ return false;
+}
+
+bool TranslatorVisitor::LDADD(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt) {
+ return AtomicRegisterOperationImpl(*this, sz ? 64 : 32, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Add);
+}
+
+bool TranslatorVisitor::LDCLR(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt) {
+ return AtomicRegisterOperationImpl(*this, sz ? 64 : 32, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Clear);
+}
+
+bool TranslatorVisitor::LDSET(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt) {
+ return AtomicRegisterOperationImpl(*this, sz ? 64 : 32, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Set);
+}
+
+bool TranslatorVisitor::SWPB(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) {
+ return AtomicRegisterOperationImpl(*this, 8, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Swap);
+}
+
+bool TranslatorVisitor::SWPH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) {
+ return AtomicRegisterOperationImpl(*this, 16, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Swap);
+}
+
+bool TranslatorVisitor::SWP(bool sz, bool A, bool R, Reg Rs, Reg Rn, Reg Rt) {
+ return AtomicRegisterOperationImpl(*this, sz ? 64 : 32, A, R, Rs, Rn, Rt, AtomicRegisterOperation::Swap);
+}
+
static bool ExclusiveSharedDecodeAndOperation(TranslatorVisitor& v, bool pair, size_t size, bool L, bool o0, std::optional<Reg> Rs, std::optional<Reg> Rt2, Reg Rn, Reg Rt) {
// Shared Decode
diff --git a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/system.cpp b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/system.cpp
index c60fdc3ae0..8425493e3f 100644
--- a/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/system.cpp
+++ b/src/dynarmic/src/dynarmic/frontend/A64/translate/impl/system.cpp
@@ -34,6 +34,27 @@ enum class SystemRegisterEncoding : u32 {
};
bool TranslatorVisitor::HINT([[maybe_unused]] Imm<4> CRm, [[maybe_unused]] Imm<3> op2) {
+ if (options.enable_pauth_returns && CRm == 0 && op2 == 7) {
+ return XPAC_1(false, Reg::R30);
+ }
+ return true;
+}
+
+bool TranslatorVisitor::XPAC_1([[maybe_unused]] bool D, Reg Rd) {
+ if (!options.enable_pauth_returns) {
+ return UnallocatedEncoding();
+ }
+ const auto bits = options.address_space_bits;
+ if (bits >= 64) {
+ return true;
+ }
+ const IR::U64 pointer = X(64, Rd);
+ const u64 address_mask = (u64{1} << bits) - 1;
+ // This compatibility model has TBI disabled; bit 55 selects the address extension.
+ const auto extension = ir.ArithmeticShiftRight(
+ ir.LogicalShiftLeft(pointer, ir.Imm8(8)), ir.Imm8(63));
+ X(64, Rd, ir.Or(ir.And(pointer, ir.Imm64(address_mask)),
+ ir.And(extension, ir.Imm64(~address_mask))));
return true;
}
diff --git a/src/dynarmic/src/dynarmic/interface/A64/config.h b/src/dynarmic/src/dynarmic/interface/A64/config.h
index 859aeba709..6220f5c95e 100644
--- a/src/dynarmic/src/dynarmic/interface/A64/config.h
+++ b/src/dynarmic/src/dynarmic/interface/A64/config.h
@@ -242,6 +242,10 @@ struct UserConfig {
/// instruction is executed.
bool hook_hint_instructions = false;
+ /// Expose authenticated returns and PAC stripping with authentication keys disabled.
+ /// This does not generate or verify PACs. PAC/AUT hint instructions remain NOPs.
+ bool enable_pauth_returns = false;
+
/// Determines what happens if the guest accesses an entry that is off the end of the
/// page table. If true, Dynarmic will silently mirror page_table's address space. If
/// false, accessing memory outside of page_table bounds will result in a call to the
diff --git a/src/dynarmic/tests/A64/lse.cpp b/src/dynarmic/tests/A64/lse.cpp
new file mode 100644
index 0000000000..e26b03c321
--- /dev/null
+++ b/src/dynarmic/tests/A64/lse.cpp
@@ -0,0 +1,526 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/generators/catch_generators.hpp>
+
+#include "dynarmic/tests/A64/testenv.h"
+#include "dynarmic/interface/exclusive_monitor.h"
+
+namespace {
+class AtomicTestEnv final : public A64TestEnv {
+public:
+ Dynarmic::ExclusiveMonitor monitor{1};
+ bool fail_once{};
+ std::optional<u64> failure_value;
+ unsigned writes{};
+
+ bool MemoryWriteExclusive8(u64 address, u8 value, u8 expected) override {
+ return WriteSmallExclusive(address, value, expected, 1);
+ }
+
+ bool MemoryWriteExclusive16(u64 address, u16 value, u16 expected) override {
+ return WriteSmallExclusive(address, value, expected, 2);
+ }
+
+ bool WriteSmallExclusive(u64 address, u64 value, u64 expected, size_t bytes) {
+ ++writes;
+ if (fail_once) {
+ fail_once = false;
+ if (failure_value) {
+ MemoryWrite(address, *failure_value, bytes);
+ }
+ return false;
+ }
+ if (MemoryRead(address, bytes) != expected) {
+ return false;
+ }
+ MemoryWrite(address, value, bytes);
+ return true;
+ }
+
+ bool MemoryWriteExclusive32(u64 address, u32 value, u32 expected) override {
+ ++writes;
+ if (fail_once) {
+ fail_once = false;
+ if (failure_value) {
+ MemoryWrite(address, *failure_value, 4);
+ }
+ return false;
+ }
+ if (MemoryRead(address, 4) != expected) {
+ return false;
+ }
+ MemoryWrite(address, value, 4);
+ return true;
+ }
+
+ bool MemoryWriteExclusive64(u64 address, u64 value, u64 expected) override {
+ ++writes;
+ if (fail_once) {
+ fail_once = false;
+ if (failure_value) {
+ MemoryWrite(address, *failure_value, 8);
+ }
+ return false;
+ }
+ if (MemoryRead(address, 8) != expected) {
+ return false;
+ }
+ MemoryWrite(address, value, 8);
+ return true;
+ }
+};
+
+u32 CasInstruction(bool wide, bool acquire, bool release, u32 rs, u32 rn, u32 rt) {
+ return 0x88A07C00U | (u32(wide) << 30) | (u32(acquire) << 22) |
+ (rs << 16) | (u32(release) << 15) | (rn << 5) | rt;
+}
+} // namespace
+
+TEST_CASE("A64: CAS variants preserve flags and return the observed value", "[a64][lse]") {
+ const bool wide = GENERATE(false, true);
+ const bool acquire = GENERATE(false, true);
+ const bool release = GENERATE(false, true);
+ const bool match = GENERATE(false, true);
+ const bool retry = GENERATE(false, true);
+ const size_t bytes = wide ? 8 : 4;
+ const u64 observed = wide ? 0x1122334455667788ULL : 0x55667788ULL;
+ const u64 desired = wide ? 0x8877665544332211ULL : 0x44332211ULL;
+ AtomicTestEnv env;
+ env.code_mem = {CasInstruction(wide, acquire, release, 1, 0, 2), 0x14000000};
+ env.MemoryWrite(0x1000, observed, bytes);
+ env.fail_once = retry;
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(0, 0x1000);
+ jit.SetRegister(1, match ? observed : observed ^ 1);
+ jit.SetRegister(2, desired);
+ jit.SetPstate(0xA0000000);
+ env.ticks_left = 4;
+ jit.Run();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(1) == observed);
+ REQUIRE(jit.GetRegister(0) == 0x1000);
+ REQUIRE(jit.GetRegister(2) == desired);
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+ REQUIRE(env.MemoryRead(0x1000, bytes) == (match ? desired : observed));
+ REQUIRE(env.writes == (retry ? 2U : 1U));
+}
+
+TEST_CASE("A64: CAS handles SP and overlapping source registers", "[a64][lse]") {
+ const bool use_sp = GENERATE(false, true);
+ const bool overlap = GENERATE(false, true);
+ AtomicTestEnv env;
+ env.code_mem = {CasInstruction(true, true, true, 1, use_sp ? 31 : 0, overlap ? 1 : 2),
+ 0x14000000};
+ env.MemoryWrite(0x1000, 42, 8);
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(0, 0x1000);
+ jit.SetSP(0x1000);
+ jit.SetRegister(1, 42);
+ jit.SetRegister(2, 84);
+ env.ticks_left = 2;
+ jit.Run();
+ REQUIRE(jit.GetRegister(1) == 42);
+ REQUIRE(jit.GetSP() == 0x1000);
+ REQUIRE(env.MemoryRead(0x1000, 8) == (overlap ? 42 : 84));
+}
+
+TEST_CASE("A64: CAS retry preserves the full comparison register", "[a64][lse]") {
+ AtomicTestEnv env;
+ env.code_mem = {CasInstruction(false, true, false, 1, 0, 2), 0x14000000};
+ env.MemoryWrite(0x1000, 0x55667788, 4);
+ env.fail_once = true;
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(0, 0x1000);
+ jit.SetRegister(1, 0xFFFFFFFF55667788ULL);
+ jit.SetRegister(2, 42);
+ jit.SetPstate(0xA0000000);
+ env.ticks_left = 2;
+ jit.Step();
+ REQUIRE(jit.GetPC() == 0);
+ REQUIRE(jit.GetRegister(1) == 0xFFFFFFFF55667788ULL);
+ REQUIRE(env.MemoryRead(0x1000, 4) == 0x55667788);
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+ jit.Step();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(1) == 0x55667788);
+ REQUIRE(env.MemoryRead(0x1000, 4) == 42);
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+}
+
+TEST_CASE("A64: CAS supports zero-register inputs", "[a64][lse]") {
+ const bool wide = GENERATE(false, true);
+ const bool zero_expected = GENERATE(false, true);
+ const bool zero_desired = GENERATE(false, true);
+ AtomicTestEnv env;
+ env.code_mem = {CasInstruction(wide, false, true, zero_expected ? 31 : 1, 0,
+ zero_desired ? 31 : 2), 0x14000000};
+ const size_t bytes = wide ? 8 : 4;
+ env.MemoryWrite(0x1000, zero_expected ? 0 : 42, bytes);
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(0, 0x1000);
+ jit.SetRegister(1, 42);
+ jit.SetRegister(2, 84);
+ env.ticks_left = 2;
+ jit.Run();
+ REQUIRE(env.MemoryRead(0x1000, bytes) == (zero_desired ? 0 : 84));
+ REQUIRE(jit.GetRegister(1) == 42);
+}
+
+TEST_CASE("A64: LDAPR zero extends all supported widths", "[a64][rcpc]") {
+ const u32 size = GENERATE(0U, 1U, 2U, 3U);
+ AtomicTestEnv env;
+ env.code_mem = {0x38BFC001U | (size << 30), 0x14000000}; // LDAPR{B,H} R1, [X0]
+ const size_t bytes = size_t{1} << size;
+ const u64 value = 0xFEDCBA9876543210ULL;
+ env.MemoryWrite(0x1000, value, 8);
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(0, 0x1000);
+ jit.SetRegister(1, ~u64{0});
+ jit.SetPstate(0x60000000);
+ env.ticks_left = 2;
+ jit.Run();
+ const u64 mask = bytes == 8 ? ~u64{0} : (u64{1} << (bytes * 8)) - 1;
+ REQUIRE(jit.GetRegister(1) == (value & mask));
+ REQUIRE(jit.GetRegister(0) == 0x1000);
+ REQUIRE(jit.GetPstate() == 0x60000000);
+}
+
+TEST_CASE("A64: CAS retries observe an intervening write", "[a64][lse]") {
+ const bool wide = GENERATE(false, true);
+ AtomicTestEnv env;
+ env.code_mem = {CasInstruction(wide, true, true, 1, 0, 2), 0x14000000};
+ env.MemoryWrite(0x1000, 42, wide ? 8 : 4);
+ env.fail_once = true;
+ env.failure_value = 13;
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(0, 0x1000);
+ jit.SetRegister(1, 42);
+ jit.SetRegister(2, 84);
+ env.ticks_left = 4;
+ jit.Run();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(1) == 13);
+ REQUIRE(env.MemoryRead(0x1000, wide ? 8 : 4) == 13);
+ REQUIRE(env.writes == 2);
+}
+
+TEST_CASE("A64: LDADD variants handle retries, overlap, and overflow", "[a64][lse]") {
+ const bool wide = GENERATE(false, true);
+ const bool acquire = GENERATE(false, true);
+ const bool release = GENERATE(false, true);
+ const bool retry = GENERATE(false, true);
+ const bool overlap = GENERATE(false, true);
+ const bool wrap = GENERATE(false, true);
+ const size_t bytes = wide ? 8 : 4;
+ const u64 mask = wide ? ~u64{0} : 0xFFFFFFFFULL;
+ const u64 observed = wrap ? mask - 1 : 42;
+ const u64 operand = wrap ? 5 : 7;
+ const u32 rt = overlap ? 1 : 2;
+ AtomicTestEnv env;
+ const u32 instruction = 0xB8200000U | (u32(wide) << 30) | (u32(acquire) << 23) |
+ (u32(release) << 22) | (1U << 16) | rt;
+ env.code_mem = {instruction, 0x14000000};
+ env.MemoryWrite(0x1000, observed, bytes);
+ env.fail_once = retry;
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(0, 0x1000);
+ jit.SetRegister(1, operand);
+ jit.SetRegister(2, 0xABCD);
+ jit.SetPstate(0x60000000);
+ env.ticks_left = 4;
+ jit.Run();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(rt) == observed);
+ REQUIRE(jit.GetRegister(1) == (overlap ? observed : operand));
+ REQUIRE(jit.GetRegister(2) == (overlap ? 0xABCD : observed));
+ REQUIRE(jit.GetPstate() == 0x60000000);
+ REQUIRE(env.MemoryRead(0x1000, bytes) == ((observed + operand) & mask));
+ REQUIRE(env.writes == (retry ? 2U : 1U));
+}
+
+TEST_CASE("A64: small CAS truncates operands and preserves adjacent memory", "[a64][lse]") {
+ const bool half = GENERATE(false, true);
+ const bool acquire = GENERATE(false, true);
+ const bool release = GENERATE(false, true);
+ const bool match = GENERATE(false, true);
+ const bool retry = GENERATE(false, true);
+ const size_t bytes = half ? 2 : 1;
+ const u64 mask = half ? 0xffff : 0xff;
+ const u64 observed = 0x8765 & mask;
+ const u64 desired = 0xabcd & mask;
+ AtomicTestEnv env;
+ env.code_mem = {0x08A07C00U | (u32(half) << 30) | (u32(acquire) << 22) |
+ (1U << 16) | (u32(release) << 15) | 2U,
+ 0x14000000};
+ env.MemoryWrite(0x1000, 0x1122334455667788ULL, 8);
+ env.MemoryWrite(0x1002, observed, bytes);
+ const u64 before = env.MemoryRead(0x1000, 8);
+ env.fail_once = retry;
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(0, 0x1002);
+ const u64 expected = 0xfedcba9876540000ULL | (match ? observed : observed ^ 1);
+ jit.SetRegister(1, expected);
+ jit.SetRegister(2, 0x9876543210000000ULL | desired);
+ jit.SetPstate(0xA0000000);
+ if (retry) {
+ env.ticks_left = 1;
+ jit.Step();
+ REQUIRE(jit.GetPC() == 0);
+ REQUIRE(jit.GetRegister(1) == expected);
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+ }
+ env.ticks_left = 4;
+ jit.Run();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(1) == observed);
+ REQUIRE(jit.GetRegister(2) == (0x9876543210000000ULL | desired));
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+ const u64 after = (before & ~(mask << 16)) | ((match ? desired : observed) << 16);
+ REQUIRE(env.MemoryRead(0x1000, 8) == after);
+ REQUIRE(env.writes == (retry ? 2U : 1U));
+}
+
+TEST_CASE("A64: LDSET variants handle retries, overlap, and existing bits", "[a64][lse]") {
+ const bool wide = GENERATE(false, true);
+ const bool acquire = GENERATE(false, true);
+ const bool release = GENERATE(false, true);
+ const bool retry = GENERATE(false, true);
+ const bool overlap = GENERATE(false, true);
+ const bool existing_bits = GENERATE(false, true);
+ const size_t bytes = wide ? 8 : 4;
+ const u64 mask = wide ? ~u64{0} : 0xFFFFFFFFULL;
+ const u64 observed = existing_bits ? mask - 1 : 42;
+ const u64 operand = existing_bits ? 5 : 7;
+ const u32 rt = overlap ? 1 : 2;
+ AtomicTestEnv env;
+ const u32 instruction = 0xB8203000U | (u32(wide) << 30) | (u32(acquire) << 23) |
+ (u32(release) << 22) | (1U << 16) | rt;
+ env.code_mem = {instruction, 0x14000000};
+ env.MemoryWrite(0x1000, observed, bytes);
+ env.fail_once = retry;
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(0, 0x1000);
+ jit.SetRegister(1, operand);
+ jit.SetRegister(2, 0xABCD);
+ jit.SetPstate(0x60000000);
+ env.ticks_left = 4;
+ jit.Run();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(rt) == observed);
+ REQUIRE(jit.GetRegister(1) == (overlap ? observed : operand));
+ REQUIRE(jit.GetRegister(2) == (overlap ? 0xABCD : observed));
+ REQUIRE(jit.GetPstate() == 0x60000000);
+ REQUIRE(env.MemoryRead(0x1000, bytes) == ((observed | operand) & mask));
+ REQUIRE(env.writes == (retry ? 2U : 1U));
+}
+
+
+TEST_CASE("A64: observed LDSETL retries with a destination that is also its address", "[a64][lse]") {
+ const bool interference = GENERATE(false, true);
+ AtomicTestEnv env;
+ env.code_mem = {0xF8733108, 0x14000000};
+ env.MemoryWrite(0x1000, 2, 8);
+ env.fail_once = interference;
+ env.failure_value = 0x10;
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(8, 0x1000);
+ jit.SetRegister(19, 8);
+ jit.SetPstate(0xA0000000);
+ if (interference) {
+ env.ticks_left = 1;
+ jit.Step();
+ REQUIRE(jit.GetRegister(8) == 0x1000);
+ REQUIRE(jit.GetPC() == 0);
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+ }
+ env.ticks_left = 4;
+ jit.Run();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(8) == (interference ? 0x10 : 2));
+ REQUIRE(jit.GetRegister(19) == 8);
+ REQUIRE(env.MemoryRead(0x1000, 8) == (interference ? 0x18 : 0xa));
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+}
+
+TEST_CASE("A64: SWP variants handle retries and overlapping source registers", "[a64][lse]") {
+ const unsigned size = GENERATE(0U, 1U, 2U, 3U);
+ const bool acquire = GENERATE(false, true);
+ const bool release = GENERATE(false, true);
+ const bool retry = GENERATE(false, true);
+ const bool overlap = GENERATE(false, true);
+ const bool existing_bits = GENERATE(false, true);
+ const size_t bytes = size_t{1} << size;
+ const u64 mask = size == 3 ? ~u64{0} : (u64{1} << (8 * bytes)) - 1;
+ const u64 observed = existing_bits ? mask - 1 : 42;
+ const u64 operand = existing_bits ? 5 : 7;
+ const u32 rt = overlap ? 1 : 2;
+ AtomicTestEnv env;
+ const u32 instruction = 0x38208000U | (size << 30) | (u32(acquire) << 23) |
+ (u32(release) << 22) | (1U << 16) | rt;
+ env.code_mem = {instruction, 0x14000000};
+ env.MemoryWrite(0x1000, observed, bytes);
+ env.fail_once = retry;
+ env.MemoryWrite(0x1000 + bytes, 0xA5, 1);
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(0, 0x1000);
+ jit.SetRegister(1, operand);
+ jit.SetRegister(2, 0xABCD);
+ jit.SetPstate(0x60000000);
+ env.ticks_left = 4;
+ jit.Run();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(rt) == observed);
+ REQUIRE(jit.GetRegister(1) == (overlap ? observed : operand));
+ REQUIRE(jit.GetRegister(2) == (overlap ? 0xABCD : observed));
+ REQUIRE(jit.GetPstate() == 0x60000000);
+ REQUIRE(env.MemoryRead(0x1000, bytes) == (operand & mask));
+ REQUIRE(env.writes == (retry ? 2U : 1U));
+ REQUIRE(env.MemoryRead(0x1000 + bytes, 1) == 0xA5);
+}
+
+
+TEST_CASE("A64: observed SWPAL retries with a destination that is also its address", "[a64][lse]") {
+ const bool interference = GENERATE(false, true);
+ AtomicTestEnv env;
+ env.code_mem = {0xB8EA8129, 0x14000000};
+ env.MemoryWrite(0x1000, 2, 4);
+ env.fail_once = interference;
+ env.failure_value = 0x10;
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(9, 0x1000);
+ jit.SetRegister(10, 8);
+ jit.SetPstate(0xA0000000);
+ if (interference) {
+ env.ticks_left = 1;
+ jit.Step();
+ REQUIRE(jit.GetRegister(9) == 0x1000);
+ REQUIRE(jit.GetPC() == 0);
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+ }
+ env.ticks_left = 4;
+ jit.Run();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(9) == (interference ? 0x10 : 2));
+ REQUIRE(jit.GetRegister(10) == 8);
+ REQUIRE(env.MemoryRead(0x1000, 4) == 8);
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+}
+
+TEST_CASE("A64: LDCLR variants handle retries, overlap, and existing bits", "[a64][lse]") {
+ const bool wide = GENERATE(false, true);
+ const bool acquire = GENERATE(false, true);
+ const bool release = GENERATE(false, true);
+ const bool retry = GENERATE(false, true);
+ const bool overlap = GENERATE(false, true);
+ const bool existing_bits = GENERATE(false, true);
+ const size_t bytes = wide ? 8 : 4;
+ const u64 mask = wide ? ~u64{0} : 0xFFFFFFFFULL;
+ const u64 observed = existing_bits ? mask - 1 : 42;
+ const u64 operand = existing_bits ? 5 : 7;
+ const u32 rt = overlap ? 1 : 2;
+ AtomicTestEnv env;
+ const u32 instruction = 0xB8201000U | (u32(wide) << 30) | (u32(acquire) << 23) |
+ (u32(release) << 22) | (1U << 16) | rt;
+ env.code_mem = {instruction, 0x14000000};
+ env.MemoryWrite(0x1000, observed, bytes);
+ env.fail_once = retry;
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(0, 0x1000);
+ jit.SetRegister(1, operand);
+ jit.SetRegister(2, 0xABCD);
+ jit.SetPstate(0x60000000);
+ env.ticks_left = 4;
+ jit.Run();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(rt) == observed);
+ REQUIRE(jit.GetRegister(1) == (overlap ? observed : operand));
+ REQUIRE(jit.GetRegister(2) == (overlap ? 0xABCD : observed));
+ REQUIRE(jit.GetPstate() == 0x60000000);
+ REQUIRE(env.MemoryRead(0x1000, bytes) == ((observed & ~operand) & mask));
+ REQUIRE(env.writes == (retry ? 2U : 1U));
+}
+
+
+TEST_CASE("A64: observed LDCLRL retries with a destination that is also its address", "[a64][lse]") {
+ const bool interference = GENERATE(false, true);
+ AtomicTestEnv env;
+ env.code_mem = {0xF8691108, 0x14000000};
+ env.MemoryWrite(0x1000, 0xf, 8);
+ env.fail_once = interference;
+ env.failure_value = 0x10;
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.global_monitor = &env.monitor;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ jit.SetRegister(8, 0x1000);
+ jit.SetRegister(9, 1);
+ jit.SetPstate(0xA0000000);
+ if (interference) {
+ env.ticks_left = 1;
+ jit.Step();
+ REQUIRE(jit.GetRegister(8) == 0x1000);
+ REQUIRE(jit.GetPC() == 0);
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+ }
+ env.ticks_left = 4;
+ jit.Run();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(8) == (interference ? 0x10 : 0xf));
+ REQUIRE(jit.GetRegister(9) == 1);
+ REQUIRE(env.MemoryRead(0x1000, 8) == (interference ? 0x10 : 0xe));
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+}
diff --git a/src/dynarmic/tests/A64/pauth_returns.cpp b/src/dynarmic/tests/A64/pauth_returns.cpp
new file mode 100644
index 0000000000..f02e40e1af
--- /dev/null
+++ b/src/dynarmic/tests/A64/pauth_returns.cpp
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/generators/catch_generators.hpp>
+
+#include "dynarmic/tests/A64/testenv.h"
+
+namespace {
+class ReturnTestEnv final : public A64TestEnv {
+public:
+ Dynarmic::A64::Jit* jit{};
+ std::optional<Dynarmic::A64::Exception> exception;
+
+ void ExceptionRaised(u64, Dynarmic::A64::Exception reason) override {
+ exception = reason;
+ jit->HaltExecution(Dynarmic::HaltReason::UserDefined1);
+ }
+};
+} // namespace
+
+TEST_CASE("A64: authenticated returns with disabled authentication keys", "[a64][pauth]") {
+ const u32 instruction = GENERATE(0xD65F0BFFU, 0xD65F0FFFU); // RETAA, RETAB
+ const bool supported = GENERATE(false, true);
+ ReturnTestEnv env;
+ env.code_mem = {instruction};
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.enable_pauth_returns = supported;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ env.jit = &jit;
+ jit.SetPC(0);
+ jit.SetRegister(30, 0x123400);
+ jit.SetSP(0x567800);
+ env.ticks_left = 1;
+ jit.Step();
+ if (supported) {
+ REQUIRE_FALSE(env.exception.has_value());
+ REQUIRE(jit.GetPC() == 0x123400);
+ } else {
+ REQUIRE(env.exception == Dynarmic::A64::Exception::UnallocatedEncoding);
+ }
+ REQUIRE(jit.GetRegister(30) == 0x123400);
+ REQUIRE(jit.GetSP() == 0x567800);
+}
+
+TEST_CASE("A64: PAC hint and return agree on disabled keys", "[a64][pauth]") {
+ ReturnTestEnv env;
+ env.code_mem = {0xD503237FU, 0xD65F0FFFU}; // PACIBSP; RETAB
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.enable_pauth_returns = true;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ env.jit = &jit;
+ jit.SetPC(0);
+ jit.SetRegister(30, 0x123400);
+ jit.SetSP(0x567800);
+ env.ticks_left = 2;
+ jit.Step();
+ REQUIRE(jit.GetPC() == 4);
+ REQUIRE(jit.GetRegister(30) == 0x123400);
+ jit.Step();
+ REQUIRE_FALSE(env.exception.has_value());
+ REQUIRE(jit.GetPC() == 0x123400);
+ REQUIRE(jit.GetRegister(30) == 0x123400);
+ REQUIRE(jit.GetSP() == 0x567800);
+}
+
+TEST_CASE("A64: PAC stripping uses the configured address width", "[a64][pauth]") {
+ const u32 instruction = GENERATE(0xDAC143F4U, 0xDAC147F4U, 0xD50320FFU);
+ const u32 bits = GENERATE(32U, 36U, 39U);
+ const bool negative = GENERATE(false, true);
+ const u32 reg = instruction == 0xD50320FFU ? 30 : 20;
+ const u64 mask = (u64{1} << bits) - 1;
+ const u64 address = 0x123456789ULL & mask;
+ const u64 pointer = (0xAAULL << 56) | ((negative ? 0xD5ULL : 0x55ULL) << 48) | address;
+ ReturnTestEnv env;
+ env.code_mem = {instruction};
+ Dynarmic::A64::UserConfig config;
+ config.callbacks = &env;
+ config.enable_pauth_returns = true;
+ config.page_table_address_space_bits = bits;
+ config.code_cache_size = 16 * 1024 * 1024;
+ Dynarmic::A64::Jit jit{config};
+ env.jit = &jit;
+ jit.SetRegister(reg, pointer);
+ jit.SetPstate(0xA0000000);
+ env.ticks_left = 1;
+ jit.Step();
+ REQUIRE_FALSE(env.exception.has_value());
+ REQUIRE(jit.GetRegister(reg) == (negative ? address | ~mask : address));
+ REQUIRE(jit.GetPstate() == 0xA0000000);
+}
diff --git a/src/dynarmic/tests/CMakeLists.txt b/src/dynarmic/tests/CMakeLists.txt
index 15a77e4b5c..1b7b22f0b7 100644
--- a/src/dynarmic/tests/CMakeLists.txt
+++ b/src/dynarmic/tests/CMakeLists.txt
@@ -17,6 +17,8 @@ add_executable(dynarmic_tests
decoder_tests.cpp
# A64
A64/a64.cpp
+ A64/pauth_returns.cpp
+ A64/lse.cpp
A64/fibonacci.cpp
A64/fp_min_max.cpp
A64/misaligned_page_table.cpp
diff --git a/src/shader_recompiler/CMakeLists.txt b/src/shader_recompiler/CMakeLists.txt
index cc9ce95d92..39484a1bd5 100644
--- a/src/shader_recompiler/CMakeLists.txt
+++ b/src/shader_recompiler/CMakeLists.txt
@@ -58,6 +58,9 @@ add_library(shader_recompiler STATIC
frontend/ir/type.h
frontend/ir/value.cpp
frontend/ir/value.h
+ frontend/ampere/lower_to_maxwell.h
+ frontend/ampere/program_override.h
+ frontend/ampere/program_address_map.h
frontend/maxwell/control_flow.cpp
frontend/maxwell/control_flow.h
frontend/maxwell/decode.cpp
diff --git a/src/shader_recompiler/environment.h b/src/shader_recompiler/environment.h
index 184b56ebf3..67486430c8 100644
--- a/src/shader_recompiler/environment.h
+++ b/src/shader_recompiler/environment.h
@@ -74,6 +74,8 @@ public:
[[nodiscard]] virtual u64 ReadInstruction(u32 address) = 0;
+ [[nodiscard]] virtual u32 TranslateProgramAddress(u32 address) const { return address; }
+
[[nodiscard]] virtual u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) = 0;
[[nodiscard]] virtual TextureType ReadTextureType(u32 raw_handle) = 0;
diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h
new file mode 100644
index 0000000000..363d1b9b31
--- /dev/null
+++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h
@@ -0,0 +1,1614 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+// Encoding reference: Mesa NAK sm70_encode.rs (MIT), Copyright 2022 Collabora, Ltd.
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <span>
+#include <optional>
+#include <string_view>
+#include <vector>
+#include "common/common_types.h"
+
+namespace Shader::Ampere {
+
+struct Instruction {
+ std::array<u32, 4> words;
+
+ u32 Bits(u32 offset, u32 count) const {
+ u64 bits = words[offset / 32];
+ if (offset / 32 + 1 < words.size()) {
+ bits |= u64{words[offset / 32 + 1]} << 32;
+ }
+ return static_cast<u32>((bits >> (offset % 32)) & ((u64{1} << count) - 1));
+ }
+};
+static_assert(sizeof(Instruction) == 16);
+
+struct LoweredProgram {
+ std::vector<u64> instructions;
+ std::vector<u32> source_instruction_offsets;
+ u32 source_size{};
+ u32 required_texture_buffer = ~0U;
+ u32 error_offset{};
+ std::string_view error;
+
+ explicit operator bool() const { return error.empty(); }
+};
+
+// Lower only operations whose semantics are representable by the existing
+// Maxwell frontend. These are new instructions, not reinterpretations of the
+// native bytes. Scheduling/control words are added by the environment adapter.
+inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
+ LoweredProgram result;
+ struct Relocation { u32 output_index; s64 target; };
+ std::vector<Relocation> relocations;
+ std::vector<u32> barrier_tokens;
+ std::vector<u32> call_targets;
+ std::vector<u32> call_links;
+ bool main_exited = false;
+ bool uses_uniform_bank = false;
+ for (const auto& instruction : code) {
+ const u32 op = instruction.Bits(0, 12);
+ uses_uniform_bank |= op == 0x3c4 || op == 0x2bd || op == 0xc0c || op == 0x986 || op == 0x9a8 || op == 0xc12 || op == 0xc17 || op == 0xe24 || op == 0xc24 || op == 0xd00 || op == 0xd09 || op == 0x9c3 || op == 0xc45 || op == 0x2a4 || op == 0x891 || op == 0x890 || op == 0x290 || op == 0x28c || op == 0x88c || op == 0x287 || op == 0x887 || op == 0x958 || op == 0xab9 || op == 0xabb || op == 0x899 || op == 0xc20 || op == 0x981 || op == 0x526 || op == 0xd08 || op == 0xc21 || op == 0xe21 || op == 0xc23 || op == 0xe23 || op == 0xc02 ||
+ op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || (op == 0x81c && instruction.Bits(67, 1));
+ }
+ const auto address_of = [](u32 index) { return 0x50U + index * 8 + ((index + 1) / 3) * 8; };
+ const auto fail = [&](u32 index, std::string_view reason) {
+ result.instructions.clear();
+ result.source_instruction_offsets.clear();
+ result.error_offset = index * sizeof(Instruction);
+ result.error = reason;
+ return result;
+ };
+ const auto finish = [&](u32 index) {
+ for (const auto& relocation : relocations) {
+ if (relocation.target > index) return fail(index, "native branch or call leaves captured program");
+ const u32 target = result.source_instruction_offsets[static_cast<size_t>(relocation.target)];
+ const s64 relative = s64{address_of(target)} - address_of(relocation.output_index) - 8;
+ if (relative < -0x800000 || relative > 0x7fffff) return fail(index, "lowered branch distance");
+ result.instructions[relocation.output_index] |= (static_cast<u64>(relative) & 0xffffff) << 20;
+ }
+ result.source_size = (index + 1) * sizeof(Instruction);
+ return result;
+ };
+ if (code.empty() || code.size() > 65536) return fail(0, "invalid native program size");
+ for (u32 index = 0; index < code.size(); ++index) {
+ auto insn = code[index];
+ result.source_instruction_offsets.push_back(static_cast<u32>(result.instructions.size()));
+ u32 opcode = insn.Bits(0, 12);
+ std::optional<u32> active_link;
+ u32 active_entry = 0;
+ if (main_exited) {
+ for (size_t call = 0; call < call_targets.size(); ++call) {
+ if (call_targets[call] <= index && call_targets[call] >= active_entry) {
+ active_entry = call_targets[call];
+ active_link = call_links[call];
+ }
+ }
+ if (active_link && insn.Bits(16, 8) == *active_link)
+ return fail(index, "native callee overwrites link register");
+ }
+ // Scratch values and emulated uniform state occupy reserved high GPRs.
+ // Reject native accesses that could alias that state.
+ for (u32 bit : {16U, 24U, 32U, 64U}) {
+ if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66) continue;
+ const u32 reg = insn.Bits(bit, 8);
+ if (reg >= (uses_uniform_bank ? 216U : 250U) && reg < 255) return fail(index, "native program uses lowering scratch registers");
+ }
+ const auto uniform_register = [&](u32 reg) -> u32 {
+ if (reg < 16) return 216 + reg;
+ if (reg == 63) return 255;
+ return ~0U;
+ };
+ if (opcode == 0x2a4 || opcode == 0x891 || opcode == 0x890 || opcode == 0x290) {
+ if (insn.Bits(12, 4) != 7) return fail(index, "native uniform arithmetic predication");
+ const u32 dst = uniform_register(insn.Bits(16, 8));
+ const u32 a = uniform_register(insn.Bits(24, 8));
+ const u32 c = uniform_register(insn.Bits(64, 8));
+ if (dst == ~0U || a == ~0U || c == ~0U) return fail(index, "native uniform arithmetic register");
+ insn.words[0] = (insn.words[0] & 0xffff) | (dst << 16) | (a << 24);
+ insn.words[2] = (insn.words[2] & ~0x080000ffU) | c;
+ if (opcode == 0x2a4 || opcode == 0x290) {
+ const u32 b = uniform_register(insn.Bits(32, 8));
+ if (b == ~0U) return fail(index, "native uniform arithmetic second register");
+ insn.words[1] = (insn.words[1] & ~0xffU) | b;
+ }
+ opcode = opcode == 0x2a4 ? 0x224 : opcode == 0x891 ? 0x811 : opcode == 0x890 ? 0x810 : 0x210;
+ }
+ const bool uniform_predication = opcode == 0xabb || opcode == 0xab9 || opcode == 0xc82 || opcode == 0x882;
+ const u32 native_predicate = insn.Bits(12, 4);
+ const bool guarded_uniform = uniform_predication && native_predicate != 7;
+ if (guarded_uniform && (native_predicate & 7) != 0)
+ return fail(index, "native uniform execution predicate");
+ u64 predicate = u64{guarded_uniform ? ((native_predicate & 8) | 6) : native_predicate} << 16;
+ const u64 dest = insn.Bits(16, 8);
+ const auto base = [&](u64 opcode_bits) { return opcode_bits | predicate | dest; };
+ const auto cbuf = [&]() -> u64 {
+ return (u64{insn.Bits(38, 16) / 4} << 20) | (u64{insn.Bits(54, 5)} << 34);
+ };
+ const auto valid_cbuf = [&] {
+ return !insn.Bits(91, 1) && insn.Bits(54, 5) < 18 &&
+ (insn.Bits(38, 16) % 4) == 0;
+ };
+ std::string_view operand_error;
+ const auto emit = [&](u64 word) { result.instructions.push_back(word); };
+ const auto save_predicate = [&] {
+ emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{0x40} << 20));
+ };
+ const auto restore_predicate = [&] {
+ emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{0x40} << 20));
+ };
+ if (guarded_uniform) {
+ save_predicate();
+ emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{0x40} << 20));
+ }
+ const auto mov_imm = [&](u32 reg, u32 value) {
+ emit(0x0100000000000000ULL | predicate | reg | (u64{15} << 12) | (u64{value} << 20));
+ };
+ const auto materialize = [&](u32 kind, u32 reg, u32 scratch, bool absolute, bool negate) -> u32 {
+ if (kind == 3) {
+ if (reg < 16) reg += 216;
+ else if (reg == 63) reg = 255;
+ else { operand_error = "native uniform source register"; return 255; }
+ } else if (kind == 1) {
+ mov_imm(scratch, insn.words[1]);
+ reg = scratch;
+ } else if (kind == 2) {
+ if (!valid_cbuf()) { operand_error = "native constant buffer address"; return 255; }
+ emit(0x4c98000000000000ULL | predicate | scratch | (u64{15} << 39) | cbuf());
+ reg = scratch;
+ }
+ if (absolute || negate) {
+ if (reg != scratch) emit(0x5c98000000000000ULL | predicate | scratch | (u64{reg} << 20) | (u64{15} << 39));
+ if (absolute) emit(0x0400000000000000ULL | predicate | scratch | (u64{scratch} << 8) | (u64{0x7fffffff} << 20));
+ if (negate) emit(0x0400000000000000ULL | predicate | scratch | (u64{scratch} << 8) | (u64{0x80000000} << 20) | (u64{2} << 53));
+ reg = scratch;
+ }
+ return reg;
+ };
+ const auto operand = [&](u32 which, bool floating = true) -> u32 {
+ const u32 form = insn.Bits(9, 3);
+ u32 reg{}, kind{}, abs{}, neg{};
+ if (which == 0) {
+ reg = insn.Bits(24, 8); abs = insn.Bits(73, 1); neg = insn.Bits(72, 1);
+ } else {
+ if (form != 1 && form != 2 && form != 3 && form != 4 && form != 5 && form != 6 && form != 7) {
+ operand_error = "native ALU source form"; return 255;
+ }
+ const bool wide = which == 1 ? (form == 4 || form == 5 || form == 6) : (form == 2 || form == 3 || form == 7);
+ if (wide) {
+ kind = form == 6 || form == 7 ? 3 : (form == 4 || form == 2) ? 1 : 2;
+ if (kind == 3) reg = insn.Bits(32, 8);
+ if (kind == 2 || kind == 3) { abs = insn.Bits(62, 1); neg = insn.Bits(63, 1); }
+ } else {
+ const bool upper = which == 1 ? (form == 2 || form == 3 || form == 7) : true;
+ reg = insn.Bits(upper ? 64 : 32, 8);
+ abs = insn.Bits(upper ? 74 : 62, 1); neg = insn.Bits(upper ? 75 : 63, 1);
+ }
+ }
+ return materialize(kind, reg, 250 + which, floating && abs, floating && neg);
+ };
+ struct HalfSource { u32 reg, swizzle; bool absolute, negate; };
+ const auto half_source = [&](u32 which) -> HalfSource {
+ if (which == 0) return {insn.Bits(24, 8), insn.Bits(74, 2),
+ insn.Bits(73, 1) != 0, insn.Bits(72, 1) != 0};
+ const u32 form = insn.Bits(9, 3);
+ const bool wide = which == 1 ? (form == 4 || form == 5) : (form == 2 || form == 3);
+ if (wide) {
+ const bool immediate = form == 2 || form == 4;
+ const u32 reg = materialize(immediate ? 1 : 2, 255, 250 + which, false, false);
+ return {reg, immediate ? 0U : insn.Bits(60, 2),
+ !immediate && insn.Bits(62, 1), !immediate && insn.Bits(63, 1)};
+ }
+ const bool upper = which == 2 || form == 2 || form == 3;
+ return {insn.Bits(upper ? 64 : 32, 8), insn.Bits(upper ? 81 : 60, 2),
+ insn.Bits(upper ? 83 : 62, 1) != 0, insn.Bits(upper ? 84 : 63, 1) != 0};
+ };
+ const auto extract_half = [&](HalfSource source, u32 lane, u32 target) {
+ const bool high = source.swizzle == 3 || (source.swizzle == 0 && lane == 1);
+ emit(0x5ca8000000000000ULL | predicate | target | (u64{2} << 8) |
+ (u64{source.swizzle == 1 ? 2U : 1U} << 10) | (u64{source.reg} << 20) |
+ (u64{high} << 41) | (u64{source.negate} << 45) | (u64{source.absolute} << 49));
+ };
+ const auto xmad = [&](u32 target, u32 a, u32 b, u32 c, bool a_high,
+ bool b_high, u32 mode, bool immediate) {
+ emit((immediate ? 0x3600000000000000ULL : 0x5b00000000000000ULL) |
+ predicate | target | (u64{a} << 8) |
+ (u64{immediate ? ((b >> (b_high ? 16 : 0)) & 0xffff) : b} << 20) |
+ (u64{!immediate && b_high} << 35) | (u64{c} << 39) |
+ (u64{mode} << 50) | (u64{a_high} << 53));
+ };
+ const auto multiply_low = [&](u32 a, u32 b, u32 c, bool immediate) {
+ xmad(250, a, b, c, false, false, 0, immediate);
+ xmad(251, a, b, 255, true, false, 0, immediate);
+ xmad(251, a, b, 251, false, true, 0, immediate);
+ emit(base(0x5c18000000000000ULL) | (u64{251} << 8) |
+ (u64{250} << 20) | (u64{16} << 39));
+ };
+ switch (opcode) {
+ case 0x304: { // F2F.F16.F32, scalar round-to-nearest conversion.
+ if (insn.words[2] != 0x00200800 || insn.Bits(24, 8) ||
+ (insn.words[1] & ~0xc00000ffU))
+ return fail(index, "native scalar float conversion mode");
+ emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(62, 1)} << 49) |
+ (u64{insn.Bits(63, 1)} << 45));
+ break;
+ }
+ case 0x23e: case 0xa3e: { // F2FP.PACK_AB / MERGE_C, round-to-nearest
+ if (insn.Bits(62, 2) || insn.Bits(73, 5) || insn.Bits(79, 2) ||
+ insn.Bits(81, 10) || insn.Bits(91, 1)) {
+ return fail(index, "native float packing modifiers");
+ }
+ const bool merge = insn.Bits(78, 1);
+ if (insn.Bits(72, 1)) return fail(index, "native float packing selector");
+ const u32 a = merge ? operand(2, false) : operand(0, false);
+ const u32 b = operand(1, false);
+ // Canonical F2F writes the converted half in its low 16 bits.
+ if (merge) {
+ emit(0x5c98000000000000ULL | predicate | 250 | (u64{a} << 20) | (u64{15} << 39));
+ } else {
+ emit(0x5ca8000000000000ULL | predicate | 250 | (u64{1} << 8) |
+ (u64{2} << 10) | (u64{a} << 20));
+ }
+ emit(0x5ca8000000000000ULL | predicate | 251 | (u64{1} << 8) |
+ (u64{2} << 10) | (u64{b} << 20));
+ emit(base(0x5c18000000000000ULL) | (u64{250} << 8) | (u64{251} << 20) | (u64{16} << 39));
+ break;
+ }
+ case 0x232: case 0x832: case 0xa32: { // HMUL2
+ if (insn.Bits(78, 2) || insn.Bits(81, 10) || insn.Bits(91, 1))
+ return fail(index, "native half multiply mode");
+ const auto a = half_source(0), b = half_source(1);
+ const u32 precision = insn.Bits(76, 1) ? 2 : insn.Bits(80, 1);
+ emit(base(0x5d08000000000000ULL) | (u64{a.reg} << 8) | (u64{b.reg} << 20) |
+ (u64{b.swizzle} << 28) | (u64{b.absolute} << 30) | (u64{a.negate != b.negate} << 31) |
+ (u64{insn.Bits(77, 1)} << 32) | (u64{precision} << 39) |
+ (u64{a.absolute} << 44) | (u64{a.swizzle} << 47));
+ break;
+ }
+ case 0x230: case 0x430: case 0x630: { // HADD2
+ if (insn.Bits(76, 1) || insn.Bits(79, 1) || insn.Bits(81, 10) || insn.Bits(91, 1))
+ return fail(index, "native half add mode");
+ const auto a = half_source(0);
+ const auto b = half_source(opcode == 0x230 ? 1 : 2);
+ if (insn.Bits(78, 1)) {
+ if (a.reg != 255 || !a.negate || a.absolute || a.swizzle != 0 ||
+ b.swizzle < 2 || insn.Bits(77, 1) || insn.Bits(80, 1))
+ return fail(index, "native float-output half add mode");
+ extract_half(b, 0, static_cast<u32>(dest));
+ } else {
+ emit(base(0x5d10000000000000ULL) | (u64{a.reg} << 8) | (u64{b.reg} << 20) |
+ (u64{b.swizzle} << 28) | (u64{b.absolute} << 30) | (u64{b.negate} << 31) |
+ (u64{insn.Bits(77, 1)} << 32) | (u64{insn.Bits(80, 1)} << 39) |
+ (u64{a.negate} << 43) | (u64{a.absolute} << 44) | (u64{a.swizzle} << 47));
+ }
+ break;
+ }
+ case 0x231: case 0x431: case 0x631: case 0x831: case 0xa31: { // HFMA2
+ const bool relu = insn.Bits(79, 1) != 0;
+ if (insn.Bits(78, 1) || insn.Bits(85, 1) || insn.Bits(91, 1) ||
+ (relu ? (insn.Bits(87, 4) != 7 ||
+ insn.Bits(77, 1)) : insn.Bits(87, 4) != 0))
+ return fail(index, "native half fused multiply-add mode");
+ if (relu && native_predicate != 7) {
+ // Guard the whole expansion before its temporary P6 updates.
+ relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1});
+ emit(0xe24000000000000fULL | (u64{native_predicate ^ 8U} << 16));
+ predicate = u64{7} << 16;
+ }
+ auto a = half_source(0);
+ auto b = half_source(1);
+ const auto c = half_source(2);
+ if (insn.Bits(86, 1)) { // H0_NH1: negate only the upper packed component.
+ if (b.swizzle != 0 || b.absolute || b.reg >= 250)
+ return fail(index, "native half mixed-sign source mode");
+ emit(0x0400000000000000ULL | predicate | 251 | (u64{b.reg} << 8) |
+ (u64{0x80000000} << 20) | (u64{2} << 53));
+ b.reg = 251;
+ }
+ if (b.absolute || c.absolute || (a.absolute && a.reg >= 248))
+ return fail(index, "native half fused multiply-add absolute source");
+ if (a.absolute) {
+ emit(0x0400000000000000ULL | predicate | 250 | (u64{a.reg} << 8) |
+ (u64{a.swizzle == 1 ? 0x7fffffffU : 0x7fff7fffU} << 20));
+ a.reg = 250;
+ a.absolute = false;
+ }
+ const u32 precision = insn.Bits(76, 1) ? 2 : insn.Bits(80, 1);
+ emit(0x5d00000000000000ULL | predicate | (relu ? 254U : dest) | (u64{a.reg} << 8) | (u64{b.reg} << 20) |
+ (u64{b.swizzle} << 28) | (u64{c.negate} << 30) | (u64{a.negate != b.negate} << 31) |
+ (u64{insn.Bits(77, 1)} << 32) | (u64{c.swizzle} << 35) |
+ (u64{precision} << 37) | (u64{c.reg} << 39) | (u64{a.swizzle} << 47));
+ if (relu) {
+ // Clamp the rounded binary16 lanes, preserving positive infinity
+ // and canonicalizing either sign of NaN without saturating to one.
+ save_predicate();
+ for (u32 lane = 0; lane < 2; ++lane) {
+ emit(0x3828000000070000ULL | 251 | (u64{254} << 8) | (u64{lane * 16} << 20));
+ emit(0x0400000000070000ULL | 251 | (u64{251} << 8) | (u64{0xffff} << 20));
+ emit(0x0400000000070000ULL | 252 | (u64{251} << 8) | (u64{0x7fff} << 20));
+ mov_imm(250, 0x7c00);
+ emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{250} << 8) |
+ (u64{252} << 20) | (u64{7} << 39) | (u64{1} << 49));
+ emit(0x3828000000070000ULL | 252 | (u64{251} << 8) | (u64{15} << 20));
+ mov_imm(250, 1);
+ emit(0x5c10000000070000ULL | 252 | (u64{252} << 8) | (u64{250} << 20) | (u64{1} << 48));
+ emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48));
+ mov_imm(250, 0x7fff);
+ emit(0x5ca0000000070000ULL | 251 | (u64{250} << 8) | (u64{251} << 20) | (u64{6} << 39));
+ if (lane == 0) {
+ emit(base(0x5c98000000000000ULL) | (u64{251} << 20) | (u64{15} << 39));
+ } else {
+ emit(0x3848000000070000ULL | 251 | (u64{251} << 8) | (u64{16} << 20));
+ emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{251} << 20) |
+ (u64{1} << 41) | (u64{7} << 48));
+ }
+ }
+ restore_predicate();
+ }
+ break;
+ }
+ case 0x240: case 0x840: { // HMNMX2, exact half promotion for min/max
+ if (insn.Bits(76, 11) || insn.Bits(91, 1))
+ return fail(index, "native half min/max mode");
+ const auto a = half_source(0);
+ const auto b = opcode == 0x840 ? HalfSource{255, 0, false, false} : half_source(1);
+ if (a.swizzle == 1 || b.swizzle == 1 || b.reg >= 250 && b.reg != 255)
+ return fail(index, "native half min/max source mode");
+ for (u32 lane = 0; lane < 2; ++lane) {
+ extract_half(a, lane, 250);
+ if (opcode == 0x840) {
+ const u32 half = (insn.words[1] >> (lane * 16)) & 0xffff;
+ mov_imm(251, half);
+ extract_half(HalfSource{251, 2, false, false}, 0, 251);
+ } else {
+ extract_half(b, lane, 251);
+ }
+ const u32 target = lane == 0 ? 252 : 250;
+ emit(0x5c60000000000000ULL | predicate | target | (u64{250} << 8) | (u64{251} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ emit(0x5ca8000000000000ULL | predicate | target | (u64{1} << 8) |
+ (u64{2} << 10) | (u64{target} << 20));
+ }
+ emit(base(0x5c18000000000000ULL) | (u64{250} << 8) | (u64{252} << 20) | (u64{16} << 39));
+ break;
+ }
+ case 0xc24: case 0xe24: { // IMAD low word with a uniform multiplier or addend.
+ if ((insn.words[2] & ~0x080002ffU) != 0x078e0000 || (insn.words[1] & ~0xffU))
+ return fail(index, "native uniform-addend multiply mode");
+ const u32 a = operand(0, false), b = operand(1, false), c = operand(2, false);
+ multiply_low(a, b, c, false);
+ break;
+ }
+ case 0x224: { // IMAD register / zero-product move
+ if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 && insn.Bits(75, 1) &&
+ (insn.words[2] & ~0xaffU) == 0x078e0000) {
+ emit(base(0x5c10000000000000ULL) | (u64{255} << 8) |
+ (u64{insn.Bits(64, 8)} << 20) | (u64{1} << 48));
+ break;
+ }
+ if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 &&
+ (insn.words[2] & ~0xffU) == 0x078e0000) {
+ emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.Bits(64, 8)} << 20));
+ break;
+ }
+ if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (insn.words[1] & ~0xffU))
+ return fail(index, "native integer register multiply mode");
+ multiply_low(insn.Bits(24, 8), insn.Bits(32, 8), insn.Bits(64, 8), false);
+ break;
+ }
+ case 0x827: { // IMAD.HI.U32, captured zero addend
+ if (insn.words[2] != 0x078e00ff) return fail(index, "native high multiply mode");
+ const u32 a = insn.Bits(24, 8), b = insn.words[1];
+ // Base-2^16 long multiplication. Each intermediate sum fits U32.
+ xmad(250, a, b, 255, false, false, 0, true);
+ xmad(251, a, b, 250, true, false, 2, true);
+ xmad(250, a, b, 251, false, true, 1, true);
+ xmad(252, a, b, 251, true, true, 2, true);
+ emit(0x3828000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{16} << 20));
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20));
+ break;
+ }
+ case 0xa24: { // IMAD, constant-buffer multiplier
+ if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || !valid_cbuf() || insn.Bits(62, 2))
+ return fail(index, "native integer constant-buffer multiply mode");
+ emit(0x4c98000000000000ULL | predicate | 252 | (u64{15} << 39) | cbuf());
+ multiply_low(insn.Bits(24, 8), 252, insn.Bits(64, 8), false);
+ break;
+ }
+ case 0x424: case 0x624: { // IMAD.MOV.U32: zero product plus immediate/cbuf
+ if (opcode == 0x624 && insn.Bits(24, 8) == 255 &&
+ (insn.words[2] & ~0x07800000U) == 0x000e06ffU) {
+ const u32 source = operand(2, false);
+ mov_imm(250, 1);
+ emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ emit(base(0x5c10000000000000ULL) | (u64{source} << 8) | (u64{250} << 20));
+ break;
+ }
+ if (insn.Bits(24, 8) != 255 || insn.Bits(64, 8) != 255 || insn.words[2] != 0x078e00ff) {
+ if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (opcode == 0x624 && insn.Bits(62, 2)))
+ return fail(index, "native multiply wide addend mode");
+ const u32 a = operand(0, false), b = operand(1, false), c = operand(2, false);
+ multiply_low(a, b, c, false);
+ break;
+ }
+ if (insn.Bits(24, 8) != 255 || insn.Bits(64, 8) != 255 ||
+ insn.words[2] != 0x078e00ff)
+ return fail(index, "native integer move mode");
+ if (opcode == 0x624) {
+ if (!valid_cbuf() || insn.Bits(62, 2))
+ return fail(index, "native integer move constant buffer");
+ emit(base(0x4c98000000000000ULL) | (u64{15} << 39) | cbuf());
+ } else {
+ emit(base(0x0100000000000000ULL) | (u64{15} << 12) |
+ (u64{insn.words[1]} << 20));
+ }
+ break;
+ }
+ case 0x824: { // Captured low-word IMAD forms
+ if (insn.words[1] == 1 && (insn.words[2] & 0xffffff00U) == 0x078e0200) {
+ emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(64, 8)} << 20));
+ break;
+ }
+ if (insn.words[1] != 48 || insn.words[2] != 0x078e02ff) {
+ if ((insn.words[2] & ~0x2ffU) != 0x078e0000)
+ return fail(index, "native integer multiply mode");
+ multiply_low(insn.Bits(24, 8), insn.words[1], insn.Bits(64, 8), true);
+ break;
+ }
+ const u64 source = insn.Bits(24, 8);
+ emit(0x5c18000000000000ULL | predicate | 250 | (source << 8) | (source << 20) | (u64{1} << 39));
+ emit(base(0x3848000000000000ULL) | (u64{250} << 8) | (u64{4} << 20));
+ break;
+ }
+ case 0x886: { // VOTEU ballot and independent uniform predicate output.
+ const u32 output = insn.Bits(81, 3), target = insn.Bits(16, 8);
+ if ((target >= 16 && target != 63) || insn.words[1] ||
+ (output > 1 && output != 7) || insn.Bits(72, 2) > 2 || native_predicate != 7)
+ return fail(index, "native uniform vote mode");
+ const u32 temporary = output == 7 ? 7 : 6 - output;
+ const u32 mask = 1U << temporary;
+ if (output != 7) emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20));
+ emit(0x50d8000000070000ULL | uniform_register(target) |
+ (u64{insn.Bits(87, 4)} << 39) | (u64{temporary} << 45) | (u64{insn.Bits(72, 2)} << 48));
+ if (output != 7) {
+ emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20));
+ emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20));
+ }
+ break;
+ }
+ case 0x3c2: { // R2UR: broadcast the first lane satisfying the execution predicate.
+ if (dest >= 16 || insn.words[1] || insn.words[2] != 0x000e0000)
+ return fail(index, "native register-to-uniform mode");
+ save_predicate();
+ emit(0x50d8000000070000ULL | 250 | (u64{native_predicate} << 39) |
+ (u64{6} << 45) | (u64{1} << 48));
+ emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49));
+ emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48));
+ emit(0x5c30000000070000ULL | 250 | (u64{250} << 20));
+ emit(0xef10000000070000ULL | 251 | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
+ emit(0x5ca0000000070000ULL | (dest + 216) | (u64{251} << 8) |
+ (u64{dest + 216} << 20) | (u64{6} << 39));
+ restore_predicate();
+ break;
+ }
+ case 0x81c: { // Observed predicate LUT constants and UP0-to-predicate copy.
+ const u32 target = insn.Bits(81, 3);
+ const u32 lut = insn.Bits(64, 3) | (insn.Bits(72, 5) << 3);
+ if ((native_predicate != 7 && insn.Bits(67, 1)) || target == 7 || insn.Bits(84, 3) != 7 ||
+ insn.Bits(16, 8) || insn.words[1])
+ return fail(index, "native predicate LUT mode");
+ if (!insn.Bits(67, 1) && (lut == 0xfe || lut == 0xa8)) {
+ emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 |
+ (u64{insn.Bits(87, 4)} << 12) | (u64{1} << 24) |
+ (u64{insn.Bits(77, 4)} << 29) | (u64{insn.Bits(68, 4)} << 39) |
+ (u64{lut == 0xfe} << 45));
+ break;
+ }
+ if (!insn.Bits(67, 1) && (lut == 0x80 ||
+ (lut == 8 && insn.Bits(68, 4) == 7 && insn.Bits(77, 4) == 7))) {
+ emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 |
+ (u64{insn.Bits(87, 4) ^ (lut == 8 ? 8U : 0U)} << 12) | (u64{insn.Bits(77, 4)} << 29) |
+ (u64{insn.Bits(68, 4)} << 39));
+ break;
+ }
+ if (insn.Bits(77, 4) != 7 || insn.Bits(87, 4) != 7)
+ return fail(index, "native predicate LUT source mode");
+ if (insn.Bits(67, 1) && insn.Bits(68, 4) <= 1 && lut == 0x80) {
+ emit(0x3828000000070000ULL | 250 | (u64{239} << 8) | (u64{6 - insn.Bits(68, 4)} << 20));
+ emit(0x3848000000070000ULL | 250 | (u64{250} << 8) | (u64{target} << 20));
+ } else if (!insn.Bits(67, 1) && insn.Bits(68, 4) == 7 && (lut == 0x80 || lut == 8)) {
+ emit(0x010000000007f000ULL | 250 | ((lut == 0x80 ? u64{1} << target : 0ULL) << 20));
+ } else return fail(index, "native predicate LUT truth table");
+ emit(0x38f0000000070000ULL | (u64{250} << 8) | (u64{1} << (target + 20)));
+ break;
+ }
+ case 0x882: { // UMOV immediate.
+ if (dest >= 16 || insn.words[2]) return fail(index, "native uniform immediate move mode");
+ mov_imm(static_cast<u32>(dest) + 216, insn.words[1]);
+ break;
+ }
+ case 0xc82: { // UMOV UR, UR, including uniform execution predicates.
+ const u32 source = uniform_register(insn.words[1]);
+ if (dest >= 16 || source == ~0U || insn.words[2] != 0x08000000)
+ return fail(index, "native uniform move mode");
+ emit(0x5c98000000000000ULL | predicate | (dest + 216) |
+ (u64{source} << 20) | (u64{15} << 39));
+ break;
+ }
+ case 0xab9: case 0xabb: { // ULDC U32/U64, direct or indexed.
+ const u32 uniform_dest = insn.Bits(16, 8), index_reg = insn.Bits(24, 8);
+ const u32 width = insn.Bits(73, 3);
+ if (uniform_dest >= 16 || (width != 4 && width != 5) ||
+ (width == 5 && ((uniform_dest & 1) || uniform_dest >= 15)) ||
+ insn.Bits(54, 5) >= 18 || insn.Bits(38, 16) % 4 || insn.Bits(62, 2) ||
+ (insn.words[2] & ~0xe00U) != (opcode == 0xab9 ? 0U : 0x08000000U) ||
+ (opcode == 0xabb && index_reg >= 16))
+ return fail(index, "native uniform constant load mode");
+ if (opcode == 0xab9 && width == 4) {
+ emit(0x4c98000000000000ULL | predicate | (uniform_dest + 216) | (u64{15} << 39) | cbuf());
+ } else {
+ emit(0xef90000000000000ULL | predicate | (uniform_dest + 216) |
+ (u64{opcode == 0xab9 ? 255U : index_reg + 216} << 8) | (u64{insn.Bits(38, 16)} << 20) |
+ (u64{insn.Bits(54, 5)} << 36) | (u64{width} << 48));
+ }
+ break;
+ }
+ case 0x899: { // USHF.L.U32; accept a zero word outside the selected half.
+ const u32 dst = insn.Bits(16, 8), low = insn.Bits(24, 8), high = insn.Bits(64, 8);
+ if (insn.words[2] == (0x08011400U | high) && low == 63 && high < 16 &&
+ dst < 16 && native_predicate == 7 && insn.words[1] <= 32) {
+ emit(0x3828000000000000ULL | predicate | (dst + 216) |
+ (u64{high + 216} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48));
+ break;
+ }
+ const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high < 16;
+ const bool low_result = insn.words[2] == 0x0800063f && low < 16;
+ if (dst >= 16 || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result))
+ return fail(index, "native uniform shift mode");
+ emit(0x3848000000000000ULL | predicate | (dst + 216) |
+ (u64{(high_result ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20));
+ break;
+ }
+ case 0xb82: { // LDC.64 indexed byte offset
+ const u32 width = insn.Bits(73, 3);
+ if ((width != 4 && width != 5) || insn.Bits(78, 2) || insn.Bits(91, 1) ||
+ insn.Bits(54, 5) >= 18 ||
+ (width == 5 && ((dest & 1) || dest >= 254))) return fail(index, "native indexed constant load mode");
+ emit(base(0xef90000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(38, 16)} << 20) | (u64{insn.Bits(54, 5)} << 36) | (u64{width} << 48));
+ break;
+ }
+ case 0xf89: { // Quad-local SHFL.BFLY with immediate lane and segmentation mask.
+ const u32 lane = insn.Bits(53, 5), mask = insn.Bits(40, 13);
+ if (insn.words[2] != 0x000e0000 || insn.Bits(32, 8) ||
+ insn.Bits(58, 6) != 3 || mask != 0x1c03 || (lane != 1 && lane != 2))
+ return fail(index, "native quad butterfly mode");
+ emit(base(0xef10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{lane} << 20) | (u64{1} << 28) | (u64{1} << 29) |
+ (u64{3} << 30) | (u64{mask} << 34) | (u64{7} << 48));
+ break;
+ }
+ case 0x822: { // FSWZADD, bounded quad derivative sign patterns.
+ const u32 swizzle = insn.words[1];
+ if ((swizzle != 0x66 && swizzle != 0x5a) || insn.words[2] > 255)
+ return fail(index, "native swizzled add mode");
+ // Maxwell encodes operand signs rather than the native operation enum.
+ emit(base(0x50f8000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(64, 8)} << 20) | (u64{swizzle == 0x66 ? 0x99U : 0xa5U} << 28));
+ break;
+ }
+ case 0xb6c: { // TXD ARRAY_2D, four channels and consecutive split destinations.
+ const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14);
+ const u32 coordinates = insn.Bits(24, 8), derivatives = insn.Bits(32, 8);
+ if ((insn.words[2] & ~0xffU) != 0x001e0f00 || insn.Bits(59, 2) ||
+ insn.Bits(61, 3) != 5 || binding >= 18 || offset > 0x1fff ||
+ (dest & 1) || dest >= 248 || insn.Bits(64, 8) != dest + 2 ||
+ coordinates >= 248 || derivatives >= 247)
+ return fail(index, "native array gradient texture mode");
+ if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
+ return fail(index, "native texture descriptor buffer conflict");
+ result.required_texture_buffer = binding;
+ emit(base(0xde00000000000000ULL) | (u64{coordinates} << 8) |
+ (u64{derivatives} << 20) | (u64{3} << 28) | (u64{15} << 31) |
+ (u64{offset} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51));
+ break;
+ }
+ case 0x589: { // SHFL.IDX, register lane and immediate segmentation mask.
+ if (insn.words[2] != 0x000e0000 || insn.Bits(53, 11))
+ return fail(index, "native shuffle mode");
+ emit(0xef10000000000000ULL | predicate | dest |
+ (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) |
+ (u64{1} << 29) | (u64{insn.Bits(40, 13)} << 34) | (u64{7} << 48));
+ break;
+ }
+ case 0x98e: { // RED.E U32, direct register address; ADD through XOR.
+ if ((insn.words[2] & ~0x03800000U) != 0x0010e100 ||
+ insn.Bits(40, 24) || (insn.Bits(24, 8) & 1))
+ return fail(index, "native global reduction mode");
+ emit(0xebf8000000000000ULL | predicate | insn.Bits(32, 8) |
+ (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(87, 3)} << 23) | (u64{1} << 48));
+ break;
+ }
+ case 0x2bd: // UFLO.U32, return the highest set-bit index.
+ if (dest >= 16 || insn.words[1] >= 16 || insn.words[2] != 0x080e0000 || native_predicate != 7)
+ return fail(index, "native uniform leading bit mode");
+ emit(0x5c30000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 20));
+ break;
+ case 0x3a1: { // MATCH.ALL.U64 with only the all-equal predicate retained.
+ const u32 source = insn.Bits(24, 8), target = insn.Bits(81, 3);
+ if (dest != 255 || source >= 254 || (source & 1) || target >= 6 ||
+ native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0000U) != 0x200)
+ return fail(index, "native warp match mode");
+ save_predicate();
+ emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48));
+ emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49));
+ emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48));
+ emit(0x5c30000000070000ULL | 250 | (u64{250} << 20));
+ for (u32 word = 0; word < 2; ++word)
+ emit(0xef10000000070000ULL | (251 + word) | (u64{source + word} << 8) |
+ (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
+ emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source} << 8) |
+ (u64{251} << 20) | (u64{7} << 39) | (u64{2} << 49));
+ emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source + 1} << 8) |
+ (u64{252} << 20) | (u64{6} << 39) | (u64{2} << 49));
+ emit(0x50d8000000070000ULL | 255 | (u64{6} << 39) | (u64{target} << 45));
+ restore_predicate();
+ break;
+ }
+ case 0x3c4: { // REDUX.SUM over the actual active lanes, modulo 2^32.
+ const u32 source = insn.Bits(24, 8);
+ if (dest >= 16 || insn.words[1] || insn.words[2] != 0xc000 || native_predicate != 7)
+ return fail(index, "native warp sum mode");
+ emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48));
+ mov_imm(static_cast<u32>(dest) + 216, 0);
+ for (u32 lane = 0; lane < 32; ++lane) {
+ emit(0xef10000000070000ULL | 251 | (u64{source} << 8) | (u64{lane} << 20) |
+ (u64{1} << 28) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
+ emit(0x3828000000070000ULL | 252 | (u64{250} << 8) | (u64{lane} << 20));
+ emit(0x0400000000070000ULL | 252 | (u64{252} << 8) | (u64{1} << 20));
+ emit(0x5c10000000070000ULL | 252 | (u64{255} << 8) | (u64{252} << 20) | (u64{1} << 48));
+ emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48));
+ emit(0x5c10000000070000ULL | (dest + 216) | (u64{dest + 216} << 8) | (u64{251} << 20));
+ }
+ break;
+ }
+ case 0xf8c: { // ATOMS.POPC.INC.32 with no returned value: one increment per active lane.
+ const u32 stride = insn.Bits(78, 2);
+ if (dest != 255 || insn.words[1] || (insn.words[2] & ~0xc000U) != 0x0d00003f)
+ return fail(index, "native shared increment mode");
+ u32 address = insn.Bits(24, 8);
+ if (stride && address != 255) {
+ emit(0x3848000000000000ULL | predicate | 251 | (u64{address} << 8) |
+ (u64{stride + 1} << 20));
+ address = 251;
+ }
+ mov_imm(250, 1);
+ emit(0xec00000000000000ULL | predicate | 255 | (u64{address} << 8) | (u64{250} << 20));
+ break;
+ }
+ case 0x38c: { // ATOMS.ADD.U32, naturally aligned shared byte offset.
+ const u32 offset = insn.Bits(40, 24);
+ if (insn.words[2] || (offset & 3)) return fail(index, "native shared atomic mode");
+ emit(0xec00000000000000ULL | predicate | dest |
+ (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) |
+ (u64{offset >> 2} << 30));
+ break;
+ }
+ case 0x9a8: { // ATOMG.E.ADD.STRONG.GPU U32 through a uniform pointer.
+ const u32 address = uniform_register(insn.Bits(64, 6));
+ if (insn.Bits(24, 8) != 255 || insn.Bits(40, 24) ||
+ (insn.words[2] & ~0x3fU) != 0x081ee140 || address == ~0U || (address & 1))
+ return fail(index, "native global atomic mode");
+ emit(0xed00000000000000ULL | predicate | dest | (u64{address} << 8) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{1} << 48));
+ break;
+ }
+ case 0xb97: { // SULD.P.2D.R.IGN, direct descriptor and red channel only.
+ const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14);
+ const u32 coordinate = insn.Bits(24, 8);
+ if (insn.words[2] != 0x001e0100 || insn.Bits(61, 3) != 3 ||
+ binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || dest == 255 || coordinate > 253)
+ return fail(index, "native formatted surface load mode");
+ if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
+ return fail(index, "native surface descriptor buffer conflict");
+ result.required_texture_buffer = binding;
+ emit(base(0xeb00000000000000ULL) | (u64{coordinate} << 8) |
+ (u64{1} << 20) | (u64{3} << 33) | (u64{offset} << 36) | (u64{1} << 51));
+ break;
+ }
+ case 0xb9b: { // SUST.P.2D/3D.IGN, direct descriptor and all four channels.
+ const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14);
+ const u32 data = insn.Bits(32, 8), coordinate = insn.Bits(24, 8);
+ const u32 dimension = insn.Bits(61, 3);
+ if (insn.words[2] != 0x00100f00 || (dimension != 3 && dimension != 5) ||
+ binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || data > 252 || coordinate > (dimension == 5 ? 252U : 253U))
+ return fail(index, "native formatted surface store mode");
+ if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
+ return fail(index, "native surface descriptor buffer conflict");
+ result.required_texture_buffer = binding;
+ emit(0xeb20000000000000ULL | predicate | data | (u64{coordinate} << 8) |
+ (u64{15} << 20) | (u64{dimension} << 33) | (u64{offset} << 36) | (u64{1} << 51));
+ break;
+ }
+ case 0x301: // BREV register.
+ if (insn.words[2] || insn.words[1] > 255) return fail(index, "native bit reverse mode");
+ emit(base(0x3828000000000000ULL) | (u64{insn.words[1]} << 8) |
+ (u64{1} << 39) | (u64{1} << 40));
+ break;
+ case 0x219: { // U32 register shifts with a zero word outside the selected result.
+ const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8), shift = insn.Bits(32, 8);
+ const bool right = insn.Bits(76, 1) != 0, high_result = insn.Bits(80, 1) != 0;
+ if (insn.words[1] > 255 || insn.Bits(73, 2) != 3 ||
+ (insn.words[2] & ~0x000118ffU) != 0x00000600U ||
+ (right ? (!high_result || low != 255) : (high_result || high != 255)))
+ return fail(index, "native register shift mode");
+ emit(base(right ? 0x5cf8000000000000ULL : 0x5bf8000000000000ULL) |
+ (u64{right ? high : 255U} << 8) | (u64{shift} << 20) |
+ (u64{right ? 255U : low} << 39) | (u64{insn.Bits(75, 1)} << 50));
+ break;
+ }
+ case 0x300: case 0xd00: { // FLO, including a uniform source.
+ if ((insn.words[2] & ~(0x600U | (opcode == 0xd00 ? 0x08000000U : 0U))) != 0x000e0000U)
+ return fail(index, "native leading bit mode");
+ const u32 source = operand(1, false);
+ emit(base(0x5c30000000000000ULL) | (u64{source} << 20) |
+ (u64{insn.Bits(63, 1)} << 40) | (u64{insn.Bits(74, 1)} << 41) |
+ (u64{insn.Bits(73, 1)} << 48));
+ break;
+ }
+ case 0x309: case 0xd09: { // POPC, including a uniform source.
+ if (insn.words[2] != (opcode == 0xd09 ? 0x08000000U : 0U))
+ return fail(index, "native population count mode");
+ const u32 source = operand(1, false);
+ emit(base(0x5c08000000000000ULL) | (u64{source} << 20) | (u64{insn.Bits(63, 1)} << 40));
+ break;
+ }
+ case 0x918:
+ if (insn.words[1] || insn.words[2]) return fail(index, "native NOP mode");
+ emit(0x50b0000000000f00ULL | predicate);
+ break;
+ case 0x992: { // MEMBAR.SC: preserve workgroup/device visibility.
+ const u32 scope = insn.Bits(76, 3);
+ if (insn.words[1] || insn.words[2] != (scope << 12) ||
+ (scope != 0 && scope != 2 && scope != 3 && scope != 5))
+ return fail(index, "native memory barrier scope");
+ const u32 legacy_scope = scope == 0 ? 0 : scope == 2 ? 1 : scope == 3 ? 2 : 3;
+ emit(0xef98000000000000ULL | predicate | (u64{legacy_scope} << 8));
+ break;
+ }
+ case 0x9ab: // ERRBAR: wait for prior memory operations on the host backend.
+ if (insn.words[1] || insn.words[2]) return fail(index, "native error barrier mode");
+ emit(0xef98000000000000ULL | predicate | (u64{1} << 8));
+ break;
+ case 0xb1d: // BAR.SYNC 0, full workgroup.
+ if (insn.words[1] || insn.words[2] || native_predicate != 7)
+ return fail(index, "native workgroup barrier mode");
+ emit(0xf0a8000000000000ULL | predicate | (u64{1} << 43) |
+ (u64{1} << 44) | (u64{1} << 39) | (u64{7} << 39));
+ break;
+ case 0x388: case 0x984: { // Shared memory load/store with scaled register offsets.
+ const u32 width = insn.Bits(73, 3), stride = insn.Bits(78, 2);
+ if (width > 6 || insn.words[2] != ((width << 9) | (stride << 14)) ||
+ (opcode == 0x984 && insn.Bits(32, 8)))
+ return fail(index, "native shared memory addressing mode");
+ u32 address = insn.Bits(24, 8);
+ if (stride && address != 255) {
+ emit(0x3848000000000000ULL | predicate | 250 |
+ (u64{address} << 8) | (u64{stride + 1} << 20));
+ address = 250;
+ }
+ const u32 data = opcode == 0x388 ? insn.Bits(32, 8) : static_cast<u32>(dest);
+ emit((opcode == 0x388 ? 0xef58000000000000ULL : 0xef48000000000000ULL) |
+ predicate | data | (u64{address} << 8) | (u64{insn.Bits(40, 24)} << 20) |
+ (u64{width} << 48));
+ break;
+ }
+ case 0x919: case 0x9c3: { // S2R / S2UR thread and workgroup identities.
+ const u32 special = insn.Bits(72, 8);
+ if (insn.words[1] || insn.words[2] != (special << 8) ||
+ (special != 0 && special != 33 && special != 34 && special != 35 &&
+ special != 37 && special != 38 && special != 39 &&
+ special != 56 && special != 57 && special != 58 && special != 59 && special != 60))
+ return fail(index, "native special register");
+ const u32 target = opcode == 0x9c3 ? uniform_register(static_cast<u32>(dest)) : static_cast<u32>(dest);
+ if (target == ~0U) return fail(index, "native uniform special destination");
+ emit(0xf0c8000000000000ULL | predicate | target | (u64{special} << 20));
+ break;
+ }
+ case 0x28c: case 0x88c: { // UISETP U32/I32 with UPT accumulation.
+ const u32 output = insn.Bits(81, 3);
+ const u32 a = uniform_register(insn.Bits(24, 8)), b = opcode == 0x88c ? 252U : uniform_register(insn.Bits(32, 8));
+ if (output > 1 || insn.Bits(84, 3) != 7 || insn.Bits(87, 4) != 7 ||
+ (insn.Bits(72, 4) & ~2U) || insn.Bits(12, 4) != 7 || a == ~0U || b == ~0U)
+ return fail(index, "native uniform comparison mode");
+ const u32 temporary = 6 - output, mask = 1U << temporary;
+ if (opcode == 0x88c) mov_imm(252, insn.words[1]);
+ emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20));
+ emit(0x5b60000000070000ULL | 7 | (u64{temporary} << 3) |
+ (u64{a} << 8) | (u64{b} << 20) | (u64{7} << 39) |
+ (u64{insn.Bits(73, 1)} << 48) | (u64{insn.Bits(76, 3)} << 49));
+ emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20));
+ emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20));
+ break;
+ }
+ case 0x287: case 0x887: { // USEL, UP0/UP1; preserve ordinary predicates.
+ const u32 condition = insn.Bits(87, 3);
+ const u32 target = uniform_register(static_cast<u32>(dest));
+ const u32 a = uniform_register(insn.Bits(24, 8)), b = opcode == 0x887 ? 252U : uniform_register(insn.Bits(32, 8));
+ if (condition > 1 || target == ~0U || a == ~0U || b == ~0U ||
+ insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x07800000U) != 0x08000000U)
+ return fail(index, "native uniform select mode");
+ if (opcode == 0x887 && index > 0 && condition == 0 && insn.words[1] <= 0x7ffff &&
+ code[index - 1].Bits(0, 12) == 0x88c &&
+ code[index - 1].Bits(24, 8) == insn.Bits(24, 8) &&
+ code[index - 1].words[1] == insn.words[1] && code[index - 1].words[2] == 0x0bf01070 &&
+ insn.words[2] == 0x08000000) {
+ // Preserve the unsigned clamp idiom for indirect-table analysis.
+ emit(0x3820000000070000ULL | target | (u64{a} << 8) |
+ (u64{insn.words[1]} << 20) | (u64{7} << 39));
+ break;
+ }
+ if (opcode == 0x887) mov_imm(252, insn.words[1]);
+ const u32 temporary = 6 - condition, mask = 1U << temporary;
+ emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20));
+ emit(0x38f0000000070000ULL | (u64{239} << 8) | (u64{mask} << 20));
+ emit(0x5ca0000000070000ULL | target | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{temporary} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20));
+ break;
+ }
+ case 0x981: { // LDG through a uniform 64-bit base plus a register offset.
+ const u32 uniform_address = uniform_register(insn.Bits(32, 6));
+ u32 address = insn.Bits(24, 8);
+ const u32 width = insn.Bits(73, 3);
+ if ((insn.words[2] & ~(0x04000000U | 0xe00U)) != 0x081ea100 ||
+ insn.Bits(38, 2) || uniform_address == ~0U ||
+ (uniform_address != 255 && (uniform_address & 1)) || width > 6 ||
+ (width == 5 && ((dest & 1) || dest >= 254)) ||
+ (width == 6 && ((dest & 3) || dest >= 252)))
+ return fail(index, "native global load uniform addressing mode");
+ if (address == 255) address = uniform_address;
+ else {
+ if (insn.Bits(90, 1) && ((address & 1) || address >= 254))
+ return fail(index, "native global load register address pair");
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{uniform_address} << 8) |
+ (u64{address} << 20) | (u64{1} << 47));
+ emit(0x5c10000000000000ULL | predicate | 251 |
+ (u64{uniform_address == 255 ? 255U : uniform_address + 1} << 8) |
+ (u64{insn.Bits(90, 1) ? address + 1 : 255} << 20) | (u64{1} << 43));
+ address = 250;
+ }
+ emit(base(0xeed0000000000000ULL) | (u64{address} << 8) |
+ (u64{insn.Bits(40, 24)} << 20) | (u64{1} << 45) | (u64{width} << 48));
+ break;
+ }
+ case 0x386: case 0x986: { // Global store, original data and 64-bit address.
+ const u32 data = insn.Bits(32, 8), width = insn.Bits(73, 3);
+ if (width > 6 || (width == 5 && ((data & 1) || data >= 254)) ||
+ (width == 6 && ((data & 3) || data >= 252)))
+ return fail(index, "native global store width/alignment");
+ u32 address = insn.Bits(24, 8);
+ bool extended = insn.Bits(72, 1) != 0;
+ if (opcode == 0x986) {
+ const u32 uniform_address = uniform_register(insn.Bits(64, 6));
+ if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U)) != 0x0810a100 ||
+ uniform_address == ~0U || (uniform_address & 1) || !extended)
+ return fail(index, "native global store uniform addressing mode");
+ if (address == 255) address = uniform_address;
+ else {
+ if (insn.Bits(90, 1) && ((address & 1) || address >= 254))
+ return fail(index, "native global store register address pair");
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{uniform_address} << 8) |
+ (u64{address} << 20) | (u64{1} << 47));
+ emit(0x5c10000000000000ULL | predicate | 251 | (u64{uniform_address + 1} << 8) |
+ (u64{insn.Bits(90, 1) ? address + 1 : 255} << 20) | (u64{1} << 43));
+ address = 250;
+ }
+ } else if (insn.words[2] != 0x0010a900) {
+ return fail(index, "native global store register addressing mode");
+ }
+ emit(0xeed8000000000000ULL | predicate | data |
+ (u64{address} << 8) | (u64{insn.Bits(40, 24)} << 20) |
+ (u64{extended} << 45) | (u64{width} << 48));
+ break;
+ }
+ case 0x95b: // Fragment discard; execution predicate is preserved.
+ if (insn.words[1] || insn.words[2] != 0x03800000)
+ return fail(index, "native fragment discard mode");
+ emit(0xe33000000000000fULL | predicate);
+ break;
+ case 0x305: case 0xb05: { // F2I NTZ, register/constant-buffer source
+ const bool half = opcode == 0x305 && insn.words[2] == 0x0010f000;
+ const bool small = insn.words[2] == 0x00212800;
+ const bool floor = (insn.words[2] & ~0x100U) == 0x00207000;
+ const bool nearest = (insn.words[2] & ~0x100U) == 0x00203000;
+ if ((!half && !small && !floor && !nearest && (insn.words[2] & ~0x100U) != 0x0021f000) || (opcode == 0x305 && (insn.words[1] & 0xffffff00U))) {
+ return fail(index, "native float-to-integer conversion mode");
+ }
+ const bool is_signed = insn.Bits(72, 1);
+ u32 source = opcode == 0xb05 ? materialize(2, 0, 252, false, false) : insn.Bits(32, 8);
+ if (half) {
+ extract_half({source, 2, false, false}, 0, 252);
+ source = 252;
+ }
+ if (!is_signed) {
+ // Clear NaNs without consuming a guest predicate. Clamp the
+ // conversion below 2^32, retaining an overflow mask for saturation.
+ emit(0x5800000000000000ULL | predicate | 250 | (u64{source} << 8) |
+ (u64{source} << 20) | (u64{7} << 39) | (u64{7} << 48));
+ emit(0x5c40000000000000ULL | predicate | 250 | (u64{source} << 8) |
+ (u64{250} << 20) | (u64{7} << 48));
+ if (!small) {
+ mov_imm(252, 0x4f800000);
+ emit(0x5800000000000000ULL | predicate | 251 | (u64{250} << 8) |
+ (u64{252} << 20) | (u64{7} << 39) | (u64{6} << 48));
+ mov_imm(252, 0x4f7fffff);
+ emit(0x5c60000000000000ULL | predicate | 250 | (u64{250} << 8) |
+ (u64{252} << 20) | (u64{7} << 39));
+ }
+ source = 250;
+ }
+ emit(base(0x5cb0000000000000ULL) | (u64{small ? 1U : 2U} << 8) | (u64{2} << 10) |
+ (u64{is_signed} << 12) | (u64{source} << 20) |
+ (u64{small || nearest ? 0U : floor ? 1U : 3U} << 39) | (u64{insn.Bits(80, 1)} << 44));
+ if (!is_signed && !small) {
+ emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{251} << 20) |
+ (u64{1} << 41) | (u64{7} << 48));
+ }
+ break;
+ }
+ case 0xb6f: { // TXQ texture dimensions, two packed scalar outputs.
+ if (insn.Bits(59, 5) || insn.Bits(64, 8) != 255 || insn.Bits(72, 4) != 3 ||
+ insn.Bits(76, 14) || insn.Bits(91, 1) || insn.Bits(54, 5) != 2 ||
+ insn.Bits(40, 14) > 0x1fff || (dest & 1) || dest >= 254)
+ return fail(index, "native texture dimension query mode");
+ result.required_texture_buffer = 2;
+ emit(0xdf48000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{1} << 22) | (u64{3} << 31) | (u64{insn.Bits(40, 14)} << 36) |
+ (u64{insn.Bits(90, 1)} << 49));
+ break;
+ }
+ case 0xb66: { // TLD.SCR.LZ, direct texel fetch
+ const u32 mask = insn.Bits(72, 4);
+ const u32 dimension = insn.Bits(61, 3);
+ const bool array = dimension == 5;
+ const bool volume = dimension == 2;
+ u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8);
+ const u32 second_dest = insn.Bits(64, 8);
+ const bool split = second_dest != 255;
+ constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12};
+ constexpr std::array<u32, 5> rgba_masks{7, 11, 13, 14, 15};
+ u32 swizzle = ~0U;
+ if (split) {
+ for (u32 i = 0; i < rgba_masks.size(); ++i) if (rgba_masks[i] == mask) swizzle = i;
+ } else {
+ for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i;
+ }
+ if (!insn.Bits(60, 1) || (dimension != 0 && dimension != 1 && !array && !volume) ||
+ swizzle == ~0U || insn.Bits(76, 5) ||
+ insn.Bits(81, 3) != 7 || insn.Bits(87, 3) != 1 ||
+ insn.Bits(59, 1) ||
+ insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 ||
+ dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) ||
+ (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) ||
+ (array && ((source_a & 1) || source_a >= 249)) ||
+ (volume && ((source_a & 1) || source_a >= 254))) {
+ return fail(index, "native texel fetch mode or descriptor binding");
+ }
+ const u32 binding = insn.Bits(54, 5);
+ if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
+ return fail(index, "native texture descriptor buffer conflict");
+ result.required_texture_buffer = binding;
+ if (array) {
+ emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a + 1} << 20));
+ emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{source_b} << 20));
+ source_b = 250;
+ }
+ emit(0xda00000000000000ULL | predicate | dest | (u64{source_a} << 8) |
+ (u64{source_b} << 20) | (u64{insn.Bits(64, 8)} << 28) |
+ (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) |
+ (u64{swizzle} << 50) | (u64{array ? 8U : volume ? 7U : dimension == 0 ? 0U : 2U} << 53));
+ break;
+ }
+ case 0x91a: // DEPBAR.LE SB5, 1: scoreboard wait, represented by IR data dependencies.
+ if (insn.words[0] != 0x791a || (insn.words[1] & ~0xfc0U) != 0xd000 || insn.words[2]) {
+ return fail(index, "native dependency barrier mode");
+ }
+ break;
+ case 0xb63: { // TLD4, scalar 2D gather with split RG/BA destinations.
+ const u32 second_dest = insn.Bits(64, 8);
+ if (!insn.Bits(60, 1) || insn.Bits(61, 3) != 1 || insn.Bits(72, 4) != 15 ||
+ insn.Bits(76, 3) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) ||
+ insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 ||
+ (dest & 1) || dest >= 254 || (second_dest & 1) || second_dest >= 254)
+ return fail(index, "native 2D texture gather mode");
+ result.required_texture_buffer = 2;
+ emit(0xdf00000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{second_dest} << 28) |
+ (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) |
+ (u64{insn.Bits(87, 2)} << 52));
+ break;
+ }
+ case 0xb60: { // TEX, scalar 2D implicit-LOD sample
+ const u32 mask = insn.Bits(72, 4);
+ const u32 second_dest = insn.Bits(64, 8);
+ constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12};
+ constexpr std::array<u32, 5> rgba_masks{7, 11, 13, 14, 15};
+ u32 swizzle = ~0U;
+ const bool split = second_dest != 255;
+ const bool depth_compare = insn.Bits(78, 1);
+ const u32 dimension = insn.Bits(61, 3), lod = insn.Bits(87, 3);
+ const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8);
+ if (!insn.Bits(60, 1) && dimension == 7 && lod == 3) {
+ const u32 components = std::popcount(mask);
+ if (depth_compare || !components || split != (components > 2) ||
+ insn.Bits(76, 1) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) ||
+ insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 ||
+ source_a >= 246 || source_b >= 250 || dest == 255 ||
+ (components > 1 && ((dest & 1) || dest >= 254)) ||
+ (components == 4 && ((second_dest & 1) || second_dest >= 254)))
+ return fail(index, "native array-cube vector sample mode");
+ result.required_texture_buffer = 2;
+ // Vector coordinates already use layer,X,Y,Z order. LOD is
+ // supplied separately, matching the Maxwell TEX operand ABI.
+ emit(0xc000000000000000ULL | predicate | 250 | (u64{source_a} << 8) |
+ (u64{source_b} << 20) | (u64{7} << 28) | (u64{mask} << 31) |
+ (u64{insn.Bits(77, 1)} << 35) | (u64{insn.Bits(40, 14)} << 36) |
+ (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | (u64{lod} << 55));
+ for (u32 i = 0; i < components; ++i)
+ emit(0x5c98000000000000ULL | predicate |
+ (i < 2 ? dest + i : second_dest + i - 2) |
+ (u64{15} << 39) | (u64{250 + i} << 20));
+ break;
+ }
+ if (dimension == 5 && depth_compare && lod == 0 && !split && mask == 1) {
+ if (!insn.Bits(60, 1) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 ||
+ insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 ||
+ (source_a & 1) || (source_b & 1) || source_a >= 248 || source_b >= 248 || dest >= 248)
+ return fail(index, "native array depth comparison sample mode");
+ result.required_texture_buffer = 2;
+ for (u32 i = 0; i < 3; ++i) {
+ const u32 source = i < 2 ? source_a + i : source_b;
+ emit(0x5c98000000000000ULL | predicate | (250 + i) |
+ (u64{15} << 39) | (u64{source} << 20));
+ }
+ emit(0xc000000000000000ULL | predicate | dest | (u64{250} << 8) |
+ (u64{source_b + 1} << 20) | (u64{3} << 28) | (u64{1} << 31) |
+ (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) |
+ (u64{1} << 50) | (u64{7} << 51));
+ break;
+ }
+ if ((dimension == 3 && lod == 3) || (dimension == 2 && lod <= 1)) {
+ const u32 components = std::popcount(mask);
+ if (!insn.Bits(60, 1) || depth_compare || !components ||
+ split != (components > 2) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 ||
+ insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 ||
+ (source_a & 1) || (dimension == 3 && (source_b & 1)) || source_a >= 249 || source_b >= 249 ||
+ dest == 255 || (components > 1 && ((dest & 1) || dest >= 254)) ||
+ (components == 4 && ((second_dest & 1) || second_dest >= 254)))
+ return fail(index, "native volume/cube sample mode");
+ result.required_texture_buffer = 2;
+ const auto copy = [&](u32 target, u32 source) {
+ emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20));
+ };
+ copy(250, source_a);
+ copy(251, source_a + 1);
+ copy(252, source_b);
+ emit(0xc000000000000000ULL | predicate | 250 | (u64{250} << 8) |
+ (u64{dimension == 3 ? source_b + 1 : 255U} << 20) | (u64{dimension == 3 ? 6U : 4U} << 28) | (u64{mask} << 31) |
+ (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) |
+ (u64{7} << 51) | (u64{lod} << 55));
+ for (u32 i = 0; i < components; ++i)
+ copy(i < 2 ? static_cast<u32>(dest) + i : second_dest + i - 2, 250 + i);
+ break;
+ }
+ if (split) {
+ for (u32 i = 0; i < rgba_masks.size(); ++i) if (rgba_masks[i] == mask) swizzle = i;
+ } else {
+ for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i;
+ }
+ // SM70 scalar coordinates and split RG/BA destinations match TEXS.
+ // Descriptor offsets are words in both encodings, but only native
+ // explicitly encodes the constant-buffer binding.
+ if (!insn.Bits(60, 1) || (dimension != 1 && dimension != 5) ||
+ swizzle == ~0U || insn.Bits(76, 2) ||
+ insn.Bits(81, 3) != 7 || (lod > 1 && (lod != 3 || dimension != 1 || depth_compare)) ||
+ (lod == 3 && ((source_a & 1) || source_a >= 249)) ||
+ insn.Bits(59, 1) ||
+ insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 ||
+ dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) ||
+ (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) ||
+ (depth_compare && (dimension != 1 || split || mask != 1 || (source_a & 1) || source_a >= 249)) ||
+ (dimension == 5 && ((source_a & 1) || source_a >= 249))) {
+ return fail(index, "native texture mode or descriptor binding");
+ }
+ result.required_texture_buffer = 2;
+ emit(0xd800000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(64, 8)} << 28) |
+ (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) |
+ (u64{swizzle} << 50) | (u64{dimension == 5 ? (lod ? 8U : 7U) : depth_compare ? (lod ? 6U : 4U) : (lod == 3 ? 3U : lod ? 2U : 1U)} << 53));
+ break;
+ }
+ case 0x816: { // Captured PRMT half-word concatenation
+ if ((insn.words[1] != 0x5410 && insn.words[1] != 0x5432 && insn.words[1] != 0x7610 && insn.words[1] != 0x7632) ||
+ (insn.words[2] & ~0xffU)) return fail(index, "native permutation mode");
+ const u32 a = insn.Bits(24, 8), b = insn.Bits(64, 8);
+ if (insn.words[1] != 0x5432 && insn.words[1] != 0x7632) {
+ emit(0x0400000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{0xffff} << 20));
+ } else {
+ emit(0x3828000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{16} << 20));
+ }
+ if (insn.words[1] == 0x7610 || insn.words[1] == 0x7632) {
+ emit(0x0400000000000000ULL | predicate | 251 | (u64{b} << 8) | (u64{0xffff0000} << 20));
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{251} << 20));
+ } else {
+ emit(base(0x5c18000000000000ULL) | (u64{b} << 8) | (u64{250} << 20) | (u64{16} << 39));
+ }
+ break;
+ }
+ case 0x307: { // FRND.F32, round to an integral floating-point value
+ if (insn.Bits(75, 2) != 2 || insn.Bits(84, 2) != 2 || insn.Bits(81, 3) ||
+ insn.Bits(86, 6)) return fail(index, "native floating round format");
+ emit(base(0x5ca8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{8 + insn.Bits(78, 2)} << 39) |
+ (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(63, 1)} << 45) |
+ (u64{insn.Bits(62, 1)} << 49));
+ break;
+ }
+ case 0x321: // ALD
+ case 0x322: { // AST
+ if (insn.Bits(76, 2) || insn.Bits(79, 1)) return fail(index, "native indexed/patch attribute access");
+ const u32 address = insn.Bits(40, 10);
+ const u32 components = insn.Bits(74, 2) + 1;
+ if (address % 4 || address + components * 4 > 0x400 || insn.Bits(24, 8) != 255) {
+ return fail(index, "native attribute address or indexing");
+ }
+ const u32 value_reg = opcode == 0x321 ? static_cast<u32>(dest) : insn.Bits(32, 8);
+ if (value_reg + components > 255) return fail(index, "native attribute register range");
+ const u32 vertex = opcode == 0x321 ? insn.Bits(32, 8) : insn.Bits(64, 8);
+ const u64 translated = (opcode == 0x321 ? 0xefd8000000000000ULL : 0xeff0000000000000ULL) |
+ predicate | value_reg | (u64{255} << 8) | (u64{address} << 20) |
+ (u64{vertex} << 39) | (u64{components - 1} << 47);
+ result.instructions.push_back(translated);
+ break;
+ }
+ case 0x387: case 0x983: { // Captured scalar local spill at byte offset zero.
+ const bool store = opcode == 0x387;
+ if (insn.Bits(24, 8) != 255 || (store && (insn.Bits(16, 8) || insn.words[1] > 247)) ||
+ (!store && (insn.words[1] || dest >= 248)) ||
+ insn.words[2] != (store ? 0x00100800U : 0x00300800U))
+ return fail(index, "native local scalar spill mode");
+ emit((store ? 0xef50000000000000ULL : 0xef40000000000000ULL) | predicate |
+ (store ? insn.words[1] : dest) | (u64{255} << 8) | (u64{4} << 48));
+ break;
+ }
+ case 0x803: { // P2R PR, register merge source, immediate mask
+ if (insn.words[1] > 0x7f || insn.words[2])
+ return fail(index, "native predicate-to-register mode");
+ emit(base(0x38e8000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.words[1]} << 20));
+ break;
+ }
+ case 0x804: { // R2P PR, register byte, immediate mask
+ if (insn.Bits(16, 8) || insn.words[1] > 0x7f || (insn.words[2] & ~0x3000U))
+ return fail(index, "native register-to-predicate mode");
+ emit(0x38f0000000000000ULL | predicate | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.words[1]} << 20) | (u64{insn.Bits(76, 2)} << 41));
+ break;
+ }
+ case 0xc02: // MOV uniform register
+ case 0x202: // MOV register
+ case 0x802: // MOV immediate
+ case 0xa02: { // MOV constant buffer
+ if (insn.Bits(72, 4) != 15 || (insn.Bits(76, 16) & ~(opcode == 0xc02 ? 0x8000U : 0U))) return fail(index, "native MOV modifiers");
+ if (opcode == 0x802) {
+ result.instructions.push_back(base(0x0100000000000000ULL) | (u64{15} << 12) |
+ (u64{insn.words[1]} << 20));
+ } else if (opcode == 0xc02) {
+ if (insn.words[1] >= 16 || insn.words[2] != 0x08000f00) return fail(index, "native uniform move source");
+ emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.words[1] + 216} << 20));
+ } else if (opcode == 0x202) {
+ if (insn.Bits(40, 24)) return fail(index, "native MOV register modifiers");
+ result.instructions.push_back(base(0x5c98000000000000ULL) | (u64{15} << 39) |
+ (u64{insn.Bits(32, 8)} << 20));
+ } else {
+ if (!valid_cbuf() || insn.Bits(62, 2)) return fail(index, "native MOV constant buffer");
+ result.instructions.push_back(base(0x4c98000000000000ULL) | (u64{15} << 39) | cbuf());
+ }
+ break;
+ }
+ case 0x223: case 0x423: case 0x623: case 0x823: case 0xa23: case 0xc23: case 0xe23: { // FFMA
+ if ((insn.Bits(76, 1) && !insn.Bits(80, 1)) ||
+ (insn.Bits(81, 11) & ~((opcode == 0xc23 || opcode == 0xe23) ? 0x400U : 0U))) return fail(index, "native FFMA denormal/source mode");
+ const u32 a = operand(0), b = operand(1), c = operand(2);
+ emit(base(0x5980000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | (u64{c} << 39) |
+ (u64{insn.Bits(77, 1)} << 50) | (u64{insn.Bits(78, 2)} << 51) |
+ (u64{insn.Bits(76, 1) ? 2U : insn.Bits(80, 1)} << 53));
+ break;
+ }
+ case 0x220: case 0x820: case 0xa20: case 0xc20: { // FMUL
+ const u32 scale = insn.Bits(84, 3);
+ if (insn.Bits(76, 1) || (scale != 3 && scale != 4 && scale != 5 && scale != 6) || insn.Bits(81, 3) ||
+ insn.Bits(87, 4) || (insn.Bits(91, 1) != 0) != (opcode == 0xc20) ||
+ (scale != 4 && (!insn.Bits(80, 1) || insn.Bits(78, 2))))
+ return fail(index, "native FMUL scale/mode");
+ const u32 a = operand(0), b = operand(1);
+ emit(base(0x5c68000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(78, 2)} << 39) | (u64{scale == 3 ? 1U : scale == 5 ? 6U : scale == 6 ? 5U : 0U} << 41) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(77, 1)} << 50));
+ break;
+ }
+ case 0x221: case 0x421: case 0x621: case 0x821: case 0xa21: case 0xe21: { // FADD
+ if (insn.Bits(81, 10) || (insn.Bits(91, 1) != 0) != (opcode == 0xe21)) return fail(index, "native FADD mode");
+ const u32 a = operand(0), b = operand(insn.Bits(9, 3) == 2 || insn.Bits(9, 3) == 3 || insn.Bits(9, 3) == 7 ? 2 : 1);
+ emit(base(0x5c58000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(78, 2)} << 39) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(77, 1)} << 50));
+ break;
+ }
+ case 0x209: case 0x809: case 0xa09: { // FMNMX
+ if (insn.Bits(74, 6) || insn.Bits(81, 6) || insn.Bits(91, 1))
+ return fail(index, "native floating min/max mode");
+ const u32 a = operand(0), b = operand(1);
+ emit(base(0x5c60000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) |
+ (u64{insn.Bits(80, 1)} << 44));
+ break;
+ }
+ case 0x308: case 0xb08: case 0xd08: { // MUFU, including uniform sources
+ const u32 operation = insn.Bits(74, 6);
+ if (operation > 8 || operation == 6 || operation == 7 || insn.Bits(80, 11) ||
+ (insn.Bits(91, 1) != 0) != (opcode == 0xd08)) return fail(index, "native MUFU mode");
+ if (insn.Bits(73, 1)) {
+ if (opcode != 0x308 || insn.Bits(60, 2) > 1)
+ return fail(index, "native half special-function source");
+ emit(0x5ca8000000000000ULL | predicate | 250 | (u64{2} << 8) | (u64{1} << 10) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(60, 1)} << 41) |
+ (u64{insn.Bits(63, 1)} << 45) | (u64{insn.Bits(62, 1)} << 49));
+ emit(0x5080000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{operation} << 20));
+ emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) | (u64{250} << 20));
+ } else {
+ const u32 src = operand(1);
+ emit(base(0x5080000000000000ULL) | (u64{src} << 8) | (u64{operation} << 20));
+ }
+ break;
+ }
+ case 0x526: { // IPA with a uniform byte offset into the varying array.
+ const u32 source = uniform_register(insn.Bits(24, 8));
+ if (source == ~0U || insn.words[1] != 255 ||
+ (insn.Bits(76, 4) != 0 && insn.Bits(76, 4) != 4) ||
+ insn.Bits(81, 3) != 7 || insn.Bits(84, 7) || !insn.Bits(91, 1))
+ return fail(index, "native indexed interpolation mode");
+ const u32 offset = insn.Bits(64, 8) * 4;
+ emit(0x1c00000000000000ULL | predicate | 250 | (u64{source} << 8) | (u64{offset} << 20));
+ emit(base(0xe000000000000000ULL) | (u64{250} << 8) | (u64{255} << 20) |
+ (u64{1} << 38) | (u64{insn.Bits(78, 1)} << 55));
+ break;
+ }
+ case 0x326: { // IPA.PASS / CONSTANT, default sample location
+ if (insn.Bits(32, 8) != 255 || (insn.Bits(76, 4) != 0 && insn.Bits(76, 4) != 4) || insn.Bits(81, 3) != 7) return fail(index, "native IPA interpolation mode");
+ emit(base(0xe000000000000000ULL) | (u64{255} << 8) | (u64{255} << 20) | (u64{insn.Bits(64, 8)} << 30) | (u64{insn.Bits(78, 1)} << 55));
+ break;
+ }
+ case 0x805: { // CS2R SRZ
+ if (insn.Bits(72, 8) != 255 || !insn.Bits(80, 1) || dest >= 249) return fail(index, "native special-register read");
+ mov_imm(static_cast<u32>(dest), 0);
+ mov_imm(static_cast<u32>(dest) + 1, 0);
+ break;
+ }
+ case 0x306: { // I2F.F32.U8, low byte, round-to-nearest
+ if (insn.words[2] != 0x1000 || (insn.words[1] & 0xffffff00U))
+ return fail(index, "native byte-to-float conversion mode");
+ emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{insn.Bits(32, 8)} << 20));
+ break;
+ }
+ case 0x245: case 0xa45: case 0xc45: { // I2FP.F32.U32
+ if ((insn.words[2] & ~(0x400U | (opcode == 0xc45 ? 0x08000000U : 0U))) != 0x00201000) return fail(index, "native packed integer-to-float format");
+ const u32 src = operand(1, false);
+ emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | (u64{insn.Bits(74, 1)} << 13) | (u64{src} << 20));
+ break;
+ }
+ case 0x207: case 0x807: case 0xa07:
+ case 0x208: case 0x808: case 0xa08: { // Bitwise SEL / FSEL
+ const u32 a = operand(0, false), b = operand(1, false);
+ emit(base(0x5ca0000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ break;
+ }
+ case 0x20a: case 0x80a: case 0xa0a: { // FSET.BF with PT/AND
+ if (insn.Bits(64, 8) || insn.Bits(74, 2) || insn.Bits(81, 6) ||
+ insn.Bits(87, 4) != 7 || insn.Bits(91, 1))
+ return fail(index, "native floating comparison output mode");
+ const u32 a = operand(0), b = operand(1);
+ emit(base(0x5800000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{7} << 39) | (u64{insn.Bits(76, 4)} << 48) | (u64{1} << 52) |
+ (u64{insn.Bits(80, 1)} << 55));
+ break;
+ }
+ case 0x20b: case 0x80b: case 0xa0b: // FSETP
+ case 0x20c: case 0x80c: case 0xa0c: case 0xc0c: { // ISETP
+ const bool floating = (opcode & 0x1ff) == 0xb;
+ if (!floating && insn.Bits(72, 1)) return fail(index, "native extended integer comparison");
+ const u32 a = operand(0, floating), b = operand(1, floating);
+ const u32 combine = insn.Bits(74, 2);
+ if (combine > 2) return fail(index, "native predicate combination");
+ u64 out = (floating ? 0x5bb0000000000000ULL : 0x5b60000000000000ULL) | predicate |
+ insn.Bits(84, 3) | (u64{insn.Bits(81, 3)} << 3) | (u64{a} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | (u64{combine} << 45);
+ if (floating) out |= (u64{insn.Bits(80, 1)} << 47) | (u64{insn.Bits(76, 4)} << 48);
+ else out |= (u64{insn.Bits(73, 1)} << 48) | (u64{insn.Bits(76, 3)} << 49);
+ emit(out);
+ break;
+ }
+ case 0x212: case 0x812: case 0xc12: { // Bounded LOP3 bitwise forms, no predicate output
+ const u32 lut = insn.Bits(72, 8);
+ if ((lut != 0xc0 && lut != 0xfc && lut != 0xf8 && lut != 0x48 && lut != 0xe2) || insn.Bits(80, 1) ||
+ insn.Bits(87, 4) != 15 ||
+ (opcode == 0x212 && insn.Bits(40, 24))) return fail(index, "native LUT/predicate mode");
+ u32 b = opcode == 0xc12 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8);
+ if (b == ~0U) return fail(index, "native uniform logic source");
+ if (opcode == 0x812) {
+ mov_imm(251, insn.words[1]);
+ b = 251;
+ }
+ const u32 a = insn.Bits(24, 8), c = insn.Bits(64, 8);
+ const auto logical = [&](u32 target, u32 lhs, u32 rhs, u32 operation, u32 pred_target = 7) {
+ emit(0x5c40000000000000ULL | predicate | target | (u64{lhs} << 8) |
+ (u64{rhs} << 20) | (u64{operation} << 41) |
+ (u64{pred_target == 7 ? 0U : 3U} << 44) | (u64{pred_target} << 48));
+ };
+ const u32 pred_target = insn.Bits(81, 3);
+ if (lut == 0xe2) {
+ logical(250, a, b, 0);
+ logical(252, c, b, 0);
+ result.instructions.back() |= u64{1} << 40;
+ logical(static_cast<u32>(dest), 250, 252, 1, pred_target);
+ } else if (lut == 0xf8) {
+ logical(250, b, c, false);
+ logical(static_cast<u32>(dest), a, 250, 1, pred_target);
+ } else if (lut == 0x48) {
+ logical(250, a, c, 2);
+ logical(static_cast<u32>(dest), 250, b, 0, pred_target);
+ } else {
+ logical(static_cast<u32>(dest), a, b, lut == 0xfc ? 1U : 0U, pred_target);
+ }
+ break;
+ }
+ case 0x217: case 0x817: case 0xc17: { // IMNMX register, immediate, or uniform operand.
+ if ((insn.words[2] & ~(0x04000200U | (opcode == 0xc17 ? 0x08000000U : 0U))) != 0x03800000 ||
+ insn.words[1] > (opcode == 0x817 ? 0x7ffffU : 255U))
+ return fail(index, "native integer min/max mode");
+ const u32 source = opcode == 0xc17 ? uniform_register(insn.words[1]) : insn.words[1];
+ if (source == ~0U) return fail(index, "native uniform min/max operand");
+ emit(base(opcode == 0x817 ? 0x3820000000000000ULL : 0x5c20000000000000ULL) |
+ (u64{insn.Bits(24, 8)} << 8) | (u64{source} << 20) |
+ (u64{insn.Bits(87, 4)} << 39) | (u64{insn.Bits(73, 1)} << 48));
+ break;
+ }
+ case 0x819: { // SHF.HI, immediate shift
+ const bool is_signed = insn.Bits(73, 2) == 2;
+ if (!insn.Bits(80, 1) && !insn.Bits(76, 1) && insn.Bits(64, 8) == 255 &&
+ insn.Bits(73, 2) == 3 && !insn.Bits(75, 1) && insn.words[1] <= 32) {
+ emit(base(0x3848000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20));
+ break;
+ }
+ if ((insn.Bits(73, 2) != 3 && !is_signed) || insn.Bits(75, 1) || !insn.Bits(80, 1) || insn.words[1] > 32) return fail(index, "native funnel-shift mode");
+ const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8), shift = insn.words[1];
+ if (insn.Bits(76, 1)) {
+ if (low != 255) return fail(index, "native nonzero low right-shift word");
+ emit(base(0x3828000000000000ULL) | (u64{high} << 8) | (u64{shift} << 20) |
+ (u64{is_signed} << 48));
+ } else {
+ if (is_signed) return fail(index, "native signed left funnel shift");
+ emit(base(0x36f8000000000000ULL) | (u64{low} << 8) | (u64{high} << 39) | (u64{shift} << 20));
+ }
+ break;
+ }
+ case 0xa11: { // LEA with constant base and carry predicate.
+ if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) ||
+ insn.Bits(81, 3) == 7 || insn.Bits(62, 2))
+ return fail(index, "native constant LEA carry mode");
+ const u32 b = operand(1, false);
+ emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39));
+ emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) |
+ (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49));
+ break;
+ }
+ case 0x211: case 0x811: { // LEA low word, no carry result
+ if (opcode == 0x211 && insn.Bits(80, 1)) {
+ if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(81, 3) != 7 ||
+ insn.Bits(32, 8) != 255 || insn.Bits(64, 8) != 255 || insn.Bits(62, 2))
+ return fail(index, "native high effective address mode");
+ emit(base(0x3828000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{32 - insn.Bits(75, 5)} << 20));
+ break;
+ }
+ if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || insn.Bits(81, 3) != 7 ||
+ (opcode == 0x211 && insn.Bits(62, 2))) return fail(index, "native LEA carry/high mode");
+ u32 b = insn.Bits(32, 8);
+ if (opcode == 0x811) { mov_imm(251, insn.words[1]); b = 251; }
+ emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39));
+ break;
+ }
+ case 0x210: case 0x810: case 0xa10: { // IADD3 with zero third source and no carry outputs
+ if (opcode == 0xa10 && insn.Bits(81, 3) != 7) {
+ if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(84, 7) != 0x7f ||
+ insn.Bits(77, 4) != 15 || insn.Bits(63, 1) || insn.Bits(72, 1))
+ return fail(index, "native constant add carry mode");
+ const u32 b = operand(1, false);
+ emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20));
+ emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) |
+ (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49));
+ break;
+ }
+ if (opcode == 0xa10 && insn.Bits(74, 1)) {
+ if (insn.Bits(24, 8) != 255 || (insn.words[2] & ~0x07800000U) != 0x007fe4ffU)
+ return fail(index, "native extended constant add mode");
+ const u32 b = operand(1, false);
+ mov_imm(250, 1);
+ emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ emit(base(0x5c10000000000000ULL) | (u64{b} << 8) | (u64{250} << 20));
+ break;
+ }
+ if (insn.Bits(64, 8) != 255 && !insn.Bits(73, 4) && insn.Bits(77, 7) == 0x7f && insn.Bits(84, 7) == 0x7f) {
+ const u32 b = operand(1, false);
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{b} << 20) | (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49));
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{insn.Bits(64, 8)} << 20));
+ break;
+ }
+ if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(77, 7) != 0x7f || insn.Bits(84, 7) != 0x7f) return fail(index, "native integer carry/third source");
+ if (opcode == 0xa10) {
+ const u32 b = operand(1, false);
+ emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20) |
+ (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49));
+ } else if (opcode == 0x210) {
+ if (insn.words[1] & ~0x800000ffU) return fail(index, "native integer add register modifiers");
+ emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(63, 1)} << 48) | (u64{insn.Bits(72, 1)} << 49));
+ } else {
+ emit(base(0x1c00000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.words[1]} << 20) | (u64{insn.Bits(72, 1)} << 56));
+ }
+ break;
+ }
+ case 0x949: case 0x958: { // Captured BRX / BRXU with absolute native-PC table values
+ if (insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x3ffffU) != (opcode == 0x958 ? 0x0b800000U : 0x03800000U))
+ return fail(index, "native computed branch mode");
+ const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32);
+ const s64 relative = static_cast<s64>(raw << 14) >> 14;
+ if (s64{index} * 16 + 16 + relative != 0)
+ return fail(index, "native computed branch relative base");
+ const s64 offset = -s64{address_of(static_cast<u32>(result.instructions.size()))} - 8;
+ if (offset < -0x800000) return fail(index, "native computed branch range");
+ const u32 source = opcode == 0x958 ? uniform_register(insn.Bits(24, 8)) : insn.Bits(24, 8);
+ if (source == ~0U) return fail(index, "native uniform computed branch source");
+ emit(0xe25000000000000fULL | predicate | (u64{source} << 8) |
+ ((static_cast<u64>(offset) & 0xffffff) << 20));
+ break;
+ }
+ case 0x944: { // Verified CALL.REL.NOINC with R55 holding the return PC
+ // A predicated NOINC forward transfer can terminate a loop at its
+ // reconvergence point without a return-link setup or matching RET.
+ if (native_predicate != 7 && insn.words[1] == 16 && insn.words[2] == 0x03c00000 &&
+ index + 2 < code.size() && code[index + 1].Bits(0, 12) == 0x947) {
+ relocations.push_back({static_cast<u32>(result.instructions.size()), index + 2});
+ emit(0xe24000000000000fULL | predicate);
+ break;
+ }
+ if (main_exited || index == 0 || insn.words[0] != 0x7944 ||
+ (insn.words[2] & ~0x3ffffU) != 0x03c00000 ||
+ (code[index - 1].words[0] & ~0x00ff0000U) != 0x00007802 ||
+ code[index - 1].Bits(16, 8) >= 250 ||
+ code[index - 1].words[1] != (index + 1) * 16 ||
+ code[index - 1].words[2] != 0xf00)
+ return fail(index, "native call linkage mode");
+ const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32);
+ const s64 relative = static_cast<s64>(raw << 14) >> 14;
+ const s64 target = s64{index} * 16 + 16 + relative;
+ if (target < 0 || target % 16) return fail(index, "native call destination alignment");
+ const u32 link = code[index - 1].Bits(16, 8);
+ for (size_t call = 0; call < call_targets.size(); ++call) {
+ if (call_targets[call] == target / 16 && call_links[call] != link)
+ return fail(index, "native inconsistent call linkage");
+ }
+ call_targets.push_back(static_cast<u32>(target / 16));
+ call_links.push_back(link);
+ relocations.push_back({static_cast<u32>(result.instructions.size()), target / 16});
+ emit(0xe26000000007000fULL);
+ break;
+ }
+ case 0x950: { // Verified RET.REL.NODEC epilogue restoring the call link
+ const u32 return_reg = insn.Bits(24, 8);
+ if (!main_exited || index < 2 || !active_link ||
+ (insn.words[0] & 0x00ffffffU) != 0x00007950 || return_reg >= 249 ||
+ (insn.words[2] & ~0x3ffffU) != 0x03c00000 ||
+ code[index - 2].words[0] != (0x00007202U | (return_reg << 16)) ||
+ code[index - 2].words[1] != *active_link || code[index - 2].words[2] != 0xf00 ||
+ code[index - 1].words[0] != (0x00007802U | ((return_reg + 1) << 16)) ||
+ code[index - 1].words[1] || code[index - 1].words[2] != 0xf00 || !barrier_tokens.empty())
+ return fail(index, "native return linkage mode");
+ const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32);
+ const s64 relative = static_cast<s64>(raw << 14) >> 14;
+ if (s64{index} * 16 + 16 + relative != 0)
+ return fail(index, "native return relative base");
+ for (const auto& relocation : relocations) {
+ if (relocation.target == index || relocation.target == index - 1)
+ return fail(index, "native branch bypasses return linkage");
+ }
+ emit(0xe32000000007000fULL);
+ bool complete = true;
+ for (u32 target : call_targets) complete &= target <= index;
+ if (complete) return finish(index);
+ break;
+ }
+ case 0x947: case 0x945: case 0x941: { // BRA / BSSY B0 / BSYNC B0
+ u64 branch_predicate = predicate;
+ if (opcode == 0x947 && insn.Bits(87, 4) != 7) {
+ if (insn.Bits(12, 4) != 7) {
+ relocations.push_back({static_cast<u32>(result.instructions.size()), index + 1});
+ emit(0xe24000000000000fULL | (u64{insn.Bits(12, 4) ^ 8U} << 16));
+ }
+ branch_predicate = u64{insn.Bits(87, 4)} << 16;
+ } else if (insn.Bits(87, 4) != 7) {
+ return fail(index, "native extra branch condition");
+ }
+ if (opcode == 0x941) {
+ if (barrier_tokens.empty() || barrier_tokens.back() != insn.Bits(16, 4) || insn.Bits(12, 4) != 7) return fail(index, "native reconvergence token");
+ barrier_tokens.pop_back();
+ emit(0xf0f800000007000fULL);
+ break;
+ }
+ if (opcode == 0x945) {
+ if (barrier_tokens.size() >= 16 || insn.Bits(16, 4) != barrier_tokens.size() || insn.Bits(12, 4) != 7) return fail(index, "native nested reconvergence token");
+ barrier_tokens.push_back(insn.Bits(16, 4));
+ }
+ s64 relative = static_cast<s32>(insn.words[1]);
+ if (opcode == 0x947) {
+ const u64 raw = insn.words[1] | (u64{insn.words[2] & 0x3ffff} << 32);
+ relative = static_cast<s64>(raw << 14) >> 14;
+ }
+ const s64 target = s64{index} * 16 + 16 + relative;
+ if (target < 0 || target % 16) return fail(index, "native branch destination alignment");
+ relocations.push_back({static_cast<u32>(result.instructions.size()), target / 16});
+ emit((opcode == 0x947 ? 0xe24000000000000fULL | branch_predicate : 0xe29000000007000fULL));
+ break;
+ }
+ case 0x94d: { // EXIT
+ if (insn.Bits(84, 2) || insn.Bits(87, 4) != 7) {
+ return fail(index, "native conditional/special EXIT");
+ }
+ result.instructions.push_back(0xe30000000000000fULL | predicate);
+ if (insn.Bits(12, 4) != 7) break;
+ if (!barrier_tokens.empty()) return fail(index, "native unclosed reconvergence scope");
+ if (call_targets.empty()) {
+ const bool pending_branch = std::any_of(relocations.begin(), relocations.end(),
+ [&](const Relocation& relocation) { return relocation.target > index; });
+ if (!pending_branch) return finish(index);
+ break;
+ }
+ if (main_exited) return fail(index, "native library exits without returning");
+ for (u32 target : call_targets) {
+ if (target <= index) return fail(index, "native call target overlaps main program");
+ }
+ main_exited = true;
+ break;
+ }
+ default:
+ return fail(index, "unsupported native opcode");
+ }
+ if (!operand_error.empty()) return fail(index, operand_error);
+ if (guarded_uniform) restore_predicate();
+ }
+ return fail(static_cast<u32>(code.size()), main_exited ? "native program lacks captured library return" : "native program has no unconditional EXIT");
+}
+
+} // namespace Shader::Ampere
diff --git a/src/shader_recompiler/frontend/ampere/program_address_map.h b/src/shader_recompiler/frontend/ampere/program_address_map.h
new file mode 100644
index 0000000000..bee32a821c
--- /dev/null
+++ b/src/shader_recompiler/frontend/ampere/program_address_map.h
@@ -0,0 +1,29 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+#include <optional>
+#include <span>
+#include <vector>
+#include "common/common_types.h"
+
+namespace Shader::Ampere {
+constexpr u64 NativeAddressMapMagic = 0x50414d4e4544454eULL;
+inline bool HasNativeAddressMap(std::span<const u64> code) {
+ return code.size() >= 2 && code.back() == NativeAddressMapMagic;
+}
+inline void AppendNativeAddressMap(std::vector<u64>& code, std::span<const u32> offsets) {
+ if (offsets.empty()) return;
+ for (u32 index : offsets) code.push_back(0x50U + index * 8 + ((index + 1) / 3) * 8);
+ code.push_back(offsets.size());
+ code.push_back(NativeAddressMapMagic);
+}
+inline std::optional<u32> NativeProgramAddress(std::span<const u64> code, u32 address) {
+ if (!HasNativeAddressMap(code) || address % 16) return {};
+ const u64 count = code[code.size() - 2];
+ if (!count || count > 65536 || count + 2 > code.size() || address / 16 >= count) return {};
+ const size_t start = code.size() - static_cast<size_t>(count) - 2;
+ const u64 target = code[start + address / 16];
+ if (target < 0x50 || target >= start * 8 || target % 8 || target % 32 == 0) return {};
+ return static_cast<u32>(target);
+}
+} // namespace Shader::Ampere
diff --git a/src/shader_recompiler/frontend/ampere/program_capture.h b/src/shader_recompiler/frontend/ampere/program_capture.h
new file mode 100644
index 0000000000..a7f2351254
--- /dev/null
+++ b/src/shader_recompiler/frontend/ampere/program_capture.h
@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h"
+
+namespace Shader::Ampere {
+class ProgramCaptureState {
+public:
+ enum class Result { Continue, Complete, InvalidTarget };
+ static constexpr u32 MaximumInstructions = 65536;
+
+ Result Observe(const Instruction& instruction, u32 index) {
+ const u32 opcode = instruction.Bits(0, 12);
+ if (opcode == 0x944 || opcode == 0x947 || opcode == 0x945) {
+ const u64 raw = instruction.words[1] | (u64{instruction.words[2] & 0x3ffff} << 32);
+ const s64 relative = opcode == 0x945 ? static_cast<s32>(instruction.words[1])
+ : static_cast<s64>(raw << 14) >> 14;
+ const s64 target = s64{index} * 16 + 16 + relative;
+ if (target < 0 || target % 16 || target >= MaximumInstructions * 16)
+ return Result::InvalidTarget;
+ // Predicated NOINC transfers to the following reconvergence
+ // point are loop exits, not calls into a trailing function library.
+ const bool loop_exit = opcode == 0x944 && instruction.Bits(12, 4) != 7 &&
+ instruction.words[1] == 16 && instruction.words[2] == 0x03c00000;
+ if (opcode == 0x944 && !loop_exit) {
+ last_call_target = std::max(last_call_target, static_cast<u32>(target / 16));
+ } else {
+ last_forward_target = std::max(last_forward_target, static_cast<u32>(target / 16));
+ }
+ }
+ if (opcode == 0x94d && instruction.Bits(12, 4) == 7 && index >= last_forward_target) {
+ if (!last_call_target) return Result::Complete;
+ main_exited = true;
+ }
+ if (opcode == 0x950 && instruction.Bits(12, 4) == 7 && main_exited &&
+ index >= last_call_target && index >= last_forward_target)
+ return Result::Complete;
+ return Result::Continue;
+ }
+
+private:
+ u32 last_call_target{};
+ u32 last_forward_target{};
+ bool main_exited{};
+};
+} // namespace Shader::Ampere
diff --git a/src/shader_recompiler/frontend/ampere/program_override.h b/src/shader_recompiler/frontend/ampere/program_override.h
new file mode 100644
index 0000000000..0943028b0c
--- /dev/null
+++ b/src/shader_recompiler/frontend/ampere/program_override.h
@@ -0,0 +1,85 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <cstring>
+#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h"
+
+namespace Shader::Ampere {
+
+inline u64 NativeProgramKey(std::span<const Instruction> source) {
+ u64 hash = 14695981039346656037ULL;
+ for (const auto& instruction : source) {
+ for (u32 word : instruction.words) {
+ for (u32 byte = 0; byte < 4; ++byte) hash = (hash ^ ((word >> (byte * 8)) & 255)) * 1099511628211ULL;
+ }
+ }
+ return hash;
+}
+
+// Private iteration format: version, lengths, texture bank, exact original
+// instructions, then the newly generated canonical instructions. Source bytes
+// are compared directly; the filename hash is only an index.
+inline std::vector<u8> EncodeProgramOverride(std::span<const Instruction> source,
+ const LoweredProgram& program) {
+ if (!program || program.source_size != source.size_bytes() || program.source_instruction_offsets.size() != source.size()) return {};
+ const std::array<u32, 6> header{0x4e454445, 0x3230564f,
+ static_cast<u32>(source.size_bytes()), static_cast<u32>(program.instructions.size()),
+ program.required_texture_buffer, static_cast<u32>(program.source_instruction_offsets.size())};
+ std::vector<u8> bytes(sizeof(header) + source.size_bytes() + program.instructions.size() * 8 + program.source_instruction_offsets.size() * 4);
+ std::memcpy(bytes.data(), header.data(), sizeof(header));
+ std::memcpy(bytes.data() + sizeof(header), source.data(), source.size_bytes());
+ std::memcpy(bytes.data() + sizeof(header) + source.size_bytes(), program.instructions.data(), program.instructions.size() * 8);
+ std::memcpy(bytes.data() + sizeof(header) + source.size_bytes() + program.instructions.size() * 8,
+ program.source_instruction_offsets.data(), program.source_instruction_offsets.size() * 4);
+ return bytes;
+}
+
+inline u32 ProgramOverrideSourceSize(std::span<const u8> bytes) {
+ std::array<u32, 6> header{};
+ if (bytes.size() < sizeof(header)) return 0;
+ std::memcpy(header.data(), bytes.data(), sizeof(header));
+ const bool mapped = header[1] == 0x3230564f;
+ if (header[0] != 0x4e454445 || (!mapped && header[1] != 0x3130564f) ||
+ (mapped ? header[5] != header[2] / 16 : header[5] != 0) ||
+ header[2] == 0 || header[2] % sizeof(Instruction) || header[2] > 1048576 ||
+ header[3] == 0 || header[3] > 1048576 || (header[4] != ~0U && header[4] >= 18)) return 0;
+ if (bytes.size() != sizeof(header) + size_t{header[2]} + size_t{header[3]} * 8 + size_t{header[5]} * 4) return 0;
+ return header[2];
+}
+
+inline LoweredProgram DecodeProgramOverride(std::span<const u8> bytes,
+ std::span<const Instruction> source) {
+ LoweredProgram result;
+ const auto fail = [&result](std::string_view message) {
+ result.instructions.clear();
+ result.source_instruction_offsets.clear();
+ result.error = message;
+ return result;
+ };
+ std::array<u32, 6> header{};
+ if (!ProgramOverrideSourceSize(bytes)) return fail("invalid native override header or length");
+ std::memcpy(header.data(), bytes.data(), sizeof(header));
+ if (header[2] != source.size_bytes()) return fail("native override source length mismatch");
+ if (std::memcmp(bytes.data() + sizeof(header), source.data(), source.size_bytes()))
+ return fail("native override source mismatch");
+ result.instructions.resize(header[3]);
+ std::memcpy(result.instructions.data(), bytes.data() + sizeof(header) + source.size_bytes(), header[3] * 8);
+ if (header[5]) {
+ result.source_instruction_offsets.resize(header[5]);
+ std::memcpy(result.source_instruction_offsets.data(),
+ bytes.data() + sizeof(header) + source.size_bytes() + size_t{header[3]} * 8,
+ size_t{header[5]} * 4);
+ u32 previous = 0;
+ for (u32 offset : result.source_instruction_offsets) {
+ if (offset >= result.instructions.size() || offset < previous)
+ return fail("native override address map mismatch");
+ previous = offset;
+ }
+ }
+ result.source_size = header[2];
+ result.required_texture_buffer = header[4];
+ return result;
+}
+
+} // namespace Shader::Ampere
diff --git a/src/shader_recompiler/frontend/maxwell/control_flow.cpp b/src/shader_recompiler/frontend/maxwell/control_flow.cpp
index 7dabfbf16e..9fb30df664 100644
--- a/src/shader_recompiler/frontend/maxwell/control_flow.cpp
+++ b/src/shader_recompiler/frontend/maxwell/control_flow.cpp
@@ -457,7 +457,7 @@ CFG::AnalysisState CFG::AnalyzeBRX(Block* block, Location pc, Instruction inst,
block->indirect_branches.reserve(targets.size());
for (const u32 target : targets) {
- Block* const branch{AddLabel(block, block->stack, target, function_id)};
+ Block* const branch{AddLabel(block, block->stack, env.TranslateProgramAddress(target), function_id)};
block->indirect_branches.push_back({
.block = branch,
.address = target,
diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt
index 875c40604c..1bcb954334 100644
--- a/src/tests/CMakeLists.txt
+++ b/src/tests/CMakeLists.txt
@@ -17,7 +17,15 @@ add_executable(tests
common/undefined_fix.cpp
common/unique_function.cpp
core/core_timing.cpp
+ core/program_metadata.cpp
+ core/native_gpu_characteristics.cpp
+ core/graphics_supervisor.cpp
core/internal_network/network.cpp
+ video_core/native_semaphore.cpp
+ video_core/native_shader.cpp
+ video_core/native_swizzle.cpp
+ video_core/image_overlap.cpp
+ video_core/pending_download.cpp
video_core/memory_tracker.cpp
input_common/calibration_configuration_job.cpp
)
diff --git a/src/tests/core/graphics_supervisor.cpp b/src/tests/core/graphics_supervisor.cpp
new file mode 100644
index 0000000000..08e272f09a
--- /dev/null
+++ b/src/tests/core/graphics_supervisor.cpp
@@ -0,0 +1,133 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <array>
+#include "core/hle/service/am/service/gpu_memory_registration.h"
+#include <catch2/catch_test_macros.hpp>
+#include "core/hle/service/am/service/graphics_supervisor_protocol.h"
+#include "core/hle/service/am/service/graphics_syncpoint_cache.h"
+#include "core/hle/service/am/service/native_image_layout.h"
+
+using Service::AM::GraphicsSupervisorProtocol::DecodeBootstrap;
+
+TEST_CASE("Graphics supervisor decodes the captured TIPC bootstrap", "[graphics-supervisor]") {
+ const std::array<u32, 16> packet{0x14, 0x8000000c, 2, 0xffff8001,
+ 0x51, 0, 5, 0x80000000};
+ const auto request = DecodeBootstrap(packet);
+ REQUIRE(request.has_value());
+ REQUIRE(request->process_handle == 0xffff8001);
+ REQUIRE(request->applet_resource_user_id == 0x51);
+ REQUIRE(request->flags == 0x8000000000000005ULL);
+ REQUIRE(request->parameters == std::array<u64, 4>{});
+ for (size_t length = 0; length < packet.size(); ++length) {
+ REQUIRE_FALSE(DecodeBootstrap(std::span{packet}.first(length)));
+ }
+}
+
+TEST_CASE("Graphics supervisor rejects incompatible packet layouts", "[graphics-supervisor]") {
+ const std::array<u32, 16> original{0x14, 0x8000000c, 2, 0xffff8001,
+ 0x51, 0, 5, 0x80000000};
+ for (const auto replacement : {0U, 0x15U, 0x10014U}) {
+ auto packet = original;
+ packet[0] = replacement;
+ REQUIRE_FALSE(DecodeBootstrap(packet));
+ }
+ for (const auto replacement : {0U, 0x8000000bU, 0x8000000dU}) {
+ auto packet = original;
+ packet[1] = replacement;
+ REQUIRE_FALSE(DecodeBootstrap(packet));
+ }
+ for (const auto replacement : {0U, 1U, 3U, 0x20U}) {
+ auto packet = original;
+ packet[2] = replacement;
+ REQUIRE_FALSE(DecodeBootstrap(packet));
+ }
+}
+
+
+TEST_CASE("Graphics syncpoint cache extends real counter values across wraparound", "[graphics-supervisor]") {
+ using namespace Service::AM::GraphicsSyncpointCache;
+ REQUIRE(ExtendCounter(Unallocated, 7) == 7);
+ REQUIRE(ExtendCounter(7, 12) == 12);
+ REQUIRE(ExtendCounter(0xfffffffeULL, 0xffffffffU) == 0xffffffffULL);
+ REQUIRE(ExtendCounter(0xffffffffULL, 0) == 0x100000000ULL);
+ REQUIRE(ExtendCounter(0x1ffffffffULL, 3) == 0x200000003ULL);
+ REQUIRE(Header[0] == 1);
+ REQUIRE(Header[2] + 1 == CounterCount);
+ REQUIRE(Header[3] % 8 == 0);
+ REQUIRE(Header[3] + CounterCount * 8 <= Header[5]);
+ REQUIRE(Header[7] + GroupCount * 8 <= 0x2000);
+}
+
+TEST_CASE("Graphics syncpoint cache retains wrap history across unallocated snapshots", "[graphics-supervisor]") {
+ using namespace Service::AM::GraphicsSyncpointCache;
+ CounterValue counter;
+ REQUIRE(counter.Observe(std::nullopt) == Unallocated);
+ REQUIRE(counter.Observe(0xfffffffeU) == 0xfffffffeULL);
+ REQUIRE(counter.Observe(1U) == 0x100000001ULL);
+ REQUIRE(counter.Observe(std::nullopt) == Unallocated);
+ REQUIRE(counter.Observe(3U) == 0x100000003ULL);
+}
+
+
+TEST_CASE("Native block image storage covers every padded GOB", "[graphics-supervisor]") {
+ using Service::AM::CalculateNativeImageRequirements;
+ std::array<u32, 16> preference{0, 2, 0x0437077f, 0, 7, 120, 1, 1};
+ const auto requirements = CalculateNativeImageRequirements(preference, 1);
+ REQUIRE(requirements.has_value());
+ REQUIRE(requirements->size == 8847360);
+ REQUIRE(requirements->alignment == 8192);
+ REQUIRE(requirements->size % requirements->alignment == 0);
+ preference[1] = 1; // The GPU descriptor uses the same block storage.
+ REQUIRE(CalculateNativeImageRequirements(preference, 1)->size == requirements->size);
+ preference[2] = 0; // A one-pixel image still occupies a complete block.
+ preference[5] = 1;
+ REQUIRE(CalculateNativeImageRequirements(preference, 1)->size == 8192);
+}
+
+TEST_CASE("Native block image sizing rejects unsupported layouts and undersized pitches", "[graphics-supervisor]") {
+ using Service::AM::CalculateNativeImageRequirements;
+ const std::array<u32, 16> original{0, 2, 0x0437077f, 0, 7, 120, 1, 1};
+ for (const auto count : {0U, 2U, 0xffffffffU}) {
+ REQUIRE_FALSE(CalculateNativeImageRequirements(original, count));
+ }
+ for (const auto index : {0, 1, 3, 6, 7, 8, 15}) {
+ auto preference = original;
+ preference[index] ^= 4;
+ REQUIRE_FALSE(CalculateNativeImageRequirements(preference, 1));
+ }
+ for (const auto block : {0U, 2U, 9U, 32U, 0xffffffffU}) {
+ auto preference = original;
+ preference[4] = block;
+ REQUIRE_FALSE(CalculateNativeImageRequirements(preference, 1));
+ }
+ for (const auto pitch : {0U, 119U, 4097U, 0xffffffffU}) {
+ auto preference = original;
+ preference[5] = pitch;
+ REQUIRE_FALSE(CalculateNativeImageRequirements(preference, 1));
+ }
+ REQUIRE_FALSE(CalculateNativeImageRequirements(std::span{original}.first(15), 1));
+}
+
+TEST_CASE("GPU memory registration ignores padding around its byte selectors", "[graphics-supervisor]") {
+ std::array<u32, 8> packet{0x75, 6, 0x1fdd6000, 0x21, 0x1fddcfff, 0x21, 0xaabbcc00, 0x101};
+ const auto request = Service::AM::DecodeGpuMemoryRegistration(packet);
+ REQUIRE(request.has_value());
+ REQUIRE(request->begin == 0x211fdd6000ULL);
+ REQUIRE(request->end == 0x211fddcfffULL);
+ REQUIRE(request->flags == 0);
+ REQUIRE(request->device == 1);
+ packet[7] = 0xffffff02;
+ REQUIRE(Service::AM::DecodeGpuMemoryRegistration(packet)->device == 2);
+ packet[0] = 0x76;
+ REQUIRE_FALSE(Service::AM::DecodeGpuMemoryRegistration(packet));
+ REQUIRE_FALSE(Service::AM::DecodeGpuMemoryRegistration(std::span<const u32>{packet.data(), 7}));
+}
+
+TEST_CASE("Native presentation slots ignore leftover IPC padding", "[graphics-supervisor]") {
+ using Service::AM::GraphicsSupervisorProtocol::DecodeBufferSlot;
+ REQUIRE(DecodeBufferSlot(0) == 0);
+ REQUIRE(DecodeBufferSlot(0x4f434601) == 1);
+ REQUIRE(DecodeBufferSlot(0xffffff0e) == 14);
+ REQUIRE(DecodeBufferSlot(0x4f43460f) == 15);
+}
diff --git a/src/tests/core/native_gpu_characteristics.cpp b/src/tests/core/native_gpu_characteristics.cpp
new file mode 100644
index 0000000000..cb79853ac2
--- /dev/null
+++ b/src/tests/core/native_gpu_characteristics.cpp
@@ -0,0 +1,389 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <algorithm>
+#include <array>
+#include <cstring>
+#include <memory>
+#include "video_core/macro.h"
+#include "video_core/native_qmd.h"
+#include "video_core/dirty_flags.h"
+#include <catch2/catch_test_macros.hpp>
+#include "core/hle/kernel/k_memory_block.h"
+#include "core/hle/service/nvdrv/devices/native_gpu_kind.h"
+#include "core/hle/service/nvdrv/devices/native_gpu_context.h"
+#include "core/hle/service/nvdrv/devices/native_gpu_characteristics.h"
+#include "core/hle/service/nvdrv/devices/gpu_address_space_layout.h"
+
+TEST_CASE("Native GPU characteristics replace topology with ioctl interface levels",
+ "[native-gpu]") {
+ std::array<u8, 0xa0> legacy;
+ legacy.fill(0xcc);
+ const u32_le arch{0x120};
+ const u32_le impl{0xb};
+ std::memcpy(legacy.data(), &arch, sizeof(arch));
+ std::memcpy(legacy.data() + 4, &impl, sizeof(impl));
+ const u32_le available{0x30000};
+ std::memcpy(legacy.data() + 0x34, &available, sizeof(available));
+ const auto native = Service::Nvidia::Devices::MakeNativeGpuCharacteristics(legacy);
+ REQUIRE(native.size() == 0xd0);
+ u32_le native_arch, native_impl;
+ std::memcpy(&native_arch, native.data(), sizeof(native_arch));
+ std::memcpy(&native_impl, native.data() + 4, sizeof(native_impl));
+ REQUIRE(static_cast<u32>(native_arch) == 0x170);
+ REQUIRE(static_cast<u32>(native_impl) == 0xe);
+ REQUIRE(std::equal(legacy.begin() + 8, legacy.begin() + 0x28, native.begin() + 8));
+ REQUIRE(std::equal(legacy.begin() + 0x2c, legacy.begin() + 0x34, native.begin() + 0x2c));
+ REQUIRE(std::equal(legacy.begin() + 0x38, legacy.begin() + 0x3c, native.begin() + 0x38));
+ REQUIRE(std::equal(legacy.begin() + 0x44, legacy.begin() + 0x50, native.begin() + 0x44));
+ REQUIRE(std::equal(legacy.begin() + 0x58, legacy.begin() + 0x70, native.begin() + 0x58));
+ u32_le sm_version, spa_version;
+ std::memcpy(&sm_version, native.data() + 0x3c, sizeof(sm_version));
+ std::memcpy(&spa_version, native.data() + 0x40, sizeof(spa_version));
+ REQUIRE(static_cast<u32>(sm_version) == 0x807);
+ REQUIRE(static_cast<u32>(spa_version) == 0x807);
+ u32_le available_sizes;
+ std::memcpy(&available_sizes, native.data() + 0x34, sizeof(available_sizes));
+ REQUIRE(static_cast<u32>(available_sizes) == 0x11000);
+ u32_le big_page_size;
+ std::memcpy(&big_page_size, native.data() + 0x28, sizeof(big_page_size));
+ REQUIRE(static_cast<u32>(big_page_size) == 0x1000);
+ const auto level = [&](size_t offset) {
+ u16_le value;
+ std::memcpy(&value, native.data() + offset, sizeof(value));
+ return static_cast<s16>(static_cast<u16>(value));
+ };
+ REQUIRE(level(0x70) == 0x1c);
+ REQUIRE(level(0x72) == 2);
+ u64_le flags;
+ std::memcpy(&flags, native.data() + 0x50, sizeof(flags));
+ REQUIRE((static_cast<u64>(flags) & (1ULL << 8)) != 0);
+ REQUIRE((static_cast<u64>(flags) & ~((1ULL << 8) | (1ULL << 26) | (1ULL << 34) | (0xfULL << 40))) == (0xccccccccccccccccULL & ~((1ULL << 8) | (1ULL << 26) | (1ULL << 34) | (0xfULL << 40))));
+ REQUIRE((static_cast<u64>(flags) & (0xfULL << 40)) == (0xfULL << 40));
+ REQUIRE((static_cast<u64>(flags) & (1ULL << 34)) != 0);
+ REQUIRE(level(0x74) == -1);
+ REQUIRE(level(0x76) == 0x1d);
+ // The captured SDK rejects the old topology value 2 here, requiring >= 6.
+ REQUIRE(level(0x78) == 0x14);
+ REQUIRE(std::all_of(native.begin() + 0x7a, native.end(), [](u8 b) { return b == 0; }));
+}
+
+TEST_CASE("Native GPU VA layout accommodates the captured NVN bootstrap reservation",
+ "[native-gpu]") {
+ using Service::Nvidia::Devices::GetGpuAddressSpaceLayout;
+ constexpr auto native = GetGpuAddressSpaceLayout(true);
+ constexpr auto legacy = GetGpuAddressSpaceLayout(false);
+ REQUIRE(native.split - native.start >= 0x7ff8000000ULL + (1ULL << 32));
+ REQUIRE(native.start == legacy.start);
+ REQUIRE(native.end == 1ULL << 40);
+ REQUIRE(native.end > native.split);
+ REQUIRE(legacy.split == 1ULL << 34);
+ REQUIRE(legacy.end == 1ULL << 37);
+}
+
+TEST_CASE("GPU region addresses retain bits above the 32-bit page index", "[native-gpu]") {
+ using Service::Nvidia::Devices::GpuPageAddress;
+ REQUIRE(GpuPageAddress(0x400000, 17) == 0x8000000000ULL);
+ REQUIRE(GpuPageAddress(0x20000, 17) == 0x400000000ULL);
+ REQUIRE(GpuPageAddress(0x8000, 12) == 0x8000000ULL);
+}
+
+TEST_CASE("GPU memory eligibility survives splitting and remains separate from cache attributes",
+ "[native-gpu]") {
+ using namespace Kernel;
+ KMemoryBlock right{KProcessAddress{0x10000}, 4, KMemoryState::Normal,
+ KMemoryPermission::UserReadWrite, KMemoryAttribute::Uncached};
+ right.UpdateAttribute(KMemoryAttribute::GpuAccessible, KMemoryAttribute::GpuAccessible);
+ KMemoryBlock left;
+ right.Split(&left, KProcessAddress{0x12000});
+ REQUIRE(left.GetNumPages() == 2);
+ REQUIRE(right.GetNumPages() == 2);
+ REQUIRE(left.GetMemoryInfo().GetSvcMemoryInfo().attribute ==
+ (Svc::MemoryAttribute::Uncached | Svc::MemoryAttribute::GpuAccessible));
+ REQUIRE(right.GetAttribute() == left.GetAttribute());
+ right.UpdateAttribute(KMemoryAttribute::GpuAccessible, KMemoryAttribute::None);
+ REQUIRE(right.GetAttribute() == KMemoryAttribute::Uncached);
+ REQUIRE(left.GetAttribute() ==
+ (KMemoryAttribute::Uncached | KMemoryAttribute::GpuAccessible));
+ REQUIRE(right.GetPermission() == KMemoryPermission::UserReadWrite);
+ REQUIRE(right.GetState() == KMemoryState::Normal);
+}
+
+TEST_CASE("Native 4 KiB allocation preference retains a separate large-page GPU pool", "[native-gpu]") {
+ using Service::Nvidia::Devices::GetGpuAddressSpaceBigPageSize;
+ REQUIRE(GetGpuAddressSpaceBigPageSize(true, 0x1000) == 0x10000);
+ REQUIRE(GetGpuAddressSpaceBigPageSize(true, 0x20000) == 0x20000);
+ REQUIRE(GetGpuAddressSpaceBigPageSize(false, 0x1000) == 0x1000);
+ REQUIRE(GetGpuAddressSpaceBigPageSize(false, 0) == 0);
+}
+
+TEST_CASE("Native generic memory kind is translated rather than treated as legacy depth memory",
+ "[native-gpu]") {
+ using Service::Nvidia::Devices::TranslateNativeUncompressedKind;
+ REQUIRE(TranslateNativeUncompressedKind(0, 0) == static_cast<u8>(Tegra::PTEKind::PITCH));
+ REQUIRE(TranslateNativeUncompressedKind(6, 6) == static_cast<u8>(Tegra::PTEKind::GENERIC_16BX2));
+ REQUIRE(TranslateNativeUncompressedKind(6, 6) != static_cast<u8>(Tegra::PTEKind::Z16_MS16_2C));
+ REQUIRE_FALSE(TranslateNativeUncompressedKind(8, 6));
+ REQUIRE_FALSE(TranslateNativeUncompressedKind(7, 7));
+ REQUIRE_FALSE(TranslateNativeUncompressedKind(0xfe, 0xfe));
+}
+
+TEST_CASE("Native mappings report the actual selected memory kind", "[native-gpu]") {
+ using Service::Nvidia::Devices::SelectNativeMemoryKind;
+ const auto normal = SelectNativeMemoryKind(6, 6);
+ REQUIRE(normal.has_value());
+ REQUIRE(normal->selected == 6);
+ REQUIRE(normal->backend == 0xfe);
+ const auto fallback = SelectNativeMemoryKind(8, 6);
+ REQUIRE(fallback.has_value());
+ REQUIRE(fallback->selected == 6);
+ REQUIRE(fallback->backend == normal->backend);
+ REQUIRE(SelectNativeMemoryKind(0, 0)->selected == 0);
+ const auto depth = SelectNativeMemoryKind(0xc, 3);
+ REQUIRE(depth.has_value());
+ REQUIRE(depth->selected == 3);
+ REQUIRE(depth->backend == static_cast<u8>(Tegra::PTEKind::S8Z24));
+ REQUIRE(SelectNativeMemoryKind(3, 3)->backend == depth->backend);
+ const auto depth16 = SelectNativeMemoryKind(0xb, 1);
+ REQUIRE(depth16.has_value());
+ REQUIRE(depth16->selected == 1);
+ REQUIRE(depth16->backend == static_cast<u8>(Tegra::PTEKind::Z16));
+ REQUIRE(SelectNativeMemoryKind(1, 1)->backend == depth16->backend);
+ REQUIRE_FALSE(SelectNativeMemoryKind(0xb, 6));
+ REQUIRE_FALSE(SelectNativeMemoryKind(0xc, 6));
+ REQUIRE_FALSE(SelectNativeMemoryKind(8, 8));
+ REQUIRE_FALSE(SelectNativeMemoryKind(8, 0));
+ REQUIRE_FALSE(SelectNativeMemoryKind(9, 6));
+}
+
+TEST_CASE("Native GPU context allocations classify explicit known engine identities", "[native-gpu]") {
+ using Service::Nvidia::Devices::GetNativeGpuContextSlot;
+ REQUIRE(GetNativeGpuContextSlot(0xc797) == 1U);
+ REQUIRE(GetNativeGpuContextSlot(0xc7c0) == 2U);
+ REQUIRE_FALSE(GetNativeGpuContextSlot(0xa143));
+ REQUIRE(GetNativeGpuContextSlot(0xc7b5) == 4U);
+ REQUIRE(GetNativeGpuContextSlot(0xc56f) == 5U);
+ REQUIRE_FALSE(GetNativeGpuContextSlot(0));
+ REQUIRE_FALSE(GetNativeGpuContextSlot(0xb197));
+ REQUIRE_FALSE(GetNativeGpuContextSlot(0xffffc797));
+ REQUIRE_FALSE(GetNativeGpuContextSlot(0xdeadbeef));
+}
+
+TEST_CASE("Native MME uploads use instruction addresses and bypass Maxwell code storage", "[native-gpu]") {
+ Tegra::MacroEngine engine{true};
+ engine.native_turing = true;
+ engine.ClearCode(8);
+ engine.AddCode(8, 0x11223344);
+ engine.AddCode(8, 0x55667788);
+ engine.AddCode(8, 0x99aabbcc);
+ REQUIRE(engine.native_code[8] == 0);
+ REQUIRE(engine.native_code[24] == 0x11223344);
+ REQUIRE(engine.native_code[25] == 0x55667788);
+ REQUIRE(engine.native_code[26] == 0x99aabbcc);
+ REQUIRE(engine.uploaded_macro_code.empty());
+ engine.ClearCode(0xffffffff);
+ engine.AddCode(0xffffffff, 1);
+ REQUIRE(engine.native_code[24] == 0x11223344);
+ engine.native_turing = false;
+ engine.AddCode(8, 7);
+ REQUIRE(engine.uploaded_macro_code.at(8).front() == 7);
+}
+
+TEST_CASE("Native MME loops consume the exact parameter count, including zero iterations", "[native-gpu]") {
+ Tegra::TuringMacroInterpreter interpreter;
+ constexpr std::array<u32, 15> code{
+ 0x00000003U, 0x18c00000U, 0x31c00300U,
+ 0x00000003U, 0x18c00000U, 0xb00c5f00U,
+ 0x00740003U, 0x18c00740U, 0x318c0300U,
+ 0x00000003U, 0x18c00000U, 0x318c0301U,
+ 0x00000003U, 0x18c00000U, 0x318c0300U,
+ };
+ std::vector<u32> output;
+ auto emit = [&](u32 method, u32 value) { REQUIRE(method == 0xd00); output.push_back(value); return true; };
+ auto read = [](u32) { return 0U; };
+ const std::array<u32, 1> zero{0};
+ auto result = interpreter.Execute(code, 0, zero, read, emit);
+ REQUIRE(result.success);
+ REQUIRE(result.consumed == 1);
+ REQUIRE(output.empty());
+ const std::array<u32, 3> two{2, 99, 88};
+ result = interpreter.Execute(code, 0, two, read, emit);
+ REQUIRE(result.success);
+ REQUIRE(result.consumed == 3);
+ REQUIRE(output == std::vector<u32>{99, 88});
+ result = interpreter.Execute(code, 0, std::span<const u32>{}, read, emit);
+ REQUIRE_FALSE(result.success);
+ REQUIRE(result.error == "parameter FIFO underflow");
+ result = interpreter.Execute(code, 5, zero, read, emit);
+ REQUIRE_FALSE(result.success);
+ REQUIRE(result.error == "instruction outside uploaded RAM");
+}
+
+TEST_CASE("Native MME pairs ALU carry with two incrementing outputs", "[native-gpu]") {
+ Tegra::TuringMacroInterpreter interpreter;
+ constexpr std::array<u32, 6> code{
+ 0x20340003U, 0x18084740U, 0x3bc00301U,
+ 0x00000003U, 0x18c00000U, 0x318c0300U,
+ };
+ const std::array<u32, 2> params{0xffffffffU, 1};
+ std::vector<std::pair<u32, u32>> writes;
+ const auto result = interpreter.Execute(code, 0, params, [](u32) { return 0U; },
+ [&](u32 method, u32 value) { writes.emplace_back(method, value); return true; });
+ REQUIRE(result.success);
+ REQUIRE(result.consumed == 2);
+ REQUIRE(writes == std::vector<std::pair<u32, u32>>{{0xd00, 0}, {0xd01, 1}});
+}
+
+TEST_CASE("Native empty QMDs release only after their dependency signals", "[native-gpu]") {
+ Tegra::NativeQmdScheduler scheduler;
+ Tegra::NativeQmdScheduler::Descriptor descriptor{};
+ descriptor[4] = 0x8040; // Captured QMD: two dependencies, no CTAs.
+ descriptor[18] = 0x00010030;
+ descriptor[24] = 0x100000;
+ descriptor[25] = 0x40880021; // One-word release, sysmembar, upper address 0x21.
+ descriptor[26] = 0;
+ u32 releases{};
+ const auto read = [&](u64, auto& out) { out = descriptor; return true; };
+ const auto release = [&](u64 address, u32 value) {
+ REQUIRE(address == 0x2100100000ULL);
+ REQUIRE(value == 42);
+ ++releases;
+ return true;
+ };
+ REQUIRE(scheduler.Signal(0x2100200000ULL, 3, read, release).empty());
+ REQUIRE(releases == 0);
+ REQUIRE(scheduler.Signal(0x2100200000ULL, 7, read, release).empty());
+ REQUIRE(releases == 0);
+ descriptor[26] = 42;
+ REQUIRE(scheduler.Signal(0x2100200000ULL, 7, read, release).empty());
+ REQUIRE(releases == 1);
+}
+
+TEST_CASE("Native QMD scheduler rejects real shader work without reporting completion", "[native-gpu]") {
+ Tegra::NativeQmdScheduler scheduler;
+ Tegra::NativeQmdScheduler::Descriptor descriptor{};
+ descriptor[4] = 1U << 14;
+ descriptor[18] = 0x30;
+ descriptor[12] = descriptor[13] = descriptor[14] = 1;
+ u32 releases{};
+ const auto result = scheduler.Signal(0x10000, 3,
+ [&](u64, auto& out) { out = descriptor; return true; },
+ [&](u64, u32) { ++releases; return true; });
+ REQUIRE(result == "native shader dispatch required");
+ REQUIRE(releases == 0);
+}
+
+TEST_CASE("Native dependent QMD signals survive until the child is uploaded", "[native-gpu]") {
+ Tegra::NativeQmdScheduler scheduler;
+ Tegra::NativeQmdScheduler::Descriptor parent{}, child{};
+ parent[4] = 1U << 14;
+ parent[18] = 0x30;
+ parent[15] = 0x200;
+ parent[16] = 9; // Enabled child decrement-dependence operation.
+ child[4] = 2U << 14;
+ child[18] = 0x30;
+ child[24] = 0x30000;
+ child[25] = 0x40880000;
+ child[26] = 99;
+ u32 releases{};
+ const auto read = [&](u64 address, auto& out) { out = address == 0x10000 ? parent : child; return true; };
+ const auto release = [&](u64 address, u32 value) {
+ REQUIRE(address == 0x30000);
+ REQUIRE(value == 99);
+ ++releases;
+ return true;
+ };
+ REQUIRE(scheduler.Signal(0x10000, 3, read, release).empty());
+ REQUIRE(scheduler.Signal(0x10000, 7, read, release).empty());
+ REQUIRE(releases == 0);
+ REQUIRE(scheduler.Signal(0x20000, 3, read, release).empty());
+ REQUIRE(releases == 0);
+ REQUIRE(scheduler.Signal(0x20000, 7, read, release).empty());
+ REQUIRE(releases == 1);
+}
+
+TEST_CASE("Native MME dual ALUs sample registers before either writeback", "[native-gpu]") {
+ Tegra::TuringMacroInterpreter interpreter;
+ constexpr std::array<u32, 12> code{
+ 0x00000003U, 0x18c00000U, 0x31c00300U,
+ 0x00000003U, 0x40c7c001U, 0xf1c00300U,
+ 0x00380483U, 0x18c00000U, 0xf190fb01U,
+ 0x00000003U, 0x18c00000U, 0x318c0300U,
+ };
+ const std::array<u32, 2> parameters{3, 0x1000000};
+ std::vector<std::pair<u32, u32>> writes;
+ const auto result = interpreter.Execute(code, 0, parameters, [](u32) { return 0U; },
+ [&](u32 method, u32 value) { writes.emplace_back(method, value); return true; });
+ REQUIRE(result.success);
+ REQUIRE(result.consumed == 2);
+ REQUIRE(writes == std::vector<std::pair<u32, u32>>{{0x120, 7}});
+}
+
+TEST_CASE("Native MME taken back edges suppress fallthrough termination", "[native-gpu]") {
+ Tegra::TuringMacroInterpreter interpreter;
+ constexpr std::array<u32, 12> code{
+ 0x00000003U, 0x18c00000U, 0x31c00300U,
+ 0x00383403U, 0x18c00000U, 0x72000b00U,
+ 0x00000003U, 0x18c017ffU, 0xf00c7701U,
+ 0x00000003U, 0x18c00000U, 0x318c0300U,
+ };
+ for (const u32 count : {1U, 3U, 16U}) {
+ std::vector<u32> writes;
+ const std::array<u32, 1> parameters{count};
+ const auto result = interpreter.Execute(code, 0, parameters, [](u32) { return 0U; },
+ [&](u32 method, u32 value) { REQUIRE(method == 0xd00); writes.push_back(value); return true; });
+ REQUIRE(result.success);
+ REQUIRE(result.consumed == 1);
+ REQUIRE(writes.size() == count);
+ REQUIRE(writes.front() == count - 1);
+ REQUIRE(writes.back() == 0);
+ }
+}
+
+TEST_CASE("Native compute QMD releases follow successful dispatch and retain reduction flags", "[native-gpu]") {
+ Tegra::NativeQmdScheduler scheduler;
+ Tegra::NativeQmdScheduler::Descriptor descriptor{};
+ descriptor[18] = 0x30;
+ descriptor[12] = descriptor[13] = descriptor[14] = 1;
+ descriptor[24] = 0x10000;
+ descriptor[25] = 0x44800021;
+ descriptor[26] = 1;
+ std::vector<int> order;
+ const auto read = [&](u64, auto& out) { out = descriptor; return true; };
+ const auto release = [&](u64 address, u32 value, u32 flags) {
+ REQUIRE(address == 0x2100010000ULL);
+ REQUIRE(value == 1);
+ REQUIRE(flags == 0x44800021);
+ order.push_back(2);
+ return true;
+ };
+ REQUIRE(scheduler.Signal(0x20000, 3, read, release,
+ [&](const auto&) { order.push_back(1); return false; }) == "native shader dispatch required");
+ REQUIRE(order == std::vector<int>{1});
+ order.clear();
+ REQUIRE(scheduler.Signal(0x20000, 3, read, release,
+ [&](const auto&) { order.push_back(1); return true; }).empty());
+ REQUIRE(order == std::vector<int>{1, 2});
+}
+
+TEST_CASE("Native vertex stream sizes use byte counts and dirty the selected stream", "[native-gpu]") {
+ using Maxwell3D = Tegra::Engines::Maxwell3D;
+ auto regs = std::make_unique<Maxwell3D::Regs>();
+ regs->reg_array.fill(0);
+ regs->reg_array[0x600 / 4 + 1] = 0x1b4e8;
+ REQUIRE(regs->NativeVertexStreamSize(0) == 111848);
+ REQUIRE(regs->vertex_stream_limits[0].Address() == 0);
+ regs->reg_array[0x600 / 4 + 31 * 2] = 0xffffff81;
+ regs->reg_array[0x600 / 4 + 31 * 2 + 1] = 0x76543210;
+ REQUIRE(regs->NativeVertexStreamSize(31) == 0x8176543210ULL);
+ REQUIRE(regs->NativeVertexStreamSize(1) == 0);
+ Maxwell3D::DirtyState::Tables tables{};
+ VideoCommon::Dirty::SetupDirtyFlags(tables);
+ for (u32 stream = 0; stream < 32; ++stream) {
+ for (u32 word = 0; word < 2; ++word) {
+ const u32 method = 0x600 / 4 + stream * 2 + word;
+ REQUIRE(tables[0][method] == VideoCommon::Dirty::VertexBuffer0 + stream);
+ REQUIRE(tables[1][method] == VideoCommon::Dirty::VertexBuffers);
+ }
+ }
+}
diff --git a/src/tests/core/program_metadata.cpp b/src/tests/core/program_metadata.cpp
new file mode 100644
index 0000000000..b5af9953f4
--- /dev/null
+++ b/src/tests/core/program_metadata.cpp
@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <cstring>
+#include <memory>
+#include <vector>
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/generators/catch_generators.hpp>
+
+#include "common/swap.h"
+#include "core/file_sys/program_metadata.h"
+#include "core/file_sys/romfs.h"
+#include "core/file_sys/vfs/vfs_vector.h"
+#include "core/loader/deconstructed_rom_directory.h"
+#include "core/loader/loader.h"
+
+namespace {
+template <typename T>
+void Put(std::vector<u8>& bytes, std::size_t offset, T value) {
+ std::memcpy(bytes.data() + offset, &value, sizeof(value));
+}
+
+std::vector<u8> MakeMetadata(bool extended) {
+ std::vector<u8> bytes(0x600);
+ std::memcpy(bytes.data(), "META", 4);
+ bytes[0xC] = 7;
+ Put(bytes, 0x1C, u32_le{0x100000});
+ Put(bytes, 0x70, u32_le{0x500});
+ Put(bytes, 0x74, u32_le{0x100});
+ Put(bytes, 0x78, u32_le{0x80});
+ Put(bytes, 0x7C, u32_le{0x380});
+ const std::size_t acid = 0x80 + (extended ? 0x300 : 0x200);
+ const u32 fac = extended ? 0x340 : 0x240;
+ std::memcpy(bytes.data() + acid, "ACID", 4);
+ Put(bytes, acid + 0x20, u32_le{fac});
+ Put(bytes, acid + 0x24, u32_le{0x2C});
+ Put(bytes, 0x80 + fac + 4, u64_le{0x1234});
+ std::memcpy(bytes.data() + 0x500, "ACI0", 4);
+ Put(bytes, 0x510, u64_le{0x0100123456780000});
+ Put(bytes, 0x520, u32_le{0x40});
+ Put(bytes, 0x524, u32_le{0x1C});
+ Put(bytes, 0x530, u32_le{0x80});
+ Put(bytes, 0x534, u32_le{4});
+ Put(bytes, 0x544, u64_le{0x5678});
+ Put(bytes, 0x580, u32_le{0x030043F7});
+ return bytes;
+}
+} // namespace
+
+TEST_CASE("Program metadata reads both ACID layouts and relative access offsets", "[loader][npdm]") {
+ const bool extended = GENERATE(false, true);
+ FileSys::ProgramMetadata metadata;
+ REQUIRE(metadata.Load(std::make_shared<FileSys::VectorVfsFile>(MakeMetadata(extended))) ==
+ Loader::ResultStatus::Success);
+ REQUIRE(metadata.UsesExtendedAcid() == extended);
+ REQUIRE(metadata.GetTitleID() == 0x0100123456780000);
+ REQUIRE(metadata.GetFilesystemPermissions() == 0x5678);
+ REQUIRE(metadata.GetMainThreadStackSize() == 0x100000);
+ REQUIRE(metadata.GetKernelCapabilities() == std::vector<u32>{0x030043F7});
+}
+
+TEST_CASE("Program metadata rejects invalid sections", "[loader][npdm]") {
+ const bool extended = GENERATE(false, true);
+ auto bytes = MakeMetadata(extended);
+ const std::size_t acid = 0x80 + (extended ? 0x300 : 0x200);
+ Loader::ResultStatus expected{};
+ SECTION("NPDM magic") {
+ bytes[0] = 0;
+ expected = Loader::ResultStatus::ErrorBadNPDMHeader;
+ }
+ SECTION("ACID past EOF") {
+ Put(bytes, 0x78, u32_le{0xFFFFFFF0});
+ expected = Loader::ResultStatus::ErrorBadACIDHeader;
+ }
+ SECTION("ACID magic") {
+ bytes[acid] = 0;
+ expected = Loader::ResultStatus::ErrorBadACIDHeader;
+ }
+ SECTION("ACI magic") {
+ bytes[0x500] = 0;
+ expected = Loader::ResultStatus::ErrorBadACIHeader;
+ }
+ SECTION("FAH outside its section") {
+ Put(bytes, 0x520, u32_le{0xF0});
+ expected = Loader::ResultStatus::ErrorBadFileAccessHeader;
+ }
+ SECTION("FAC outside its section") {
+ Put(bytes, acid + 0x20, u32_le{0xFFFFFFF0});
+ expected = Loader::ResultStatus::ErrorBadFileAccessControl;
+ }
+ SECTION("Truncated FAH") {
+ Put(bytes, 0x524, u32_le{0x10});
+ expected = Loader::ResultStatus::ErrorBadFileAccessHeader;
+ }
+ SECTION("Capabilities outside their section") {
+ Put(bytes, 0x530, u32_le{0x100});
+ expected = Loader::ResultStatus::ErrorBadKernelCapabilityDescriptors;
+ }
+ SECTION("Unaligned capability count") {
+ Put(bytes, 0x534, u32_le{3});
+ expected = Loader::ResultStatus::ErrorBadKernelCapabilityDescriptors;
+ }
+ FileSys::ProgramMetadata metadata;
+ REQUIRE(metadata.Load(std::make_shared<FileSys::VectorVfsFile>(std::move(bytes))) == expected);
+}
+
+TEST_CASE("Reloading legacy metadata clears extended ACID state", "[loader][npdm]") {
+ FileSys::ProgramMetadata metadata;
+ REQUIRE(metadata.Load(std::make_shared<FileSys::VectorVfsFile>(MakeMetadata(true))) ==
+ Loader::ResultStatus::Success);
+ REQUIRE(metadata.UsesExtendedAcid());
+ REQUIRE(metadata.Load(std::make_shared<FileSys::VectorVfsFile>(MakeMetadata(false))) ==
+ Loader::ResultStatus::Success);
+ REQUIRE_FALSE(metadata.UsesExtendedAcid());
+ REQUIRE(metadata.Load(nullptr) == Loader::ResultStatus::ErrorBadNPDMHeader);
+}
+
+TEST_CASE("ExeFS main takes precedence over standalone NSO", "[loader][exefs]") {
+ auto directory = std::make_shared<FileSys::VectorVfsDirectory>();
+ auto main = std::make_shared<FileSys::VectorVfsFile>(std::vector<u8>{'N', 'S', 'O', '0'},
+ "main", directory);
+ directory->AddFile(main);
+ directory->AddFile(std::make_shared<FileSys::VectorVfsFile>(MakeMetadata(false), "main.npdm"));
+ REQUIRE(Loader::IdentifyFile(main) == Loader::FileType::DeconstructedRomDirectory);
+ directory->DeleteFile("main.npdm");
+ REQUIRE(Loader::AppLoader_DeconstructedRomDirectory::IdentifyType(main) == Loader::FileType::Error);
+}
+
+TEST_CASE("Extracted ExeFS exposes loose RomFS", "[loader][exefs]") {
+ const bool sibling = GENERATE(false, true);
+ auto root = std::make_shared<FileSys::VectorVfsDirectory>();
+ auto exefs = std::make_shared<FileSys::VectorVfsDirectory>(
+ std::vector<FileSys::VirtualFile>{}, std::vector<FileSys::VirtualDir>{}, "exefs", root);
+ root->AddDirectory(exefs);
+ auto main = std::make_shared<FileSys::VectorVfsFile>(std::vector<u8>{'N', 'S', 'O', '0'},
+ "main", exefs);
+ exefs->AddFile(main);
+ exefs->AddFile(std::make_shared<FileSys::VectorVfsFile>(MakeMetadata(false), "main.npdm"));
+ auto romfs = std::make_shared<FileSys::VectorVfsDirectory>(
+ std::vector<FileSys::VirtualFile>{}, std::vector<FileSys::VirtualDir>{}, "romfs");
+ romfs->AddFile(std::make_shared<FileSys::VectorVfsFile>(std::vector<u8>{1, 2, 3}, "asset.bin"));
+ (sibling ? root : exefs)->AddDirectory(romfs);
+ Loader::AppLoader_DeconstructedRomDirectory loader(main);
+ FileSys::VirtualFile image;
+ REQUIRE(loader.ReadRomFS(image) == Loader::ResultStatus::Success);
+ const auto extracted = FileSys::ExtractRomFS(image);
+ REQUIRE(extracted != nullptr);
+ const auto asset = extracted->GetFile("asset.bin");
+ REQUIRE(asset != nullptr);
+ REQUIRE(asset->ReadAllBytes() == std::vector<u8>{1, 2, 3});
+}
diff --git a/src/tests/video_core/image_overlap.cpp b/src/tests/video_core/image_overlap.cpp
new file mode 100644
index 0000000000..a4a08c72ea
--- /dev/null
+++ b/src/tests/video_core/image_overlap.cpp
@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/generators/catch_generators.hpp>
+#include "video_core/texture_cache/image_base.h"
+#include "video_core/texture_cache/image_info.h"
+#include "video_core/texture_cache/image_view_info.h"
+#include "video_core/texture_cache/util.h"
+
+namespace {
+VideoCommon::ImageInfo ArrayInfo(bool native) {
+ VideoCommon::ImageInfo info;
+ info.type = VideoCommon::ImageType::e2D;
+ info.format = VideoCore::Surface::PixelFormat::A8B8G8R8_UNORM;
+ info.size = {64, 64, 1};
+ info.native_tiling = native;
+ info.layer_stride = VideoCommon::CalculateLayerStride(info);
+ info.maybe_unaligned_layer_stride = info.layer_stride;
+ return info;
+}
+}
+
+TEST_CASE("Repeated views inside an existing array do not grow its storage", "[texture-overlap]") {
+ const auto single = ArrayInfo(GENERATE(false, true));
+ auto storage = single;
+ storage.resources.layers = 3;
+ constexpr GPUVAddr gpu = 0x100000;
+ constexpr VAddr cpu = 0x200000;
+ for (u32 frame = 0; frame < 100; ++frame) {
+ const VideoCommon::ImageBase existing{storage, gpu, cpu};
+ const u32 offset = (1 + frame % 2) * single.layer_stride;
+ const auto joined = VideoCommon::ResolveOverlap(single, gpu + offset, cpu + offset,
+ existing, true, false, true);
+ REQUIRE(joined);
+ REQUIRE(joined->gpu_addr == gpu);
+ REQUIRE(joined->cpu_addr == cpu);
+ REQUIRE(joined->resources.layers == 3);
+ storage.resources = joined->resources;
+ REQUIRE(VideoCommon::CalculateUnswizzledSizeBytes(storage) == 64 * 64 * 4 * 3);
+ }
+}
+
+TEST_CASE("Array overlap extends only to the end of the new view", "[texture-overlap]") {
+ auto storage = ArrayInfo(GENERATE(false, true));
+ storage.resources.layers = 3;
+ auto view = storage;
+ view.resources.layers = 4;
+ constexpr GPUVAddr gpu = 0x100000;
+ constexpr VAddr cpu = 0x200000;
+ const VideoCommon::ImageBase existing{storage, gpu, cpu};
+ const u32 offset = 2 * storage.layer_stride;
+ const auto joined = VideoCommon::ResolveOverlap(view, gpu + offset, cpu + offset,
+ existing, true, false, true);
+ REQUIRE(joined);
+ REQUIRE(joined->resources.layers == 6);
+}
+
+TEST_CASE("Native texture views retain their direct address without a legacy array prefix", "[texture-overlap]") {
+ Tegra::Texture::TICEntry raw{};
+ raw.format.Assign(Tegra::Texture::TextureFormat::A8B8G8R8);
+ raw.r_type.Assign(Tegra::Texture::ComponentType::UNORM);
+ raw.g_type.Assign(Tegra::Texture::ComponentType::UNORM);
+ raw.b_type.Assign(Tegra::Texture::ComponentType::UNORM);
+ raw.a_type.Assign(Tegra::Texture::ComponentType::UNORM);
+ raw.header_version.Assign(Tegra::Texture::TICHeaderVersion::BlockLinear);
+ raw.block_height.Assign(4);
+ raw.width_minus_one.Assign(1919);
+ raw.height_minus_1.Assign(1079);
+ const bool array = GENERATE(false, true);
+ raw.texture_type.Assign(array ? Tegra::Texture::TextureType::Texture2DArray : Tegra::Texture::TextureType::Texture2D);
+ raw.depth_minus_1.Assign(array ? 2 : 0);
+ raw.normalized_coords.Assign(1);
+ raw.address_high.Assign(0x81);
+ raw.address_low = 0x870000;
+ raw.layer_base_0_2.Assign(1);
+ raw.layer_base_3_7.Assign(4);
+ REQUIRE(raw.BaseLayer() == 33);
+ REQUIRE(VideoCommon::TextureViewHeader(raw, false) == raw);
+ const auto native = VideoCommon::TextureViewHeader(raw, true);
+ REQUIRE(native.BaseLayer() == 0);
+ REQUIRE(native.Address() == raw.Address());
+ const VideoCommon::ImageInfo info(native);
+ REQUIRE(info.size.width == 1920);
+ REQUIRE(info.size.height == 1080);
+ REQUIRE(info.resources.layers == (array ? 3 : 1));
+ REQUIRE(VideoCommon::CalculateUnswizzledSizeBytes(info) == size_t{1920} * 1080 * 4 * (array ? 3U : 1U));
+ const VideoCommon::ImageViewInfo view(native, 0);
+ REQUIRE(view.range.base.layer == 0);
+ REQUIRE(view.type == (array ? VideoCommon::ImageViewType::e2DArray : VideoCommon::ImageViewType::e2D));
+ REQUIRE(view.range.extent.layers == (array ? 3 : 1));
+ REQUIRE(VideoCommon::ImageInfo(raw).resources.layers == (array ? 36 : 34));
+}
diff --git a/src/tests/video_core/native_semaphore.cpp b/src/tests/video_core/native_semaphore.cpp
new file mode 100644
index 0000000000..8dcafd91bc
--- /dev/null
+++ b/src/tests/video_core/native_semaphore.cpp
@@ -0,0 +1,63 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <array>
+#include <optional>
+#include <vector>
+#include <catch2/catch_test_macros.hpp>
+#include "common/common_types.h"
+#include "video_core/native_semaphore.h"
+
+TEST_CASE("Native semaphore predicates require the actual producer value", "[native-gpu]") {
+ const Tegra::NativeSemaphoreWait equal{0x1000, 840, 1};
+ REQUIRE_FALSE(equal.Matches(0));
+ REQUIRE_FALSE(equal.Matches(839));
+ REQUIRE(equal.Matches(840));
+ REQUIRE_FALSE(equal.Matches(841));
+ const Tegra::NativeSemaphoreWait gequal{0x1000, 840, 4};
+ REQUIRE_FALSE(gequal.Matches(0));
+ REQUIRE_FALSE(gequal.Matches(839));
+ REQUIRE(gequal.Matches(840));
+ REQUIRE(gequal.Matches(841));
+ const Tegra::NativeSemaphoreWait mask{0x1000, 0x24, 8};
+ REQUIRE_FALSE(mask.Matches(0));
+ REQUIRE_FALSE(mask.Matches(0x10));
+ REQUIRE(mask.Matches(4));
+ REQUIRE(mask.Matches(0x20));
+ REQUIRE_FALSE(Tegra::NativeSemaphoreWait::Supported(0));
+ REQUIRE_FALSE(Tegra::NativeSemaphoreWait::Supported(2));
+ REQUIRE_FALSE(Tegra::NativeSemaphoreWait{0x1000, 0, 2}.Matches(0));
+}
+
+TEST_CASE("Native channels resume in dependency order without completing blocked work", "[native-gpu]") {
+ const std::array<size_t, 3> channels{0, 1, 2};
+ std::array<std::optional<Tegra::NativeSemaphoreWait>, 3> waits{
+ Tegra::NativeSemaphoreWait{0x1000, 1, 4},
+ Tegra::NativeSemaphoreWait{0x2000, 1, 1},
+ Tegra::NativeSemaphoreWait{0x3000, 1, 4},
+ };
+ std::array<u32, 3> words{};
+ std::array<u32, 3> following_commands{};
+ std::vector<size_t> order;
+ const auto ready = [&](size_t channel) {
+ return waits[channel] && waits[channel]->Matches(words[channel]);
+ };
+ const auto dispatch = [&](size_t channel) {
+ REQUIRE(ready(channel));
+ waits[channel].reset();
+ ++following_commands[channel];
+ order.push_back(channel);
+ if (channel) words[channel - 1] = 1;
+ };
+ REQUIRE(Tegra::ResumeReadyNativeChannels(channels, ready, dispatch) == 0);
+ REQUIRE(order.empty());
+ REQUIRE(following_commands == std::array<u32, 3>{0, 0, 0});
+ // A real independent producer releases the last channel. Its remaining
+ // FIFO commands release the next channel, then the original consumer.
+ words[2] = 1;
+ REQUIRE(Tegra::ResumeReadyNativeChannels(channels, ready, dispatch) == 3);
+ REQUIRE(order == std::vector<size_t>{2, 1, 0});
+ REQUIRE(following_commands == std::array<u32, 3>{1, 1, 1});
+ REQUIRE(Tegra::ResumeReadyNativeChannels(channels, ready, dispatch) == 0);
+ REQUIRE(following_commands == std::array<u32, 3>{1, 1, 1});
+}
diff --git a/src/tests/video_core/native_shader.cpp b/src/tests/video_core/native_shader.cpp
new file mode 100644
index 0000000000..76424b7dbf
--- /dev/null
+++ b/src/tests/video_core/native_shader.cpp
@@ -0,0 +1,2841 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <bit>
+#include <algorithm>
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+#include <limits>
+#include <unordered_map>
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/generators/catch_generators.hpp>
+#include "video_core/texture_cache/format_lookup_table.h"
+#include "shader_recompiler/environment.h"
+#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h"
+#include "shader_recompiler/frontend/ampere/program_override.h"
+#include "shader_recompiler/frontend/ampere/program_capture.h"
+#include "shader_recompiler/frontend/ampere/program_address_map.h"
+#include "shader_recompiler/frontend/ir/basic_block.h"
+#include "shader_recompiler/frontend/ir/modifiers.h"
+#include "shader_recompiler/frontend/maxwell/decode.h"
+#include "shader_recompiler/frontend/maxwell/instruction.h"
+#include "shader_recompiler/frontend/maxwell/translate_program.h"
+#include "shader_recompiler/host_translate_info.h"
+#include "shader_recompiler/ir_opt/passes.h"
+#include "shader_recompiler/frontend/maxwell/translate/impl/impl.h"
+
+namespace {
+namespace IR = Shader::IR;
+using Native = Shader::Ampere::Instruction;
+
+class EmptyEnvironment : public Shader::Environment {
+public:
+ u64 ReadInstruction(u32) override { throw std::logic_error("unexpected instruction read"); }
+ u32 ReadCbufValue(u32, u32) override { throw std::logic_error("unexpected constant folding"); }
+ Shader::TextureType ReadTextureType(u32) override { return {}; }
+ Shader::TexturePixelFormat ReadTexturePixelFormat(u32) override { return {}; }
+ bool IsTexturePixelFormatInteger(u32) override { return false; }
+ u32 ReadViewportTransformState() override { return 1; }
+ u32 TextureBoundBuffer() const override { return 0; }
+ u32 LocalMemorySize() const override { return 0; }
+ u32 SharedMemorySize() const override { return 0; }
+ std::array<u32, 3> WorkgroupSize() const override { return {}; }
+ bool HasHLEMacroState() const override { return false; }
+ std::optional<Shader::ReplaceConstant> GetReplaceConstBuffer(u32, u32) override { return {}; }
+ void Dump(u64, u64) override {}
+};
+
+constexpr std::array<Native, 7> VertexTransform{{
+ {{0xff007321, 0x000080ff, 0x00000400, 0x000e2200}}, // load two input components
+ {{0x00037a02, 0x00c00000, 0x00000f00, 0x000fe400}}, // scale X
+ {{0x00027a02, 0x00c00100, 0x00000f00, 0x000fe200}}, // scale Y
+ {{0x00007623, 0x00c00200, 0x00010003, 0x001fe200}}, // X * scale X + offset X
+ {{0x01017623, 0x00c00300, 0x00010002, 0x000fc400}}, // Y * scale Y + offset Y
+ {{0xff007322, 0x00007000, 0x000004ff, 0x000fe200}}, // store position X/Y
+ {{0x0000794d, 0x00000000, 0x03800000, 0x000fea00}},
+}};
+
+// Evaluate the existing frontend's emitted IR, independently of native bit
+// decoding, to check actual attribute/constant/register data flow.
+std::array<float, 2> EvaluateVertex(IR::Block& block, std::array<float, 2> input,
+ std::array<float, 4> constants, s32 vertex_id = 0) {
+ std::unordered_map<const IR::Inst*, u32> values;
+ std::unordered_map<const IR::Inst*, std::array<u32, 2>> pairs;
+ std::array<u32, 256> regs{};
+ std::array<float, 2> output{};
+ const auto get = [&](const IR::Value& value) {
+ if (!value.IsImmediate()) return values.at(value.Inst());
+ if (value.Type() == IR::Type::U1) return static_cast<u32>(value.U1());
+ return value.Type() == IR::Type::F32 ? std::bit_cast<u32>(value.F32()) : value.U32();
+ };
+ for (const auto& inst : block.Instructions()) {
+ const auto arg = [&](size_t i) { return get(inst.Arg(i)); };
+ switch (inst.GetOpcode()) {
+ case IR::Opcode::GetRegister:
+ values[&inst] = regs[static_cast<u32>(inst.Arg(0).Reg())];
+ break;
+ case IR::Opcode::SetRegister:
+ if (inst.Arg(0).Reg() != IR::Reg::RZ) regs[static_cast<u32>(inst.Arg(0).Reg())] = arg(1);
+ break;
+ case IR::Opcode::GetAttributeU32:
+ REQUIRE(static_cast<u32>(inst.Arg(0).Attribute()) == 0x2fc / 4);
+ values[&inst] = static_cast<u32>(vertex_id);
+ break;
+ case IR::Opcode::ShiftLeftLogical32:
+ values[&inst] = arg(0) << arg(1);
+ break;
+ case IR::Opcode::ShiftRightArithmetic32:
+ values[&inst] = static_cast<u32>(static_cast<s32>(arg(0)) >> arg(1));
+ break;
+ case IR::Opcode::SLessThan:
+ values[&inst] = static_cast<s32>(arg(0)) < static_cast<s32>(arg(1));
+ break;
+ case IR::Opcode::ULessThan:
+ values[&inst] = arg(0) < arg(1);
+ break;
+ case IR::Opcode::SelectU32:
+ values[&inst] = arg(0) ? arg(1) : arg(2);
+ break;
+ case IR::Opcode::SMin32:
+ values[&inst] = static_cast<u32>(std::min(static_cast<s32>(arg(0)), static_cast<s32>(arg(1))));
+ break;
+ case IR::Opcode::SMax32:
+ values[&inst] = static_cast<u32>(std::max(static_cast<s32>(arg(0)), static_cast<s32>(arg(1))));
+ break;
+ case IR::Opcode::INeg32:
+ values[&inst] = 0U - arg(0);
+ break;
+ case IR::Opcode::IAdd32:
+ values[&inst] = arg(0) + arg(1);
+ break;
+ case IR::Opcode::ConvertF32S32:
+ values[&inst] = std::bit_cast<u32>(static_cast<float>(static_cast<s32>(arg(0))));
+ break;
+ case IR::Opcode::GetAttribute: {
+ const auto attr = static_cast<u32>(inst.Arg(0).Attribute());
+ if (attr == 0x2fc / 4) {
+ values[&inst] = static_cast<u32>(vertex_id);
+ break;
+ }
+ REQUIRE(attr >= 0x80 / 4);
+ REQUIRE(attr < 0x88 / 4);
+ values[&inst] = std::bit_cast<u32>(input[attr - 0x80 / 4]);
+ break;
+ }
+ case IR::Opcode::GetCbufU32x2:
+ REQUIRE(arg(0) == 3);
+ REQUIRE(arg(1) % 4 == 0);
+ REQUIRE(arg(1) + 8 <= sizeof(constants));
+ pairs[&inst] = {std::bit_cast<u32>(constants[arg(1) / 4]), std::bit_cast<u32>(constants[arg(1) / 4 + 1])};
+ break;
+ case IR::Opcode::CompositeExtractU32x2:
+ values[&inst] = pairs.at(inst.Arg(0).Inst()).at(arg(1));
+ break;
+ case IR::Opcode::GetCbufF32:
+ case IR::Opcode::GetCbufU32:
+ REQUIRE(arg(0) == 3);
+ REQUIRE(arg(1) < 16);
+ values[&inst] = std::bit_cast<u32>(constants[arg(1) / 4]);
+ break;
+ case IR::Opcode::BitCastF32U32:
+ case IR::Opcode::BitCastU32F32:
+ values[&inst] = arg(0);
+ break;
+ case IR::Opcode::LogicalAnd:
+ values[&inst] = arg(0) && arg(1);
+ break;
+ case IR::Opcode::LogicalOr:
+ values[&inst] = arg(0) || arg(1);
+ break;
+ case IR::Opcode::LogicalNot:
+ values[&inst] = !arg(0);
+ break;
+ case IR::Opcode::FPIsNan32:
+ values[&inst] = std::isnan(std::bit_cast<float>(arg(0)));
+ break;
+ case IR::Opcode::FPOrdGreaterThanEqual32:
+ values[&inst] = std::bit_cast<float>(arg(0)) >= std::bit_cast<float>(arg(1));
+ break;
+ case IR::Opcode::FPRoundEven32:
+ values[&inst] = std::bit_cast<u32>(std::nearbyint(std::bit_cast<float>(arg(0))));
+ break;
+ case IR::Opcode::FPFloor32:
+ values[&inst] = std::bit_cast<u32>(std::floor(std::bit_cast<float>(arg(0))));
+ break;
+ case IR::Opcode::FPTrunc32:
+ values[&inst] = std::bit_cast<u32>(std::trunc(std::bit_cast<float>(arg(0))));
+ break;
+ case IR::Opcode::FPClamp32:
+ values[&inst] = std::bit_cast<u32>(std::clamp(std::bit_cast<float>(arg(0)),
+ std::bit_cast<float>(arg(1)), std::bit_cast<float>(arg(2))));
+ break;
+ case IR::Opcode::ConvertU32F32:
+ values[&inst] = static_cast<u32>(std::bit_cast<float>(arg(0)));
+ break;
+ case IR::Opcode::ConvertF32U32:
+ values[&inst] = std::bit_cast<u32>(static_cast<float>(arg(0)));
+ break;
+ case IR::Opcode::BitFieldUExtract:
+ values[&inst] = (arg(0) >> arg(1)) & ((1U << arg(2)) - 1);
+ break;
+ case IR::Opcode::BitwiseOr32:
+ values[&inst] = arg(0) | arg(1);
+ break;
+ case IR::Opcode::FPOrdLessThan32:
+ values[&inst] = std::bit_cast<float>(arg(0)) < std::bit_cast<float>(arg(1));
+ break;
+ case IR::Opcode::BitwiseXor32:
+ values[&inst] = arg(0) ^ arg(1);
+ break;
+ case IR::Opcode::BitwiseNot32:
+ values[&inst] = ~arg(0);
+ break;
+ case IR::Opcode::BitwiseAnd32:
+ values[&inst] = arg(0) & arg(1);
+ break;
+ case IR::Opcode::FPAdd32:
+ values[&inst] = std::bit_cast<u32>(std::bit_cast<float>(arg(0)) + std::bit_cast<float>(arg(1)));
+ break;
+ case IR::Opcode::FPMul32:
+ values[&inst] = std::bit_cast<u32>(std::bit_cast<float>(arg(0)) * std::bit_cast<float>(arg(1)));
+ break;
+ case IR::Opcode::FPMin32:
+ values[&inst] = std::bit_cast<u32>(std::fmin(std::bit_cast<float>(arg(0)), std::bit_cast<float>(arg(1))));
+ break;
+ case IR::Opcode::FPMax32:
+ values[&inst] = std::bit_cast<u32>(std::fmax(std::bit_cast<float>(arg(0)), std::bit_cast<float>(arg(1))));
+ break;
+ case IR::Opcode::SelectF32:
+ values[&inst] = arg(0) ? arg(1) : arg(2);
+ break;
+ case IR::Opcode::FPFma32: {
+ const auto control = inst.Flags<IR::FpControl>();
+ REQUIRE(control.fmz_mode == IR::FmzMode::FTZ);
+ REQUIRE(control.rounding == IR::FpRounding::RN);
+ values[&inst] = std::bit_cast<u32>(std::fma(std::bit_cast<float>(arg(0)),
+ std::bit_cast<float>(arg(1)), std::bit_cast<float>(arg(2))));
+ break;
+ }
+ case IR::Opcode::SetAttribute: {
+ const auto attr = static_cast<u32>(inst.Arg(0).Attribute());
+ REQUIRE(attr >= 0x70 / 4);
+ REQUIRE(attr < 0x78 / 4);
+ output[attr - 0x70 / 4] = std::bit_cast<float>(arg(1));
+ break;
+ }
+ default:
+ FAIL("Unexpected IR operation " << IR::NameOf(inst.GetOpcode()));
+ }
+ }
+ return output;
+}
+} // namespace
+
+TEST_CASE("Native vertex lowering preserves attribute transforms through existing shader IR", "[native-shader]") {
+ const auto lowered = Shader::Ampere::LowerToMaxwell(VertexTransform);
+ REQUIRE(lowered);
+ REQUIRE(lowered.source_size == VertexTransform.size() * 16);
+ REQUIRE(lowered.instructions.size() >= VertexTransform.size());
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const u64 word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::ALD: visitor.ALD(word); break;
+ case Opcode::AST: visitor.AST(word); break;
+ case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break;
+ case Opcode::FFMA_rc: visitor.FFMA_rc(word); break;
+ case Opcode::FFMA_reg: visitor.FFMA_reg(word); break;
+ case Opcode::MOV_reg: visitor.MOV_reg(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected lowered vertex instruction");
+ }
+ }
+ REQUIRE(EvaluateVertex(block, {2, 4}, {3, 7, 5, 11}) == std::array<float, 2>{11, 39});
+ REQUIRE(EvaluateVertex(block, {-4, 0.5f}, {-2, 4, 1, -3}) == std::array<float, 2>{9, -1});
+}
+
+TEST_CASE("Native lowering rejects unsupported forms without a partial executable", "[native-shader]") {
+ auto code = VertexTransform;
+ code[3].words[2] |= 1U << 12; // DNZ has no accepted lowering yet.
+ auto result = Shader::Ampere::LowerToMaxwell(code);
+ REQUIRE_FALSE(result);
+ REQUIRE(result.error_offset == 3 * 16);
+ REQUIRE(result.instructions.empty());
+ code = VertexTransform;
+ code[1].words[2] &= ~(15U << 8); // Partial quad-lane mask.
+ result = Shader::Ampere::LowerToMaxwell(code);
+ REQUIRE_FALSE(result);
+ REQUIRE(result.instructions.empty());
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell({}));
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(std::span{VertexTransform}.first(6)));
+}
+
+TEST_CASE("Native instruction predicates survive lowering", "[native-shader]") {
+ auto code = VertexTransform;
+ code[3].words[0] = (code[3].words[0] & ~0xf000U) | 0x9000U;
+ const auto result = Shader::Ampere::LowerToMaxwell(code);
+ REQUIRE(result);
+ const auto pred = Shader::Maxwell::Instruction{result.instructions[result.source_instruction_offsets[3]]}.Pred();
+ REQUIRE(pred.index == 1);
+ REQUIRE(pred.negated);
+}
+
+// These private input fixtures are read from the investigation directory, not
+// embedded or distributed with the emulator's source tree.
+TEST_CASE("Captured native shader pair compiles through existing control flow and shader IR", "[.native-shader-captured]") {
+ const char* directory = std::getenv("EDEN_NATIVE_SHADER_FIXTURES");
+ REQUIRE(directory != nullptr);
+ class FixtureEnvironment final : public EmptyEnvironment {
+ public:
+ FixtureEnvironment(const std::vector<u64>& code_, std::span<const u8> header, Shader::Stage stage_)
+ : code{code_} {
+ std::memcpy(&sph, header.data(), sizeof(sph));
+ stage = stage_;
+ start_address = 0;
+ is_proprietary_driver = true;
+ }
+ u64 ReadInstruction(u32 address) override { return code.at(address / 8); }
+ u32 TranslateProgramAddress(u32 address) const override {
+ const auto translated = Shader::Ampere::NativeProgramAddress(code, address);
+ if (!translated) throw std::logic_error("invalid private native branch target " + std::to_string(address));
+ return *translated;
+ }
+ u32 LocalMemorySize() const override { return static_cast<u32>(sph.LocalMemorySize()) + sph.common3.shader_local_memory_crs_size; }
+ u32 TextureBoundBuffer() const override { return 2; }
+ u32 ReadCbufValue(u32 bank, u32 offset) override {
+ if (bank == 1) {
+ REQUIRE(offset % 4 == 0);
+ return cbuf1.at(offset / 4);
+ }
+ REQUIRE(bank == 2);
+ return 0; // Synthetic texture handle; only compilation is tested here.
+ }
+ Shader::TextureType ReadTextureType(u32) override { return Shader::TextureType::Color2D; }
+ const std::vector<u64>& code;
+ std::vector<u32> cbuf1;
+ };
+ for (const int stage : {1, 5}) {
+ const auto path = std::filesystem::path{directory} / ("native-graphics-shader-" + std::to_string(stage) + ".bin");
+ std::ifstream file(path, std::ios::binary | std::ios::ate);
+ REQUIRE(file.good());
+ const auto size = static_cast<size_t>(file.tellg());
+ REQUIRE(size >= 0x90);
+ std::vector<u8> bytes(size);
+ file.seekg(0);
+ file.read(reinterpret_cast<char*>(bytes.data()), bytes.size());
+ std::vector<Native> input((size - 0x80) / 16);
+ std::memcpy(input.data(), bytes.data() + 0x80, input.size() * 16);
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ INFO("stage=" << stage << " offset=" << lowered.error_offset << " error=" << lowered.error);
+ REQUIRE(lowered);
+ Shader::Ampere::ProgramCaptureState capture;
+ u32 captured_size{};
+ for (u32 i = 0; i < input.size(); ++i) {
+ const auto status = capture.Observe(input[i], i);
+ REQUIRE(status != Shader::Ampere::ProgramCaptureState::Result::InvalidTarget);
+ if (status == Shader::Ampere::ProgramCaptureState::Result::Complete) {
+ captured_size = (i + 1) * sizeof(Native);
+ break;
+ }
+ }
+ REQUIRE(captured_size == lowered.source_size);
+ std::vector<u64> code(0x50 / 8);
+ for (const u64 word : lowered.instructions) {
+ if ((code.size() * 8) % 32 == 0) code.push_back(0);
+ code.push_back(word);
+ }
+ code.push_back(0);
+ Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets);
+ FixtureEnvironment env{code, std::span{bytes}.first(0x50), stage == 1 ? Shader::Stage::VertexB : Shader::Stage::Fragment};
+ auto constants_path = std::filesystem::path{directory} / ("cbuf1-" + std::to_string(stage) + ".bin");
+ if (!std::filesystem::exists(constants_path)) constants_path = std::filesystem::path{directory} / "cbuf1.bin";
+ std::ifstream constants(constants_path, std::ios::binary | std::ios::ate);
+ if (constants) {
+ env.cbuf1.resize(static_cast<size_t>(constants.tellg()) / 4);
+ constants.seekg(0);
+ constants.read(reinterpret_cast<char*>(env.cbuf1.data()), env.cbuf1.size() * 4);
+ }
+ Shader::ObjectPool<Shader::Maxwell::Flow::Block> flow_pool;
+ Shader::ObjectPool<IR::Inst> inst_pool;
+ Shader::ObjectPool<IR::Block> block_pool;
+ Shader::Maxwell::Flow::CFG cfg{env, flow_pool, 0x50};
+ Shader::HostTranslateInfo host;
+ host.support_int64 = true;
+ host.support_float64 = true;
+ host.support_float16 = true;
+ const auto program = Shader::Maxwell::TranslateProgram(inst_pool, block_pool, env, cfg, host);
+ REQUIRE(program.stage == env.ShaderStage());
+ REQUIRE_FALSE(program.blocks.empty());
+ REQUIRE_FALSE(program.post_order_blocks.empty());
+ }
+}
+
+TEST_CASE("Native scalar texture sampling preserves coordinates and split outputs", "[native-shader]") {
+ std::array<Native, 2> input{{
+ {{0x04007b60, 0x30800805, 0x041e0f02, 0x100fc200}},
+ VertexTransform.back(),
+ }};
+ const bool predicated = GENERATE(false, true);
+ if (predicated) input[0].words[0] = (input[0].words[0] & ~0xf000U) | 0x3000;
+ const bool explicit_lod = GENERATE(false, true);
+ if (explicit_lod) input[0].words[2] |= 1U << 23;
+ const auto result = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(result);
+ REQUIRE(result.required_texture_buffer == 2);
+ REQUIRE(Shader::Maxwell::Instruction{result.instructions.front()}.Pred().index == (predicated ? 3U : 7U));
+ REQUIRE_FALSE(Shader::Maxwell::Instruction{result.instructions.front()}.Pred().negated);
+ REQUIRE(Shader::Maxwell::Decode(result.instructions.front()) == Shader::Maxwell::Opcode::TEXS);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.TEXS(result.instructions.front());
+ std::vector<u32> reads;
+ std::vector<u32> writes;
+ u32 samples = 0;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::GetRegister) reads.push_back(static_cast<u32>(inst.Arg(0).Reg()));
+ if (inst.GetOpcode() == IR::Opcode::SetRegister) writes.push_back(static_cast<u32>(inst.Arg(0).Reg()));
+ if (inst.GetOpcode() == (explicit_lod ? IR::Opcode::BoundImageSampleExplicitLod : IR::Opcode::BoundImageSampleImplicitLod)) {
+ ++samples;
+ if (explicit_lod) REQUIRE(inst.Arg(2).F32() == 0.0f);
+ const auto* coords = inst.Arg(1).Inst();
+ REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructF32x2);
+ for (size_t component = 0; component < 2; ++component) {
+ const auto* cast = coords->Arg(component).Inst();
+ REQUIRE(cast->GetOpcode() == IR::Opcode::BitCastF32U32);
+ const auto* reg = cast->Arg(0).Inst();
+ REQUIRE(reg->GetOpcode() == IR::Opcode::GetRegister);
+ REQUIRE(static_cast<u32>(reg->Arg(0).Reg()) == 4 + component);
+ }
+ REQUIRE(inst.Arg(0).U32() == 0x20);
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type == Shader::TextureType::Color2D);
+ }
+ }
+ REQUIRE(samples == 1);
+ std::ranges::sort(reads);
+ REQUIRE(reads == std::vector<u32>{4, 5});
+ REQUIRE(writes == std::vector<u32>{0, 1, 2, 3});
+ // Reject unsupported LOD modes, a different cbuf bank, and sparse results.
+ for (u32 bit : {89U, 54U, 81U}) {
+ auto unsupported = input;
+ unsupported[0].words[bit / 32] ^= 1U << (bit % 32);
+ const auto rejected = Shader::Ampere::LowerToMaxwell(unsupported);
+ REQUIRE_FALSE(rejected);
+ REQUIRE(rejected.instructions.empty());
+ }
+}
+
+TEST_CASE("Native texel fetch keeps integer coordinates and RGB outputs", "[native-shader]") {
+ std::array<Native, 3> input{{
+ {{0x00027305, 0x00000004, 0x0021f100, 0x001ff000}},
+ {{0x02007b66, 0x30800a05, 0x049e0702, 0x101fc200}},
+ VertexTransform.back(),
+ }};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ REQUIRE(lowered.required_texture_buffer == 2);
+ REQUIRE(Shader::Maxwell::Decode(lowered.instructions[0]) == Shader::Maxwell::Opcode::F2I_reg);
+ REQUIRE(Shader::Maxwell::Decode(lowered.instructions[1]) == Shader::Maxwell::Opcode::TLDS);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.F2I_reg(lowered.instructions[0]);
+ visitor.TLDS(lowered.instructions[1]);
+ std::vector<u32> writes;
+ u32 fetches = 0;
+ bool truncates = false;
+ bool handles_nan = false;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::SetRegister) writes.push_back(static_cast<u32>(inst.Arg(0).Reg()));
+ if (inst.GetOpcode() == IR::Opcode::FPTrunc32) {
+ truncates = true;
+ REQUIRE(inst.Flags<IR::FpControl>().fmz_mode == IR::FmzMode::FTZ);
+ }
+ if (inst.GetOpcode() == IR::Opcode::FPIsNan32) handles_nan = true;
+ if (inst.GetOpcode() == IR::Opcode::BoundImageFetch) {
+ ++fetches;
+ REQUIRE(inst.Arg(0).U32() == 0x28);
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type == Shader::TextureType::Color2D);
+ const auto* coords = inst.Arg(1).Inst();
+ REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructU32x2);
+ REQUIRE(coords->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{2});
+ REQUIRE(coords->Arg(1).Inst()->Arg(0).Reg() == IR::Reg{5});
+ REQUIRE(inst.Arg(3).U32() == 0);
+ }
+ }
+ REQUIRE(fetches == 1);
+ REQUIRE(truncates);
+ REQUIRE(handles_nan);
+ REQUIRE(writes == std::vector<u32>{2, 0, 1, 2});
+ auto unsupported = input;
+ unsupported[0].words[2] ^= 1U << 13; // NTZ is required by this conversion lowering.
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(unsupported));
+ unsupported = input;
+ unsupported[1].words[2] ^= 1U << 14; // multisample fetch is not implemented here.
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(unsupported));
+}
+
+TEST_CASE("Native texture component masks preserve sampled components", "[native-shader]") {
+ for (const u32 mask : {1U, 2U, 4U, 8U, 3U, 9U, 10U, 12U, 7U, 11U, 13U, 14U, 15U}) {
+ const bool split = std::popcount(mask) > 2;
+ std::array<Native, 2> input{{
+ {{0x04007b60, 0x30800805, 0x041e0f02, 0x100fc200}}, VertexTransform.back(),
+ }};
+ input[0].words[2] = (input[0].words[2] & ~0xfffU) | (split ? 6U : 255U) | (mask << 8);
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.TEXS(lowered.instructions.front());
+ std::vector<u32> extracted;
+ std::vector<u32> writes;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::CompositeExtractF32x4) extracted.push_back(inst.Arg(1).U32());
+ if (inst.GetOpcode() == IR::Opcode::SetRegister) writes.push_back(static_cast<u32>(inst.Arg(0).Reg()));
+ }
+ std::vector<u32> expected;
+ for (u32 component = 0; component < 4; ++component) if (mask & (1U << component)) expected.push_back(component);
+ REQUIRE(extracted == expected);
+ REQUIRE(writes.size() == expected.size());
+ for (u32 i = 0; i < writes.size(); ++i) REQUIRE(writes[i] == (i < 2 ? i : i + 4));
+ }
+}
+
+TEST_CASE("Native half packing and arithmetic preserve lane order numerically", "[native-shader]") {
+ const std::array<Native, 15> input{{
+ {{0x0102723e, 0x00000000, 0x000000ff, 0}}, // PACK_AB R2, R1, R0
+ {{0x02027232, 0x00000002, 0x00000000, 0}}, // HMUL2 R2, R2, R2
+ {{0xff037230, 0x20000002, 0x00004100, 0}}, // extract low half to F32 R3
+ {{0xff047230, 0x30000002, 0x00004100, 0}}, // extract high half to F32 R4
+ {{0xff05723e, 0x00000000, 0x00004002, 0}}, // MERGE_C R5, R0, R2
+ {{0x02067230, 0x20000002, 0x00000800, 0}}, // add two low halves: (2,2)
+ {{0x02077231, 0x20000002, 0x00040802, 0}}, // fused 1*1+1: (2,2)
+ {{0x02087240, 0x30000002, 0x03800800, 0}}, // min of low/high: (1,1)
+ {{0x02097840, 0x3c003c00, 0x07800c00, 0}}, // max of high/one: (4,4)
+ {{0x000a7308, 0x00000008, 0x00001200, 0}}, // reciprocal of half one
+ {{0x020b7816, 0x00005410, 0x00000007, 0}}, // concatenate low halves
+ {{0xff0c7230, 0xa0000008, 0x00004100, 0}}, // extract negated low half
+ {{0x020d7230, 0x30000002, 0x00000f00, 0}}, // -abs(high) + high: (0,0)
+ {{0x020e7431, 0x3c003c00, 0x00400802, 0}}, // (1,1) * (1,-4) + (1,1)
+ VertexTransform.back(),
+ }};
+ auto code = input;
+ const u32 selector = GENERATE(0x5410U, 0x5432U, 0x7610U, 0x7632U);
+ code[10].words[1] = selector;
+ if (selector == 0x7610 || selector == 0x7632) code[10].words[2] = 2;
+ const auto lowered = Shader::Ampere::LowerToMaxwell(code);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const u64 word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::F2F_reg: visitor.F2F_reg(word); break;
+ case Opcode::IADD_reg: visitor.IADD_reg(word); break;
+ case Opcode::ISCADD_reg: visitor.ISCADD_reg(word); break;
+ case Opcode::HMUL2_reg: visitor.HMUL2_reg(word); break;
+ case Opcode::HADD2_reg: visitor.HADD2_reg(word); break;
+ case Opcode::HFMA2_reg: visitor.HFMA2_reg(word); break;
+ case Opcode::FMNMX_reg: visitor.FMNMX_reg(word); break;
+ case Opcode::MUFU: visitor.MUFU(word); break;
+ case Opcode::MOV32I: visitor.MOV32I(word); break;
+ case Opcode::SHR_imm: visitor.SHR_imm(word); break;
+ case Opcode::LOP32I: visitor.LOP32I(word); break;
+ case Opcode::MOV_reg: visitor.MOV_reg(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected canonical half instruction");
+ }
+ }
+ // Independent values with exact binary16 representations exercise packing,
+ // multiplication and extraction through the actual emitted IR data flow.
+ const std::array<std::pair<u16, float>, 5> halves{{{u16{0x4200}, 3.0f}, {u16{0}, 0.0f}, {u16{0x3c00}, 1.0f}, {u16{0x4000}, 2.0f}, {u16{0x4400}, 4.0f}}};
+ const auto from_half = [&](u16 bits) {
+ for (const auto& [encoded, value] : halves) if (encoded == (bits & 0x7fff)) return (bits & 0x8000) ? -value : value;
+ throw std::logic_error("unexpected half value");
+ };
+ const auto to_half = [&](float value) {
+ for (const auto& [encoded, reference] : halves) if (std::abs(value) == reference) return static_cast<u16>(encoded | (std::signbit(value) ? 0x8000 : 0));
+ throw std::logic_error("unexpected float value");
+ };
+ std::array<u32, 256> regs{};
+ regs[0] = std::bit_cast<u32>(1.0f);
+ regs[1] = std::bit_cast<u32>(2.0f);
+ std::unordered_map<const IR::Inst*, u64> values;
+ const auto get = [&](const IR::Value& value) -> u64 {
+ if (!value.IsImmediate()) return values.at(value.Inst());
+ if (value.Type() == IR::Type::U1) return value.U1();
+ if (value.Type() == IR::Type::F32) return std::bit_cast<u32>(value.F32());
+ if (value.Type() == IR::Type::U16 || value.Type() == IR::Type::F16) return value.U16();
+ return value.U32();
+ };
+ for (const auto& inst : block.Instructions()) {
+ const auto arg = [&](size_t i) { return get(inst.Arg(i)); };
+ switch (inst.GetOpcode()) {
+ case IR::Opcode::GetRegister: values[&inst] = regs[static_cast<u32>(inst.Arg(0).Reg())]; break;
+ case IR::Opcode::SetRegister: regs[static_cast<u32>(inst.Arg(0).Reg())] = static_cast<u32>(arg(1)); break;
+ case IR::Opcode::BitCastF32U32:
+ case IR::Opcode::BitCastU32F32: values[&inst] = arg(0); break;
+ case IR::Opcode::ConvertF16F32: values[&inst] = to_half(std::bit_cast<float>(static_cast<u32>(arg(0)))); break;
+ case IR::Opcode::ConvertF32F16: values[&inst] = std::bit_cast<u32>(from_half(static_cast<u16>(arg(0)))); break;
+ case IR::Opcode::CompositeConstructF16x2: values[&inst] = arg(0) | (arg(1) << 32); break;
+ case IR::Opcode::CompositeExtractF16x2: values[&inst] = (arg(0) >> (arg(1) * 32)) & 0xffff; break;
+ case IR::Opcode::PackFloat2x16: values[&inst] = (arg(0) & 0xffff) | ((arg(0) >> 32) << 16); break;
+ case IR::Opcode::UnpackFloat2x16: values[&inst] = (arg(0) & 0xffff) | ((arg(0) >> 16) << 32); break;
+ case IR::Opcode::FPMul16: values[&inst] = to_half(from_half(static_cast<u16>(arg(0))) * from_half(static_cast<u16>(arg(1)))); break;
+ case IR::Opcode::FPAdd16: values[&inst] = to_half(from_half(static_cast<u16>(arg(0))) + from_half(static_cast<u16>(arg(1)))); break;
+ case IR::Opcode::FPFma16: values[&inst] = to_half(std::fma(from_half(static_cast<u16>(arg(0))), from_half(static_cast<u16>(arg(1))), from_half(static_cast<u16>(arg(2))))); break;
+ case IR::Opcode::FPNeg16: values[&inst] = arg(0) ^ 0x8000; break;
+ case IR::Opcode::FPAbs16: values[&inst] = arg(0) & 0x7fff; break;
+ case IR::Opcode::FPAdd32: values[&inst] = std::bit_cast<u32>(std::bit_cast<float>(static_cast<u32>(arg(0))) + std::bit_cast<float>(static_cast<u32>(arg(1)))); break;
+ case IR::Opcode::FPMin32: values[&inst] = std::bit_cast<u32>(std::fmin(std::bit_cast<float>(static_cast<u32>(arg(0))), std::bit_cast<float>(static_cast<u32>(arg(1))))); break;
+ case IR::Opcode::FPMax32: values[&inst] = std::bit_cast<u32>(std::fmax(std::bit_cast<float>(static_cast<u32>(arg(0))), std::bit_cast<float>(static_cast<u32>(arg(1))))); break;
+ case IR::Opcode::FPRecip32: values[&inst] = std::bit_cast<u32>(1.0f / std::bit_cast<float>(static_cast<u32>(arg(0)))); break;
+ case IR::Opcode::SelectF32: values[&inst] = arg(0) ? arg(1) : arg(2); break;
+ case IR::Opcode::BitwiseAnd32: values[&inst] = arg(0) & arg(1); break;
+ case IR::Opcode::BitwiseXor32: values[&inst] = arg(0) ^ arg(1); break;
+
+ case IR::Opcode::ShiftRightLogical32: values[&inst] = static_cast<u32>(arg(0)) >> arg(1); break;
+ case IR::Opcode::SLessThan: values[&inst] = static_cast<s32>(arg(0)) < static_cast<s32>(arg(1)); break;
+ case IR::Opcode::ULessThan: values[&inst] = static_cast<u32>(arg(0)) < static_cast<u32>(arg(1)); break;
+ case IR::Opcode::SelectU32: values[&inst] = arg(0) ? arg(1) : arg(2); break;
+ case IR::Opcode::IAdd32: values[&inst] = static_cast<u32>(arg(0) + arg(1)); break;
+ case IR::Opcode::ShiftLeftLogical32: values[&inst] = static_cast<u32>(arg(0) << arg(1)); break;
+ default: FAIL("Unexpected IR operation in half arithmetic");
+ }
+ }
+ REQUIRE(regs[2] == 0x44003c00);
+ REQUIRE(regs[5] == 0x3c003c00);
+ REQUIRE(regs[6] == 0x40004000U);
+ REQUIRE(regs[7] == 0x40004000U);
+ REQUIRE(regs[8] == 0x3c003c00U);
+ REQUIRE(regs[9] == 0x44004400U);
+ REQUIRE(regs[10] == 0x3c00U);
+ REQUIRE(regs[11] == (selector == 0x7632 ? 0x44004400U : selector == 0x7610 ? 0x44003c00U : selector == 0x5432 ? 0x40004400U : 0x40003c00U));
+ REQUIRE(std::bit_cast<float>(regs[12]) == -1.0f);
+ REQUIRE(regs[13] == 0U);
+ REQUIRE(regs[14] == 0xc2004000U);
+
+ REQUIRE(std::bit_cast<float>(regs[3]) == 1.0f);
+ REQUIRE(std::bit_cast<float>(regs[4]) == 4.0f);
+}
+
+TEST_CASE("Live shader overrides require the exact original instructions", "[native-shader]") {
+ const auto program = Shader::Ampere::LowerToMaxwell(VertexTransform);
+ REQUIRE(program);
+ const auto bytes = Shader::Ampere::EncodeProgramOverride(VertexTransform, program);
+ REQUIRE_FALSE(bytes.empty());
+ REQUIRE(Shader::Ampere::ProgramOverrideSourceSize(bytes) == VertexTransform.size() * 16);
+ REQUIRE(Shader::Ampere::ProgramOverrideSourceSize(std::span{bytes}.first(12)) == 0);
+ auto invalid_source = bytes;
+ invalid_source[8] ^= 1;
+ REQUIRE(Shader::Ampere::ProgramOverrideSourceSize(invalid_source) == 0);
+ const auto decoded = Shader::Ampere::DecodeProgramOverride(bytes, VertexTransform);
+ REQUIRE(decoded);
+ REQUIRE(decoded.instructions == program.instructions);
+ REQUIRE(decoded.source_instruction_offsets == program.source_instruction_offsets);
+ auto invalid_map = bytes;
+ const u32 invalid_target = static_cast<u32>(program.instructions.size());
+ std::memcpy(invalid_map.data() + invalid_map.size() - program.source_instruction_offsets.size() * 4,
+ &invalid_target, sizeof(invalid_target));
+ const auto invalid_mapping = Shader::Ampere::DecodeProgramOverride(invalid_map, VertexTransform);
+ REQUIRE_FALSE(invalid_mapping);
+ REQUIRE(invalid_mapping.instructions.empty());
+
+ REQUIRE(decoded.source_size == program.source_size);
+ REQUIRE(decoded.required_texture_buffer == program.required_texture_buffer);
+ auto different = VertexTransform;
+ different[0].words[1] ^= 4;
+ REQUIRE(Shader::Ampere::NativeProgramKey(different) != Shader::Ampere::NativeProgramKey(VertexTransform));
+ const auto mismatch = Shader::Ampere::DecodeProgramOverride(bytes, different);
+ REQUIRE_FALSE(mismatch);
+ REQUIRE(mismatch.instructions.empty());
+ REQUIRE_FALSE(Shader::Ampere::DecodeProgramOverride(std::span{bytes}.first(12), VertexTransform));
+ REQUIRE_FALSE(Shader::Ampere::DecodeProgramOverride(std::span{bytes}.first(bytes.size() - 1), VertexTransform));
+ auto wrong_version = bytes;
+ wrong_version[4] ^= 1;
+ REQUIRE_FALSE(Shader::Ampere::DecodeProgramOverride(wrong_version, VertexTransform));
+ auto wrong_binding = bytes;
+ const u32 bank = 18;
+ std::memcpy(wrong_binding.data() + 16, &bank, sizeof(bank));
+ REQUIRE_FALSE(Shader::Ampere::DecodeProgramOverride(wrong_binding, VertexTransform));
+}
+
+TEST_CASE("Native integer shader forms preserve signed arithmetic and immediate values", "[native-shader]") {
+ // Synthetic vertex program: output (id + arithmetic_shift(id, 1), 4.0).
+ const std::array<Native, 7> input{{
+ {{0xff007321, 0x0002fcff, 0x00000000, 0x000e2200}},
+ {{0xff017819, 0x00000001, 0x00011400, 0x000fe400}},
+ {{0x00007824, 0x00000001, 0x078e0201, 0x000fe200}},
+ {{0x00007245, 0x00000000, 0x00201400, 0x000fc600}},
+ {{0xff017424, 0x40800000, 0x078e00ff, 0x000fe200}},
+ {{0xff007322, 0x00007000, 0x000004ff, 0x000fe200}},
+ VertexTransform.back(),
+ }};
+ auto program = input;
+ const u32 mode = GENERATE(0U, 1U, 2U, 3U, 4U, 5U, 6U, 7U, 8U);
+ if (mode == 7) program[2] = Native{{0x00007217, 1, 0x03800200, 0}};
+ if (mode == 8) program[2] = Native{{0x00007217, 1, 0x07800200, 0}};
+ if (mode == 5) program[2] = Native{{0x00007210, 0x00000001, 0x07ffe1ff, 0}};
+ if (mode == 6) program[2] = Native{{0x00007817, 0x0000003f, 0x07800200, 0}};
+ if (mode == 4) program[2] = Native{{0x00007817, 0x0000003f, 0x03800200, 0}};
+ if (mode == 1) program[2] = Native{{0x00007210, 0x00000001, 0x07ffe0ff, 0x000fe200}};
+ if (mode == 3) program[2] = Native{{0x00007210, 0x80000001, 0x07ffe0ff, 0x000fe200}};
+ if (mode == 2) program[1] = Native{{0x00017819, 0x00000001, 0x000006ff, 0x000fe200}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(program);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const u64 word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::ALD: visitor.ALD(word); break;
+ case Opcode::AST: visitor.AST(word); break;
+ case Opcode::MOV32I: visitor.MOV32I(word); break;
+ case Opcode::SHR_imm: visitor.SHR_imm(word); break;
+ case Opcode::SHL_imm: visitor.SHL_imm(word); break;
+ case Opcode::IMNMX_reg: visitor.IMNMX_reg(word); break;
+ case Opcode::IMNMX_imm: visitor.IMNMX_imm(word); break;
+ case Opcode::IADD_reg: visitor.IADD_reg(word); break;
+ case Opcode::I2F_reg: visitor.I2F_reg(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected lowered integer shader instruction");
+ }
+ }
+ for (const s32 id : {0, 1, 2, -1, -3, 12345}) {
+ REQUIRE(EvaluateVertex(block, {}, {}, id) ==
+ std::array<float, 2>{static_cast<float>(mode == 2 ? id * 3 : mode == 3 ? id - (id >> 1) : mode == 4 ? std::min<s32>(id, 63) : mode == 5 ? -id + (id >> 1) : mode == 6 ? std::max<s32>(id, 63) : mode == 7 ? std::min<s32>(id, id >> 1) : mode == 8 ? std::max<s32>(id, id >> 1) : id + (id >> 1)), 4.0f});
+ }
+}
+
+TEST_CASE("Native single-channel texel fetch writes only its selected result", "[native-shader]") {
+ std::array<Native, 2> input{{
+ {{0x04017b66, 0x30800805, 0x049e01ff, 0x001fc200}},
+ VertexTransform.back(),
+ }};
+ const u32 component = GENERATE(0U, 1U, 2U, 3U);
+ input[0].words[2] = (input[0].words[2] & ~0xf00U) | (1U << (8 + component));
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ REQUIRE(lowered.required_texture_buffer == 2);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.TLDS(lowered.instructions.front());
+ std::vector<u32> writes;
+ u32 fetches = 0;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::SetRegister)
+ writes.push_back(static_cast<u32>(inst.Arg(0).Reg()));
+ if (inst.GetOpcode() == IR::Opcode::BoundImageFetch) {
+ ++fetches;
+ REQUIRE(inst.Arg(0).U32() == 0x20);
+ const auto* coords = inst.Arg(1).Inst();
+ REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructU32x2);
+ REQUIRE(coords->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{4});
+ REQUIRE(coords->Arg(1).Inst()->Arg(0).Reg() == IR::Reg{5});
+ }
+ if (inst.GetOpcode() == IR::Opcode::CompositeExtractF32x4)
+ REQUIRE(inst.Arg(1).U32() == component);
+ }
+ REQUIRE(fetches == 1);
+ REQUIRE(writes == std::vector<u32>{1});
+}
+
+TEST_CASE("Native IMAD move loads the actual constant-buffer value", "[native-shader]") {
+ const std::array<Native, 3> input{{
+ {{0xff007624, 0x00c00100, 0x078e00ff, 0x000fe200}},
+ {{0xff007322, 0x00007000, 0x000000ff, 0x000fe200}},
+ VertexTransform.back(),
+ }};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ REQUIRE(Shader::Maxwell::Decode(lowered.instructions.front()) == Shader::Maxwell::Opcode::MOV_cbuf);
+ visitor.MOV_cbuf(lowered.instructions[0]);
+ visitor.AST(lowered.instructions[1]);
+ REQUIRE(EvaluateVertex(block, {}, {13, -2.5f, 99, 7}) == std::array<float, 2>{-2.5f, 0.0f});
+ auto unsupported = input;
+ unsupported[0].words[1] |= 1U << 30;
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(unsupported));
+}
+
+TEST_CASE("Depth-stencil descriptors ignore the unused blue and alpha component types", "[native-gpu]") {
+ using Tegra::Texture::ComponentType;
+ using Tegra::Texture::TextureFormat;
+ using VideoCore::Surface::PixelFormat;
+ for (const auto unused : {ComponentType::UNORM, ComponentType::UINT}) {
+ REQUIRE(VideoCommon::PixelFormatFromTextureInfo(TextureFormat::S8Z24,
+ ComponentType::UNORM, ComponentType::UINT, unused, unused, false) ==
+ PixelFormat::D24_UNORM_S8_UINT);
+ }
+}
+
+TEST_CASE("Native floating min/max clamps actual shader input", "[native-shader]") {
+ const std::array<Native, 5> input{{
+ {{0xff007321, 0x000080ff, 0x00000000, 0x000e2200}},
+ {{0x00007a09, 0x00c00000, 0x07810000, 0x000fe400}},
+ {{0x00007a09, 0x00c00100, 0x03810000, 0x000fe400}},
+ {{0xff007322, 0x00007000, 0x000000ff, 0x000fe200}},
+ VertexTransform.back(),
+ }};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const auto word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::ALD: visitor.ALD(word); break;
+ case Opcode::AST: visitor.AST(word); break;
+ case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break;
+ case Opcode::FMNMX_reg: visitor.FMNMX_reg(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected native min/max lowering");
+ }
+ }
+ for (const float value : {-20.0f, -1.0f, 0.25f, 2.0f, 100.0f})
+ REQUIRE(EvaluateVertex(block, {value, 0}, {-1, 2, 0, 0}) ==
+ std::array<float, 2>{std::clamp(value, -1.0f, 2.0f), 0});
+}
+
+TEST_CASE("Native boolean float comparison preserves register moves and absolute inputs", "[native-shader]") {
+ const std::array<Native, 5> input{{
+ {{0xff007321, 0x000080ff, 0x00000000, 0}},
+ {{0xff017224, 0x000000ff, 0x078e0000, 0}},
+ {{0x01007a0a, 0x00c00000, 0x03811200, 0}},
+ {{0xff007322, 0x00007000, 0x000000ff, 0}},
+ VertexTransform.back(),
+ }};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const auto word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::ALD: visitor.ALD(word); break;
+ case Opcode::AST: visitor.AST(word); break;
+ case Opcode::MOV_reg: visitor.MOV_reg(word); break;
+ case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break;
+ case Opcode::LOP32I: visitor.LOP32I(word); break;
+ case Opcode::FSET_reg: visitor.FSET_reg(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected native boolean float comparison lowering");
+ }
+ }
+ for (const float value : {-3.0f, -1.0f, 0.0f, 2.0f, 4.0f})
+ REQUIRE(EvaluateVertex(block, {value, 0}, {2, 0, 0, 0}) ==
+ std::array<float, 2>{std::abs(value) < 2.0f ? 1.0f : 0.0f, 0});
+}
+
+TEST_CASE("Native unsigned conversion saturates, clears NaNs, and extracts the low byte", "[native-shader]") {
+ std::array<Native, 5> input{{
+ {{0xff007321, 0x000080ff, 0x00000000, 0}},
+ {{0x00017305, 0x00000000, 0x0021f000, 0}},
+ {{0x00007306, 0x00000001, 0x00001000, 0}},
+ {{0xff007322, 0x00007000, 0x000000ff, 0}},
+ VertexTransform.back(),
+ }};
+ const bool constant_source = GENERATE(false, true);
+ if (constant_source) input[1] = Native{{0x00017b05, 0x00c00000, 0x0021f000, 0}};
+ const u32 conversion_mode = GENERATE(0U, 1U, 2U);
+ const bool short_round = conversion_mode == 1;
+ if (conversion_mode == 2) input[1].words[2] = 0x00207000;
+ if (short_round) input[1].words[2] = 0x00212800;
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const auto word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::ALD: visitor.ALD(word); break;
+ case Opcode::AST: visitor.AST(word); break;
+ case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break;
+ case Opcode::MOV32I: visitor.MOV32I(word); break;
+ case Opcode::FSET_reg: visitor.FSET_reg(word); break;
+ case Opcode::FMNMX_reg: visitor.FMNMX_reg(word); break;
+ case Opcode::LOP_reg: visitor.LOP_reg(word); break;
+ case Opcode::F2I_reg: visitor.F2I_reg(word); break;
+ case Opcode::I2F_reg: visitor.I2F_reg(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected unsigned conversion lowering");
+ }
+ }
+ const std::array<std::pair<float, float>, 9> cases{{
+ {std::numeric_limits<float>::quiet_NaN(), 0.0f},
+ {-std::numeric_limits<float>::infinity(), 0.0f}, {-3.0f, 0.0f}, {1.9f, 1.0f},
+ {300.75f, 44.0f}, {65535.0f, 255.0f}, {4294967040.0f, 0.0f},
+ {4294967296.0f, 255.0f}, {std::numeric_limits<float>::infinity(), 255.0f},
+ }};
+ for (const auto& [value, expected] : cases) {
+ const float rounded = std::isnan(value) ? 0.0f : static_cast<float>(static_cast<u32>(
+ std::clamp(std::nearbyint(static_cast<double>(value)), 0.0, 65535.0)) & 255);
+ REQUIRE(EvaluateVertex(block, {value, 0}, {value, 0, 0, 0}) == std::array<float, 2>{short_round ? rounded : expected, 0});
+ }
+}
+
+TEST_CASE("Native depth-comparison sample preserves XY coordinates and reference depth", "[native-shader]") {
+ const std::array<Native, 2> input{{
+ {{0x04017b60, 0x30800806, 0x041e41ff, 0}},
+ VertexTransform.back(),
+ }};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.TEXS(lowered.instructions.front());
+ u32 samples = 0;
+ std::vector<u32> writes;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::SetRegister)
+ writes.push_back(static_cast<u32>(inst.Arg(0).Reg()));
+ if (inst.GetOpcode() == IR::Opcode::BoundImageSampleDrefImplicitLod) {
+ ++samples;
+ REQUIRE(inst.Arg(0).U32() == 0x20);
+ const auto* coords = inst.Arg(1).Inst();
+ REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructF32x2);
+ for (u32 i = 0; i < 2; ++i)
+ REQUIRE(coords->Arg(i).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{4 + i});
+ REQUIRE(inst.Arg(2).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{6});
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().is_depth == 1);
+ }
+ }
+ REQUIRE(samples == 1);
+ REQUIRE(writes == std::vector<u32>{1});
+}
+
+TEST_CASE("Native array texel fetch preserves layer and separates XY registers", "[native-shader]") {
+ std::array<Native, 2> input{{
+ {{0x06007b66, 0xb0800809, 0x009e0f02, 0}},
+ VertexTransform.back(),
+ }};
+ const bool sparse = GENERATE(false, true);
+ if (sparse) input[0].words[2] = 0x009e0b23;
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.MOV_reg(lowered.instructions[0]);
+ visitor.MOV_reg(lowered.instructions[1]);
+ visitor.TLDS(lowered.instructions[2]);
+ u32 fetches = 0;
+ std::vector<u32> writes, components;
+ std::vector<std::pair<u32, u32>> copies;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::SetRegister && static_cast<u32>(inst.Arg(0).Reg()) < 250)
+ writes.push_back(static_cast<u32>(inst.Arg(0).Reg()));
+ if (inst.GetOpcode() == IR::Opcode::CompositeExtractF32x4)
+ components.push_back(inst.Arg(1).U32());
+ if (inst.GetOpcode() == IR::Opcode::SetRegister &&
+ static_cast<u32>(inst.Arg(0).Reg()) >= 250) {
+ REQUIRE(inst.Arg(1).Inst()->GetOpcode() == IR::Opcode::GetRegister);
+ copies.emplace_back(static_cast<u32>(inst.Arg(0).Reg()),
+ static_cast<u32>(inst.Arg(1).Inst()->Arg(0).Reg()));
+ }
+ if (inst.GetOpcode() == IR::Opcode::BoundImageFetch) {
+ ++fetches;
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type == Shader::TextureType::ColorArray2D);
+ const auto* coords = inst.Arg(1).Inst();
+ REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructU32x3);
+ REQUIRE(coords->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{250});
+ REQUIRE(coords->Arg(1).Inst()->Arg(0).Reg() == IR::Reg{251});
+ const auto* layer = coords->Arg(2).Inst();
+ REQUIRE(layer->GetOpcode() == IR::Opcode::BitFieldUExtract);
+ REQUIRE(layer->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{6});
+ REQUIRE(layer->Arg(1).U32() == 0);
+ REQUIRE(layer->Arg(2).U32() == 16);
+ }
+ }
+ REQUIRE(fetches == 1);
+ REQUIRE(writes == (sparse ? std::vector<u32>{0, 1, 35} : std::vector<u32>{0, 1, 2, 3}));
+ REQUIRE(components == (sparse ? std::vector<u32>{0, 1, 3} : std::vector<u32>{0, 1, 2, 3}));
+ REQUIRE(copies == std::vector<std::pair<u32, u32>>{{250, 7}, {251, 9}});
+}
+
+TEST_CASE("Native bitwise LUT forms preserve source bits without changing predicates", "[native-shader]") {
+ std::array<Native, 5> input{{
+ {{0x00007a02, 0x00c00000, 0x00000f00, 0}},
+ {{0x00027a02, 0x00c00200, 0x00000f00, 0}},
+ {{0x00007812, 0x80000000, 0x078ef802, 0}},
+ {{0xff007322, 0x00007000, 0x000000ff, 0}},
+ VertexTransform.back(),
+ }};
+ const u32 mode = GENERATE(0U, 1U, 2U);
+ if (mode == 2) input[2] = Native{{0x00007812, 0x80000000, 0x078ee202, 0}};
+ if (mode == 1) input[2] = Native{{0x00007212, 0x00000002, 0x078efcff, 0}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const auto word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::AST: visitor.AST(word); break;
+ case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break;
+ case Opcode::MOV32I: visitor.MOV32I(word); break;
+ case Opcode::LOP_reg: visitor.LOP_reg(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected native bitwise LUT lowering");
+ }
+ }
+ for (const auto& inst : block.Instructions()) REQUIRE(inst.GetOpcode() != IR::Opcode::SetPred);
+ for (const float a : {-1.0f, 1.0f}) for (const float c : {-2.0f, 2.0f}) {
+ const u32 c_bits = std::bit_cast<u32>(c);
+ const u32 a_bits = std::bit_cast<u32>(a);
+ const u32 expected = mode == 2 ? (a_bits & 0x80000000U) | (c_bits & 0x7fffffffU) :
+ a_bits | (mode == 1 ? c_bits : c_bits & 0x80000000U);
+ REQUIRE(EvaluateVertex(block, {}, {a, 0, c, 0}) ==
+ std::array<float, 2>{std::bit_cast<float>(expected), 0});
+ }
+}
+
+TEST_CASE("Native one-dimensional texel fetch preserves its integer coordinate", "[native-shader]") {
+ const std::array<Native, 2> input{{
+ {{0x04017b66, 0x108008ff, 0x009e01ff, 0}},
+ VertexTransform.back(),
+ }};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.TLDS(lowered.instructions.front());
+ u32 fetches = 0;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::BoundImageFetch) {
+ ++fetches;
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type == Shader::TextureType::Color1D);
+ REQUIRE(inst.Arg(1).Inst()->GetOpcode() == IR::Opcode::GetRegister);
+ REQUIRE(inst.Arg(1).Inst()->Arg(0).Reg() == IR::Reg{4});
+ }
+ }
+ REQUIRE(fetches == 1);
+}
+
+TEST_CASE("Native call and return preserve values through the actual shader frontend", "[native-shader]") {
+ std::array<Native, 9> input{{
+ {{0x00007802, 0x40000000, 0x00000f00, 0}}, // argument 2.0
+ {{0x00377802, 0x00000030, 0x00000f00, 0}}, // native return PC
+ {{0x00007944, 0x00000020, 0x03c00000, 0}}, // call offset0x50
+ {{0xff007322, 0x00007003, 0x000000ff, 0}},
+ VertexTransform.back(),
+ {{0x00037421, 0x40400000, 0x00010000, 0}}, // callee: add 3.0
+ {{0x00087202, 0x00000037, 0x00000f00, 0}},
+ {{0x00097802, 0x00000000, 0x00000f00, 0}},
+ {{0x08007950, 0xffffff70, 0x03c3ffff, 0}},
+ }};
+ const bool alternate_link = GENERATE(false, true);
+ if (alternate_link) {
+ input[1].words[0] = 0x00397802;
+ input[6] = Native{{0x00007202, 0x00000039, 0x00000f00, 0}};
+ input[7].words[0] = 0x00017802;
+ input[8].words[0] = 0x00007950;
+ }
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ REQUIRE(lowered.source_size == input.size() * 16);
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(std::span{input}.first(5)));
+ auto broken = input;
+ broken[1].words[1] += 16;
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(broken));
+ broken = input;
+ broken.back().words[1] += 16;
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(broken));
+ std::vector<u64> code(0x50 / 8);
+ for (const auto word : lowered.instructions) {
+ if ((code.size() * 8) % 32 == 0) code.push_back(0);
+ code.push_back(word);
+ }
+ class CallEnvironment final : public EmptyEnvironment {
+ public:
+ explicit CallEnvironment(const std::vector<u64>& code_) : code{code_} { stage = Shader::Stage::VertexB; }
+ u64 ReadInstruction(u32 address) override { return code.at(address / 8); }
+ const std::vector<u64>& code;
+ } env{code};
+ Shader::ObjectPool<Shader::Maxwell::Flow::Block> flow_pool;
+ Shader::ObjectPool<IR::Inst> inst_pool;
+ Shader::ObjectPool<IR::Block> block_pool;
+ Shader::Maxwell::Flow::CFG cfg{env, flow_pool, 0x50};
+ REQUIRE(cfg.Functions().size() == 2);
+ Shader::HostTranslateInfo host;
+ const auto program = Shader::Maxwell::TranslateProgram(inst_pool, block_pool, env, cfg, host);
+ u32 stores = 0;
+ for (const auto* block : program.blocks) {
+ for (const auto& inst : block->Instructions()) {
+ if (inst.GetOpcode() != IR::Opcode::SetAttribute) continue;
+ ++stores;
+ REQUIRE(inst.Arg(1).IsImmediate());
+ REQUIRE(inst.Arg(1).F32() == 5.0f);
+ }
+ }
+ REQUIRE(stores == 1);
+}
+
+TEST_CASE("Native scaled multiplication preserves captured power-of-two scales", "[native-shader]") {
+ std::array<Native, 4> input{{
+ VertexTransform.front(),
+ {{0x00007220, 0x00000001, 0x00310000, 0}},
+ {{0xff007322, 0x00007000, 0x000000ff, 0}},
+ VertexTransform.back(),
+ }};
+ const u32 scale = GENERATE(3U, 4U, 5U, 6U);
+ input[1].words[2] = (scale << 20) | 0x10000;
+ const float factor = scale == 3 ? 0.5f : scale == 5 ? 2.0f : scale == 6 ? 4.0f : 1.0f;
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.ALD(lowered.instructions[0]);
+ visitor.FMUL_reg(lowered.instructions[1]);
+ visitor.AST(lowered.instructions[2]);
+ REQUIRE(EvaluateVertex(block, {8, 4}, {}) == std::array<float, 2>{32 * factor, 0});
+ REQUIRE(EvaluateVertex(block, {-2, 3}, {}) == std::array<float, 2>{-6 * factor, 0});
+}
+
+TEST_CASE("Native integer multiply preserves high words and modular multiply-add", "[native-shader]") {
+ const u32 mode = GENERATE(0U, 1U, 2U, 3U, 4U);
+ Native instruction{{0x00037827, 0xcccccccd, 0x078e00ff, 0}};
+ if (mode == 1) instruction = Native{{0x00037824, 0xfffffff6, 0x078e0202, 0}};
+ if (mode == 2) instruction = Native{{0x00037a24, 0x00c00000, 0x078e0202, 0}};
+ if (mode == 4) instruction = Native{{0x00037811, 0x00000070, 0x078e20ff, 0}};
+ if (mode == 3) instruction = Native{{0x00037224, 0x00000001, 0x078e0202, 0}};
+ const std::array<Native, 2> input{{instruction, VertexTransform.back()}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const auto word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::XMAD_imm: visitor.XMAD_imm(word); break;
+ case Opcode::XMAD_reg: visitor.XMAD_reg(word); break;
+ case Opcode::MOV32I: visitor.MOV32I(word); break;
+ case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break;
+ case Opcode::ISCADD_reg: visitor.ISCADD_reg(word); break;
+ case Opcode::SHR_imm: visitor.SHR_imm(word); break;
+ case Opcode::IADD_reg: visitor.IADD_reg(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected canonical integer multiplication instruction");
+ }
+ }
+ for (u32 a : {0U, 1U, 9U, 10U, 65535U, 65536U, 0x80000000U, 0xfffffffeU, 0xffffffffU, 0x12345678U}) {
+ std::array<u32, 256> regs{};
+ regs[0] = a;
+ regs[1] = 0x76543210;
+ regs[2] = 0xfedcba98;
+ std::unordered_map<const IR::Inst*, u32> values;
+ const auto get = [&](const IR::Value& value) -> u32 {
+ if (!value.IsImmediate()) return values.at(value.Inst());
+ return value.Type() == IR::Type::U1 ? static_cast<u32>(value.U1()) : value.U32();
+ };
+ for (const auto& inst : block.Instructions()) {
+ const auto arg = [&](size_t i) { return get(inst.Arg(i)); };
+ switch (inst.GetOpcode()) {
+ case IR::Opcode::GetRegister: values[&inst] = regs[static_cast<u32>(inst.Arg(0).Reg())]; break;
+ case IR::Opcode::SetRegister: regs[static_cast<u32>(inst.Arg(0).Reg())] = arg(1); break;
+ case IR::Opcode::GetCbufU32:
+ REQUIRE(arg(0) == 3);
+ REQUIRE(arg(1) == 0);
+ values[&inst] = 0x76543210;
+ break;
+ case IR::Opcode::BitFieldUExtract: values[&inst] = (arg(0) >> arg(1)) & ((1U << arg(2)) - 1); break;
+ case IR::Opcode::IMul32: values[&inst] = arg(0) * arg(1); break;
+ case IR::Opcode::IAdd32: values[&inst] = arg(0) + arg(1); break;
+ case IR::Opcode::ShiftLeftLogical32: values[&inst] = arg(0) << arg(1); break;
+ case IR::Opcode::ShiftRightLogical32: values[&inst] = arg(0) >> arg(1); break;
+ case IR::Opcode::ULessThan: values[&inst] = arg(0) < arg(1); break;
+ case IR::Opcode::SLessThan: values[&inst] = static_cast<s32>(arg(0)) < static_cast<s32>(arg(1)); break;
+ case IR::Opcode::SelectU32: values[&inst] = arg(0) ? arg(1) : arg(2); break;
+ default: FAIL("Unexpected multiplication IR operation " << IR::NameOf(inst.GetOpcode()));
+ }
+ }
+ const u32 b = mode == 0 ? 0xcccccccd : mode == 1 ? 0xfffffff6 : 0x76543210;
+ const u64 product = u64{a} * b;
+ const u32 expected = mode == 4 ? (a << 4) + 0x70U : mode == 0 ? static_cast<u32>(product >> 32)
+ : static_cast<u32>(product + 0xfedcba98U);
+ REQUIRE(regs[3] == expected);
+ }
+}
+
+TEST_CASE("Native uniform constants keep their indexed load and arithmetic values", "[native-shader]") {
+ const std::array<Native, 7> input{{
+ {{0xff007321, 0x000080ff, 0x00000000, 0}},
+ {{0x00047ab9, 0x00c00000, 0x00000800, 0}},
+ {{0x04047899, 0x00000002, 0x0800063f, 0}},
+ {{0x04057abb, 0x00c00000, 0x08000800, 0}},
+ {{0x00007c20, 0x00000005, 0x08410000, 0}},
+ {{0x00007e21, 0x80000005, 0x08010000, 0}},
+ VertexTransform.back(),
+ }};
+ auto code = input;
+ // Store the computed scalar instead of ending immediately.
+ std::vector<Native> program(code.begin(), code.end() - 1);
+ program.push_back(Native{{0xff007322, 0x00007000, 0x000000ff, 0}});
+ program.push_back(VertexTransform.back());
+ const auto lowered = Shader::Ampere::LowerToMaxwell(program);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const auto word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::ALD: visitor.ALD(word); break;
+ case Opcode::AST: visitor.AST(word); break;
+ case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break;
+ case Opcode::MOV_reg: visitor.MOV_reg(word); break;
+ case Opcode::LOP32I: visitor.LOP32I(word); break;
+ case Opcode::SHL_imm: visitor.SHL_imm(word); break;
+ case Opcode::LDC: visitor.LDC(word); break;
+ case Opcode::FMUL_reg: visitor.FMUL_reg(word); break;
+ case Opcode::FADD_reg: visitor.FADD_reg(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected uniform instruction lowering");
+ }
+ }
+ const std::array<float, 4> constants{std::bit_cast<float>(1U), 3.0f, 7.0f, 9.0f};
+ REQUIRE(EvaluateVertex(block, {2, 0}, constants) == std::array<float, 2>{3, 0});
+ REQUIRE(EvaluateVertex(block, {4, 0}, constants) == std::array<float, 2>{9, 0});
+ auto conflict = program;
+ conflict[0].words[0] = 0xfff87321; // Native R248 conflicts with the reserved uniform bank.
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(conflict));
+}
+
+TEST_CASE("Native predicate-output LUTs preserve boolean bit tests", "[native-shader]") {
+ const std::array<Native, 3> input{{
+ {{0x00ff7812, 0x00000001, 0x0780c0ff, 0}},
+ {{0x01ff7812, 0x00000001, 0x07824802, 0}},
+ VertexTransform.back(),
+ }};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const auto word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::MOV32I: visitor.MOV32I(word); break;
+ case Opcode::LOP_reg: visitor.LOP_reg(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected predicate LUT lowering");
+ }
+ }
+ for (u32 a = 0; a < 4; ++a) for (u32 b = 0; b < 4; ++b) for (u32 c = 0; c < 4; ++c) {
+ std::array<u32, 256> regs{};
+ regs[0] = a; regs[1] = b; regs[2] = c;
+ std::array<bool, 7> predicates{false, false, true, false, true, false, true};
+ std::unordered_map<const IR::Inst*, u32> values;
+ const auto get = [&](const IR::Value& value) -> u32 {
+ if (!value.IsImmediate()) return values.at(value.Inst());
+ return value.Type() == IR::Type::U1 ? static_cast<u32>(value.U1()) : value.U32();
+ };
+ for (const auto& inst : block.Instructions()) {
+ const auto arg = [&](size_t i) { return get(inst.Arg(i)); };
+ switch (inst.GetOpcode()) {
+ case IR::Opcode::GetRegister: values[&inst] = regs[static_cast<u32>(inst.Arg(0).Reg())]; break;
+ case IR::Opcode::SetRegister: regs[static_cast<u32>(inst.Arg(0).Reg())] = arg(1); break;
+ case IR::Opcode::BitwiseAnd32: values[&inst] = arg(0) & arg(1); break;
+ case IR::Opcode::BitwiseOr32: values[&inst] = arg(0) | arg(1); break;
+ case IR::Opcode::BitwiseXor32: values[&inst] = arg(0) ^ arg(1); break;
+ case IR::Opcode::INotEqual: values[&inst] = arg(0) != arg(1); break;
+ case IR::Opcode::SetPred: predicates[static_cast<u32>(inst.Arg(0).Pred())] = arg(1) != 0; break;
+ default: FAIL("Unexpected predicate LUT IR " << IR::NameOf(inst.GetOpcode()));
+ }
+ }
+ REQUIRE(predicates[0] == ((a & 1) != 0));
+ REQUIRE(predicates[1] == (((b ^ c) & 1) != 0));
+ REQUIRE(std::array<bool, 5>{predicates[2], predicates[3], predicates[4], predicates[5], predicates[6]} ==
+ std::array<bool, 5>{true, false, true, false, true});
+ }
+}
+
+TEST_CASE("Native cube sampling preserves XYZ, explicit LOD, and split RGB destinations", "[native-shader]") {
+ std::array<Native, 2> input{{
+ {{0x04007b60, 0x70800808, 0x059e0703, 0}},
+ VertexTransform.back(),
+ }};
+ const bool volume = GENERATE(false, true);
+ if (volume) {
+ input[0].words[1] = (input[0].words[1] & ~(7U << 29)) | (2U << 29);
+ input[0].words[2] = (input[0].words[2] & ~(7U << 23)) | (1U << 23);
+ }
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const auto word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::MOV_reg: visitor.MOV_reg(word); break;
+ case Opcode::TEX: visitor.TEX(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected cube sampling lowering");
+ }
+ }
+ u32 samples = 0;
+ std::vector<std::pair<u32, u32>> copies;
+ std::vector<u32> writes;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::SetRegister) {
+ const u32 target = static_cast<u32>(inst.Arg(0).Reg());
+ if (target < 248) writes.push_back(target);
+ if (inst.Arg(1).Inst()->GetOpcode() == IR::Opcode::GetRegister)
+ copies.emplace_back(target, static_cast<u32>(inst.Arg(1).Inst()->Arg(0).Reg()));
+ }
+ if (inst.GetOpcode() == IR::Opcode::BoundImageSampleExplicitLod) {
+ ++samples;
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type == (volume ? Shader::TextureType::Color3D : Shader::TextureType::ColorCube));
+ const auto* coords = inst.Arg(1).Inst();
+ REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructF32x3);
+ for (u32 i = 0; i < 3; ++i)
+ REQUIRE(coords->Arg(i).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{250 + i});
+ if (volume) REQUIRE(inst.Arg(2).F32() == 0.0f);
+ else REQUIRE(inst.Arg(2).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{9});
+ }
+ }
+ REQUIRE(samples == 1);
+ REQUIRE(writes == std::vector<u32>{0, 1, 3});
+ REQUIRE(copies == std::vector<std::pair<u32, u32>>{{250, 4}, {251, 5}, {252, 8}, {0, 250}, {1, 251}, {3, 252}});
+}
+
+TEST_CASE("Native branch maps survive code caching and reject invalid addresses", "[native-shader]") {
+ std::vector<u64> code(64);
+ const std::array<u32, 4> offsets{0, 2, 4, 8};
+ Shader::Ampere::AppendNativeAddressMap(code, offsets);
+ REQUIRE(Shader::Ampere::HasNativeAddressMap(code));
+ REQUIRE(Shader::Ampere::NativeProgramAddress(code, 0) == 0x50U);
+ REQUIRE(Shader::Ampere::NativeProgramAddress(code, 16) == 0x68U);
+ const auto restored = code;
+ REQUIRE(Shader::Ampere::NativeProgramAddress(restored, 48) == 0xa8U);
+ REQUIRE_FALSE(Shader::Ampere::NativeProgramAddress(code, 1));
+ REQUIRE_FALSE(Shader::Ampere::NativeProgramAddress(code, 64));
+ code[code.size() - 2] = 100000;
+ REQUIRE_FALSE(Shader::Ampere::NativeProgramAddress(code, 0));
+}
+
+TEST_CASE("Native computed jumps retain table values while mapping code destinations", "[native-shader]") {
+ const std::array<Native, 12> input{{
+ {{0xff007321, 0x0002fcff, 0x00000000, 0}},
+ {{0x00007817, 0x00000001, 0x03800000, 0}},
+ {{0x00027819, 0x00000002, 0x000006ff, 0}},
+ {{0x02007b82, 0x00400000, 0x00000800, 0}},
+ {{0xff037421, 0x3f000000, 0x00010000, 0}},
+ {{0x00007949, 0xffffffa0, 0x0383ffff, 0}},
+ {{0x00007802, 0x3f800000, 0x00000f00, 0}},
+ {{0x00007947, 0x00000020, 0x03800000, 0}},
+ {{0x00007802, 0x40000000, 0x00000f00, 0}},
+ {{0x00007947, 0x00000000, 0x03800000, 0}},
+ {{0xff007322, 0x00007000, 0x000000ff, 0}},
+ VertexTransform.back(),
+ }};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ std::vector<u64> code(0x50 / 8);
+ for (const auto word : lowered.instructions) {
+ if ((code.size() * 8) % 32 == 0) code.push_back(0);
+ code.push_back(word);
+ }
+ code.push_back(0);
+ Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets);
+ class BranchEnvironment final : public EmptyEnvironment {
+ public:
+ explicit BranchEnvironment(const std::vector<u64>& code_) : code{code_} { stage = Shader::Stage::VertexB; }
+ u64 ReadInstruction(u32 address) override { return code.at(address / 8); }
+ u32 ReadCbufValue(u32 bank, u32 offset) override {
+ REQUIRE(bank == 1);
+ REQUIRE((offset == 0 || offset == 4));
+ return offset == 0 ? 0x60 : 0x80;
+ }
+ u32 TranslateProgramAddress(u32 address) const override {
+ const auto target = Shader::Ampere::NativeProgramAddress(code, address);
+ if (!target) throw std::logic_error("invalid synthetic branch target");
+ return *target;
+ }
+ const std::vector<u64>& code;
+ } env{code};
+ Shader::ObjectPool<Shader::Maxwell::Flow::Block> flow_pool;
+ Shader::ObjectPool<IR::Inst> inst_pool;
+ Shader::ObjectPool<IR::Block> block_pool;
+ Shader::Maxwell::Flow::CFG cfg{env, flow_pool, 0x50};
+ u32 jumps = 0;
+ for (const auto& block : cfg.Functions().front().blocks) {
+ if (block.end_class != Shader::Maxwell::Flow::EndClass::IndirectBranch) continue;
+ ++jumps;
+ REQUIRE(block.branch_offset == 0);
+ REQUIRE(block.indirect_branches.size() == 2);
+ for (const auto& branch : block.indirect_branches) {
+ REQUIRE((branch.address == 0x60 || branch.address == 0x80));
+ REQUIRE(branch.block->begin.Offset() == env.TranslateProgramAddress(branch.address));
+ REQUIRE(branch.block->begin.Offset() != branch.address);
+ }
+ }
+ REQUIRE(jumps == 1);
+ Shader::HostTranslateInfo host;
+ const auto program = Shader::Maxwell::TranslateProgram(inst_pool, block_pool, env, cfg, host);
+ REQUIRE_FALSE(program.blocks.empty());
+}
+
+TEST_CASE("Native forward branches retain paths beyond an early unconditional exit", "[native-shader]") {
+ const Native exit{{0x0000794d, 0, 0x03800000, 0x000fea00}};
+ const std::array<Native, 4> source{{
+ {{0x00000947, 0x20, 0x03800000, 0x000fea00}},
+ exit,
+ {{0x00017802, 42, 0x00000f00, 0x000fe400}},
+ exit,
+ }};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(source);
+ REQUIRE(lowered);
+ REQUIRE(lowered.source_size == sizeof(source));
+ REQUIRE(lowered.source_instruction_offsets.size() == source.size());
+ REQUIRE(lowered.instructions.size() == 4);
+ const auto truncated = Shader::Ampere::LowerToMaxwell(std::span{source}.first(2));
+ REQUIRE_FALSE(truncated);
+ REQUIRE(truncated.instructions.empty());
+}
+
+TEST_CASE("Native register-to-predicate writes only masked predicate bits", "[native-shader]") {
+ const std::array<Native, 2> source{{{{0x05007804, 7, 0, 0x000fe200}}, VertexTransform.back()}};
+ for (u32 byte = 0; byte < 4; ++byte) {
+ auto selected = source;
+ selected[0].words[2] = byte << 12;
+ const auto lowered = Shader::Ampere::LowerToMaxwell(selected);
+ REQUIRE(lowered);
+ REQUIRE(Shader::Maxwell::Decode(lowered.instructions[0]) == Shader::Maxwell::Opcode::R2P_imm);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.R2P_imm(lowered.instructions[0]);
+ for (u32 value = 0; value < 128; ++value) {
+ std::array<bool, 7> predicates{true, false, true, false, true, false, true};
+ std::unordered_map<const IR::Inst*, u32> values;
+ const auto get = [&](const IR::Value& v) -> u32 {
+ if (!v.IsImmediate()) return values.at(v.Inst());
+ return v.Type() == IR::Type::U1 ? static_cast<u32>(v.U1()) : v.U32();
+ };
+ for (const auto& inst : block.Instructions()) {
+ const auto arg = [&](size_t i) { return get(inst.Arg(i)); };
+ switch (inst.GetOpcode()) {
+ case IR::Opcode::GetRegister:
+ REQUIRE(static_cast<u32>(inst.Arg(0).Reg()) == 5);
+ values[&inst] = value << (byte * 8); break;
+ case IR::Opcode::GetPred: values[&inst] = predicates[static_cast<u32>(inst.Arg(0).Pred())]; break;
+ case IR::Opcode::SetPred: predicates[static_cast<u32>(inst.Arg(0).Pred())] = arg(1) != 0; break;
+ case IR::Opcode::BitFieldUExtract: values[&inst] = (arg(0) >> arg(1)) & ((1U << arg(2)) - 1); break;
+ case IR::Opcode::GetZeroFromOp: values[&inst] = arg(0) == 0; break;
+ case IR::Opcode::IEqual: values[&inst] = arg(0) == arg(1); break;
+ case IR::Opcode::LogicalNot: values[&inst] = !arg(0); break;
+ case IR::Opcode::SelectU1: values[&inst] = arg(0) ? arg(1) : arg(2); break;
+ default: FAIL("Unexpected R2P IR " << IR::NameOf(inst.GetOpcode()));
+ }
+ }
+ for (u32 bit = 0; bit < 7; ++bit)
+ REQUIRE(predicates[bit] == (bit < 3 ? bool(value & (1U << bit)) : bool(bit % 2 == 0)));
+ }
+ }
+ auto invalid = source;
+ invalid[0].words[2] = 1;
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(invalid));
+}
+
+TEST_CASE("Native 2D explicit LOD sampling preserves its paired coordinates", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0xc007b60, 0x30800812, 0x59e070b, 0x100f4200}}, VertexTransform.back()}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.TEXS(lowered.instructions[0]);
+ u32 samples = 0;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() != IR::Opcode::BoundImageSampleExplicitLod) continue;
+ ++samples;
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type == Shader::TextureType::Color2D);
+ const auto* coords = inst.Arg(1).Inst();
+ REQUIRE(coords->GetOpcode() == IR::Opcode::CompositeConstructF32x2);
+ for (u32 i = 0; i < 2; ++i)
+ REQUIRE(coords->Arg(i).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{12 + i});
+ REQUIRE(inst.Arg(2).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{18});
+ }
+ REQUIRE(samples == 1);
+}
+
+namespace {
+using Warp = std::array<u32, 32>;
+void ExecuteNativeWarp(IR::Block& block, u32 active,
+ std::array<std::array<u32, 256>, 32>& registers,
+ std::array<std::array<bool, 7>, 32>& predicates) {
+ std::unordered_map<const IR::Inst*, Warp> values;
+ const auto get = [&](const IR::Value& value) -> Warp {
+ if (!value.IsImmediate()) return values.at(value.Inst());
+ Warp result;
+ result.fill(value.Type() == IR::Type::U1 ? static_cast<u32>(value.U1()) : value.U32());
+ return result;
+ };
+ for (const auto& inst : block.Instructions()) {
+ std::array<Warp, 4> args{};
+ for (size_t i = 0; i < inst.NumArgs(); ++i) {
+ if (inst.Arg(i).Type() != IR::Type::Reg && inst.Arg(i).Type() != IR::Type::Pred)
+ args[i] = get(inst.Arg(i));
+ }
+ Warp result{};
+ const auto op = inst.GetOpcode();
+ if (op == IR::Opcode::SubgroupBallot || op == IR::Opcode::VoteAny || op == IR::Opcode::VoteAll) {
+ u32 mask = 0;
+ for (u32 lane = 0; lane < 32; ++lane)
+ if ((active & (1U << lane)) && args[0][lane]) mask |= 1U << lane;
+ result.fill(op == IR::Opcode::SubgroupBallot ? mask : op == IR::Opcode::VoteAny ? mask != 0 : mask == active);
+ } else {
+ for (u32 lane = 0; lane < 32; ++lane) {
+ const auto a = args[0][lane], b = args[1][lane], c = args[2][lane];
+ switch (op) {
+ case IR::Opcode::GetRegister: result[lane] = static_cast<u32>(inst.Arg(0).Reg()) == 255 ? 0 : registers[lane][static_cast<u32>(inst.Arg(0).Reg())]; break;
+ case IR::Opcode::SetRegister:
+ if ((active & (1U << lane)) && static_cast<u32>(inst.Arg(0).Reg()) != 255) registers[lane][static_cast<u32>(inst.Arg(0).Reg())] = b;
+ break;
+ case IR::Opcode::GetPred: result[lane] = predicates[lane][static_cast<u32>(inst.Arg(0).Pred())]; break;
+ case IR::Opcode::SetPred:
+ if (active & (1U << lane)) predicates[lane][static_cast<u32>(inst.Arg(0).Pred())] = b != 0;
+ break;
+ case IR::Opcode::BitCastF32U32: case IR::Opcode::BitCastU32F32: result[lane] = a; break;
+ case IR::Opcode::SelectU32: case IR::Opcode::SelectU1: result[lane] = a ? b : c; break;
+ case IR::Opcode::BitwiseAnd32: result[lane] = a & b; break;
+ case IR::Opcode::BitwiseOr32: result[lane] = a | b; break;
+ case IR::Opcode::ShiftRightLogical32: result[lane] = b < 32 ? a >> b : 0; break;
+ case IR::Opcode::ShiftLeftLogical32: result[lane] = b < 32 ? a << b : 0; break;
+ case IR::Opcode::INeg32: result[lane] = 0U - a; break;
+ case IR::Opcode::IAdd32: result[lane] = a + b; break;
+ case IR::Opcode::FindUMsb32: result[lane] = a ? 31U - std::countl_zero(a) : ~0U; break;
+ case IR::Opcode::BitFieldUExtract: result[lane] = (a >> b) & ((1U << c) - 1); break;
+ case IR::Opcode::ULessThan: result[lane] = a < b; break;
+ case IR::Opcode::SLessThan: result[lane] = static_cast<s32>(a) < static_cast<s32>(b); break;
+ case IR::Opcode::IEqual: result[lane] = a == b; break;
+ case IR::Opcode::GetZeroFromOp: result[lane] = a == 0; break;
+ case IR::Opcode::LogicalAnd: result[lane] = a && b; break;
+ case IR::Opcode::LogicalNot: result[lane] = !a; break;
+ case IR::Opcode::GetInBoundsFromOp: result[lane] = get(inst.Arg(0).Inst()->Arg(1))[lane] <= 31; break;
+ case IR::Opcode::FSwizzleAdd: {
+ const u32 mode = (c >> ((lane & 3) * 2)) & 3;
+ const float x = std::bit_cast<float>(a), y = std::bit_cast<float>(b);
+ const float ma = mode == 0 || mode == 2 ? -1.0f : mode == 1 ? 1.0f : 0.0f;
+ const float mb = mode == 0 || mode == 1 ? -1.0f : mode == 2 ? 1.0f : 0.0f;
+ result[lane] = std::bit_cast<u32>(x * ma + y * mb);
+ break;
+ }
+ case IR::Opcode::ShuffleButterfly: result[lane] = args[0][lane ^ b]; break;
+ case IR::Opcode::ShuffleIndex: result[lane] = (active & (1U << (b & 31))) ? args[0][b & 31] : 0xdeadbeefU; break;
+ default: FAIL("Unexpected warp-uniform IR " << IR::NameOf(op));
+ }
+ }
+ }
+ values.emplace(&inst, result);
+ }
+}
+}
+
+TEST_CASE("Native warp uniform transfers use the first selected active lane and preserve predicates", "[native-shader]") {
+ const u32 operation = GENERATE(0U, 1U, 2U, 3U, 4U, 5U, 6U, 7U);
+ const Native instruction = operation < 2
+ ? Native{{operation == 0 ? 0x020513c2U : 0x020593c2U, 0, 0x000e0000, 0}}
+ : operation < 4 ? Native{{0x003f7886, 0, operation == 2 ? 0x00800100U : 0x00800000U, 0}}
+ : Native{{0x0000781c, 0, operation == 4 ? 0x03f2f008U : operation == 5 ? 0x03f0f070U : operation == 6 ? 0x03f0e170U : 0x00f41070U, 0}};
+ const std::array<Native, 2> source{{instruction, VertexTransform.back()}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(source);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const auto word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::PSETP: visitor.PSETP(word); break;
+ case Opcode::SHR_imm: visitor.SHR_imm(word); break;
+ case Opcode::SHL_imm: visitor.SHL_imm(word); break;
+ case Opcode::MOV32I: visitor.MOV32I(word); break;
+ case Opcode::P2R_imm: visitor.P2R_imm(word); break;
+ case Opcode::R2P_imm: visitor.R2P_imm(word); break;
+ case Opcode::VOTE: visitor.VOTE(word); break;
+ case Opcode::IADD_reg: visitor.IADD_reg(word); break;
+ case Opcode::LOP_reg: visitor.LOP_reg(word); break;
+ case Opcode::FLO_reg: visitor.FLO_reg(word); break;
+ case Opcode::SHFL: visitor.SHFL(word); break;
+ case Opcode::SEL_reg: visitor.SEL_reg(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected warp-uniform lowering");
+ }
+ }
+ using Warp = std::array<u32, 32>;
+ for (const u32 active : {1U, 0x80000000U, 0x80000001U, 0x55555555U, 0xffffffffU}) {
+ for (const u32 selected : {0U, 1U, 0x80000000U, 0xaaaaaaaaU, 0xffffffffU}) {
+ std::array<std::array<u32, 256>, 32> registers{};
+ std::array<std::array<bool, 7>, 32> predicates{};
+ for (u32 lane = 0; lane < 32; ++lane) {
+ registers[lane][2] = 100 + lane * 17;
+ registers[lane][221] = 0xcafebeef;
+ registers[lane][239] = (selected & (1U << lane)) ? 64 : 0;
+ for (u32 p = 0; p < 7; ++p) predicates[lane][p] = ((lane + p) & 1) != 0;
+ predicates[lane][1] = ((selected & (1U << lane)) != 0) != (operation == 4);
+ }
+ const auto original_predicates = predicates;
+ ExecuteNativeWarp(block, active, registers, predicates);
+ const u32 ballot = active & (operation == 1 ? ~selected : selected);
+ const u32 first = ballot ? std::countr_zero(ballot) : 0;
+ for (u32 lane = 0; lane < 32; ++lane) {
+ if (!(active & (1U << lane))) continue;
+ auto expected_predicates = original_predicates[lane];
+ if (operation == 7) expected_predicates[2] = original_predicates[lane][1] && original_predicates[lane][0];
+ if (operation == 4) expected_predicates[1] = (selected & (1U << lane)) != 0;
+ if (operation == 5 || operation == 6) expected_predicates[0] = operation == 5;
+ REQUIRE(predicates[lane] == expected_predicates);
+ if (operation < 2) REQUIRE(registers[lane][221] == (ballot ? 100 + first * 17 : 0xcafebeef));
+ else if (operation < 4) REQUIRE(registers[lane][239] == (operation == 2 ? (ballot ? 64U : 0U) : (ballot == active ? 64U : 0U)));
+ }
+ }
+ }
+}
+
+TEST_CASE("Native uniform 64-bit loads populate consecutive registers", "[native-shader]") {
+ const std::array<Native, 6> input{{
+ {{0x00057ab9, 0x00c00000, 0x00000800, 0}},
+ {{0x05067abb, 0x00c00100, 0x08000a00, 0}},
+ {{0x00007c02, 6, 0x08000f00, 0}},
+ {{0x00017c02, 7, 0x08000f00, 0}},
+ {{0xff007322, 0x00007000, 0x000004ff, 0}},
+ VertexTransform.back(),
+ }};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (const auto word : lowered.instructions) {
+ using Shader::Maxwell::Opcode;
+ switch (Shader::Maxwell::Decode(word)) {
+ case Opcode::MOV_cbuf: visitor.MOV_cbuf(word); break;
+ case Opcode::MOV_reg: visitor.MOV_reg(word); break;
+ case Opcode::LDC: visitor.LDC(word); break;
+ case Opcode::AST: visitor.AST(word); break;
+ case Opcode::EXIT: break;
+ default: FAIL("Unexpected uniform pair lowering");
+ }
+ }
+ REQUIRE(EvaluateVertex(block, {}, {std::bit_cast<float>(0U), 3, 7, 9}) == std::array<float, 2>{3, 7});
+}
+
+TEST_CASE("Native texture dimension queries retain mip input and width-height outputs", "[native-shader]") {
+ const std::array<Native, 2> source{{{{0x03027b6f, 0x00800800, 0x000003ff, 0}}, VertexTransform.back()}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(source);
+ REQUIRE(lowered);
+ REQUIRE(lowered.required_texture_buffer == 2);
+ REQUIRE(Shader::Maxwell::Decode(lowered.instructions[0]) == Shader::Maxwell::Opcode::TXQ);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.TXQ(lowered.instructions[0]);
+ std::vector<std::pair<u32, u32>> outputs;
+ u32 queries = 0;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::BoundImageQueryDimensions) {
+ ++queries;
+ REQUIRE(inst.Arg(0).U32() == 32);
+ REQUIRE(inst.Arg(1).Inst()->Arg(0).Reg() == IR::Reg{3});
+ REQUIRE(inst.Arg(2).U1());
+ }
+ if (inst.GetOpcode() == IR::Opcode::SetRegister) {
+ const auto* extract = inst.Arg(1).Inst();
+ REQUIRE(extract->GetOpcode() == IR::Opcode::CompositeExtractU32x4);
+ outputs.emplace_back(static_cast<u32>(inst.Arg(0).Reg()), extract->Arg(1).U32());
+ }
+ }
+ REQUIRE(queries == 1);
+ REQUIRE(outputs == std::vector<std::pair<u32, u32>>{{2, 0}, {3, 1}});
+ auto invalid = source;
+ invalid[0].words[1] |= 1U << 30;
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(invalid));
+}
+
+TEST_CASE("Native scalar global stores retain both uniform address words and source data", "[native-shader]") {
+ std::array<Native, 2> input{{
+ {{0xff007986, 7, 0x0c10a904, 0}},
+ VertexTransform.back(),
+ }};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ REQUIRE(Shader::Maxwell::Decode(lowered.instructions.front()) == Shader::Maxwell::Opcode::STG);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.STG(lowered.instructions.front());
+ std::vector<u32> registers;
+ u32 stores{};
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::GetRegister)
+ registers.push_back(static_cast<u32>(inst.Arg(0).Reg()));
+ if (inst.GetOpcode() == IR::Opcode::CompositeConstructU32x2) {
+ REQUIRE(inst.Arg(0).Inst()->Arg(0).Reg() == IR::Reg::R220);
+ REQUIRE(inst.Arg(1).Inst()->Arg(0).Reg() == IR::Reg::R221);
+ }
+ if (inst.GetOpcode() == IR::Opcode::WriteGlobal32) {
+ REQUIRE(inst.Arg(1).Inst()->Arg(0).Reg() == IR::Reg::R7);
+ ++stores;
+ }
+ }
+ std::sort(registers.begin(), registers.end());
+ REQUIRE(registers == std::vector<u32>{7, 220, 221});
+ REQUIRE(stores == 1);
+ input[0].words[2] |= 1; // An unaligned uniform address pair is unsupported.
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(input));
+ input[0].words[2] = 0x0c10ab04; // A different store width must not become scalar.
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(input));
+}
+
+TEST_CASE("Captured native compute compiles with shared memory and atomics", "[.native-compute-captured]") {
+ const char* directory = std::getenv("EDEN_NATIVE_SHADER_FIXTURES");
+ REQUIRE(directory != nullptr);
+ std::ifstream file(std::filesystem::path{directory} / "compute.source.bin", std::ios::binary | std::ios::ate);
+ REQUIRE(file.good());
+ const auto size = static_cast<size_t>(file.tellg());
+ REQUIRE(size % 16 == 0);
+ std::vector<Native> input(size / 16);
+ file.seekg(0);
+ file.read(reinterpret_cast<char*>(input.data()), size);
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ INFO("offset=" << lowered.error_offset << " error=" << lowered.error);
+ REQUIRE(lowered);
+ std::vector<u64> code(0x50 / 8);
+ for (u64 word : lowered.instructions) {
+ if ((code.size() * 8) % 32 == 0) code.push_back(0);
+ code.push_back(word);
+ }
+ code.push_back(0);
+ Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets);
+ std::array<u32, 64> qmd{};
+ std::ifstream descriptor(std::filesystem::path{directory} / "compute.qmd.bin", std::ios::binary);
+ REQUIRE(descriptor.good());
+ descriptor.read(reinterpret_cast<char*>(qmd.data()), sizeof(qmd));
+ const std::array<u32, 3> group{qmd[18] >> 16, qmd[19] & 0xffff, qmd[19] >> 16};
+ const u32 shared_size = qmd[17] & 0x3ffff;
+ class ComputeFixtureEnvironment final : public EmptyEnvironment {
+ public:
+ ComputeFixtureEnvironment(const std::vector<u64>& code_, u32 shared_, std::array<u32, 3> group_, Shader::TextureType texture_)
+ : code{code_}, shared{shared_}, group{group_}, texture{texture_} {
+ stage = Shader::Stage::Compute;
+ start_address = 0x50;
+ is_proprietary_driver = true;
+ }
+ u64 ReadInstruction(u32 address) override { return code.at(address / 8); }
+ u32 TranslateProgramAddress(u32 address) const override {
+ const auto translated = Shader::Ampere::NativeProgramAddress(code, address);
+ if (!translated) throw std::logic_error("invalid compute branch target");
+ return *translated;
+ }
+ u32 ReadCbufValue(u32, u32) override { return 0; }
+ u32 TextureBoundBuffer() const override { return 0; }
+ u32 SharedMemorySize() const override { return shared; }
+ std::array<u32, 3> WorkgroupSize() const override { return group; }
+ Shader::TextureType ReadTextureType(u32) override { return texture; }
+ const std::vector<u64>& code;
+ u32 shared;
+ std::array<u32, 3> group;
+ Shader::TextureType texture;
+ } env{code, shared_size, group,
+ std::any_of(input.begin(), input.end(), [](const auto& instruction) {
+ return instruction.Bits(0, 12) == 0xb66 && instruction.Bits(61, 3) == 2;
+ }) ? Shader::TextureType::Color3D : std::any_of(input.begin(), input.end(), [](const auto& instruction) {
+ return (instruction.Bits(0, 12) == 0xb66 && instruction.Bits(61, 3) == 1) ||
+ instruction.Bits(0, 12) == 0xb97 || instruction.Bits(0, 12) == 0xb9b;
+ }) ? Shader::TextureType::Color2D : Shader::TextureType::Color1D};
+ Shader::ObjectPool<Shader::Maxwell::Flow::Block> flow_pool;
+ Shader::ObjectPool<IR::Inst> inst_pool;
+ Shader::ObjectPool<IR::Block> block_pool;
+ Shader::Maxwell::Flow::CFG cfg{env, flow_pool, env.StartAddress()};
+ Shader::HostTranslateInfo host;
+ host.support_int64 = true;
+ host.support_float64 = true;
+ host.support_float16 = true;
+ host.support_conditional_barrier = true;
+ const auto program = Shader::Maxwell::TranslateProgram(inst_pool, block_pool, env, cfg, host);
+ REQUIRE(program.stage == Shader::Stage::Compute);
+ REQUIRE_FALSE(program.blocks.empty());
+ u32 barriers{}, shared_atomics{}, global_atomics{}, stores{}, image_stores{}, image_loads{};
+ for (const auto* block : program.blocks) for (const auto& inst : block->Instructions()) {
+ image_loads += inst.GetOpcode() == IR::Opcode::ImageRead || inst.GetOpcode() == IR::Opcode::BoundImageRead;
+ barriers += inst.GetOpcode() == IR::Opcode::Barrier;
+ shared_atomics += inst.GetOpcode() == IR::Opcode::SharedAtomicIAdd32;
+ global_atomics += inst.GetOpcode() == IR::Opcode::GlobalAtomicIAdd32 || inst.GetOpcode() == IR::Opcode::StorageAtomicIAdd32;
+ stores += inst.GetOpcode() == IR::Opcode::WriteGlobal32 || inst.GetOpcode() == IR::Opcode::WriteStorage32;
+ image_stores += inst.GetOpcode() == IR::Opcode::ImageWrite || inst.GetOpcode() == IR::Opcode::BoundImageWrite;
+ }
+ const bool surface = std::any_of(input.begin(), input.end(), [](const auto& instruction) { return instruction.Bits(0, 12) == 0xb9b; });
+ if (surface) {
+ const auto expected_barriers = std::count_if(input.begin(), input.end(), [](const auto& instruction) {
+ return instruction.Bits(0, 12) == 0xb1d;
+ });
+ REQUIRE(barriers >= expected_barriers);
+ REQUIRE(image_stores >= 1);
+ const auto expected_loads = std::count_if(input.begin(), input.end(), [](const auto& instruction) {
+ return instruction.Bits(0, 12) == 0xb97;
+ });
+ REQUIRE(image_loads >= expected_loads);
+ } else if (std::any_of(input.begin(), input.end(), [](const auto& instruction) { return instruction.Bits(0, 12) == 0xf8c; })) {
+ REQUIRE(barriers >= 2);
+ REQUIRE(shared_atomics >= 4);
+ REQUIRE(global_atomics >= 2);
+ } else if (shared_size) {
+ REQUIRE(barriers >= 5);
+ REQUIRE(shared_atomics >= 2);
+ REQUIRE(global_atomics >= 1);
+ REQUIRE(stores >= 8);
+ } else {
+ REQUIRE(barriers == 0);
+ REQUIRE(stores >= 4);
+ }
+}
+
+namespace {
+double DecodeTestHalf(u16 bits) {
+ const u32 exponent = (bits >> 10) & 31, mantissa = bits & 1023;
+ const double value = exponent == 31 ? (mantissa ? std::numeric_limits<double>::quiet_NaN()
+ : std::numeric_limits<double>::infinity())
+ : exponent == 0 ? std::ldexp(static_cast<double>(mantissa), -24)
+ : std::ldexp(static_cast<double>(1024 + mantissa), static_cast<int>(exponent) - 25);
+ return (bits & 0x8000) ? -value : value;
+}
+u16 EncodeTestHalf(double value) {
+ const u16 sign = std::signbit(value) ? 0x8000 : 0;
+ value = std::abs(value);
+ if (std::isnan(value)) return 0xfe01; // Deliberately noncanonical: ReLU must canonicalize it.
+ if (value >= 65520) return sign | 0x7c00;
+ if (value < std::ldexp(1.0, -14))
+ return sign | static_cast<u16>(std::nearbyint(std::ldexp(value, 24)));
+ int exponent{};
+ std::frexp(value, &exponent);
+ --exponent;
+ u32 mantissa = static_cast<u32>(std::nearbyint(std::ldexp(value, 10 - exponent)));
+ if (mantissa == 2048) { ++exponent; mantissa = 1024; }
+ return sign | static_cast<u16>(((exponent + 15) << 10) | (mantissa - 1024));
+}
+struct ScalarNativeMachine {
+ std::array<u32, 256> regs{};
+ std::array<bool, 7> predicates{};
+ std::unordered_map<u64, u32> constants, global, shared;
+ u32 workgroup_barriers{}, device_barriers{};
+ void Execute(IR::Block& block) {
+ std::unordered_map<const IR::Inst*, u64> values;
+ bool carry{}, zero{}, sign{}, overflow{};
+ const auto get = [&](const IR::Value& value) -> u64 {
+ if (!value.IsImmediate()) return values.at(value.Inst());
+ if (value.Type() == IR::Type::U1) return value.U1();
+ if (value.Type() == IR::Type::F16 || value.Type() == IR::Type::U16) return value.U16();
+ if (value.Type() == IR::Type::U64) return value.U64();
+ if (value.Type() == IR::Type::F32) return std::bit_cast<u32>(value.F32());
+ return value.U32();
+ };
+ for (const auto& inst : block.Instructions()) {
+ const auto arg = [&](size_t i) { return get(inst.Arg(i)); };
+ u64 result{};
+ switch (inst.GetOpcode()) {
+ case IR::Opcode::BitCastF32U32: case IR::Opcode::BitCastU32F32: result = arg(0); break;
+ case IR::Opcode::UnpackFloat2x16: result = (arg(0) & 0xffff) | ((arg(0) >> 16) << 32); break;
+ case IR::Opcode::PackFloat2x16: result = (arg(0) & 0xffff) | ((arg(0) >> 32) << 16); break;
+ case IR::Opcode::CompositeConstructF16x2: result = arg(0) | (arg(1) << 32); break;
+ case IR::Opcode::CompositeExtractF16x2: result = (arg(0) >> (arg(1) * 32)) & 0xffff; break;
+ case IR::Opcode::FPNeg16: result = arg(0) ^ 0x8000; break;
+ case IR::Opcode::FPAbs16: result = arg(0) & 0x7fff; break;
+ case IR::Opcode::FPFma16: result = EncodeTestHalf(std::fma(DecodeTestHalf(static_cast<u16>(arg(0))),
+ DecodeTestHalf(static_cast<u16>(arg(1))), DecodeTestHalf(static_cast<u16>(arg(2))))); break;
+ case IR::Opcode::FPFloor32: result = std::bit_cast<u32>(std::floor(std::bit_cast<float>(static_cast<u32>(arg(0))))); break;
+ case IR::Opcode::ConvertF32F16: result = std::bit_cast<u32>(static_cast<float>(DecodeTestHalf(static_cast<u16>(arg(0))))); break;
+ case IR::Opcode::FPOrdEqual32: result = std::bit_cast<float>(static_cast<u32>(arg(0))) == std::bit_cast<float>(static_cast<u32>(arg(1))); break;
+ case IR::Opcode::FPOrdGreaterThanEqual32: result = std::bit_cast<float>(static_cast<u32>(arg(0))) >= std::bit_cast<float>(static_cast<u32>(arg(1))); break;
+ case IR::Opcode::FPMax32: result = std::bit_cast<u32>(std::fmax(std::bit_cast<float>(static_cast<u32>(arg(0))), std::bit_cast<float>(static_cast<u32>(arg(1))))); break;
+ case IR::Opcode::FPMin32: result = std::bit_cast<u32>(std::fmin(std::bit_cast<float>(static_cast<u32>(arg(0))), std::bit_cast<float>(static_cast<u32>(arg(1))))); break;
+ case IR::Opcode::ConvertU32F32: result = static_cast<u32>(std::bit_cast<float>(static_cast<u32>(arg(0)))); break;
+ case IR::Opcode::FPTrunc32: result = std::bit_cast<u32>(std::trunc(std::bit_cast<float>(static_cast<u32>(arg(0))))); break;
+ case IR::Opcode::FPNeg32: result = arg(0) ^ 0x80000000U; break;
+ case IR::Opcode::FPAbs32: result = arg(0) & 0x7fffffffU; break;
+ case IR::Opcode::ConvertF16F32: result = EncodeTestHalf(std::bit_cast<float>(static_cast<u32>(arg(0)))); break;
+ case IR::Opcode::FPRoundEven32: result = std::bit_cast<u32>(std::nearbyint(std::bit_cast<float>(static_cast<u32>(arg(0))))); break;
+ case IR::Opcode::FPIsNan32: result = std::isnan(std::bit_cast<float>(static_cast<u32>(arg(0)))); break;
+ case IR::Opcode::FPClamp32: result = std::bit_cast<u32>(std::clamp(std::bit_cast<float>(static_cast<u32>(arg(0))), std::bit_cast<float>(static_cast<u32>(arg(1))), std::bit_cast<float>(static_cast<u32>(arg(2))))); break;
+ case IR::Opcode::ConvertS32F32: {
+ const float value = std::bit_cast<float>(static_cast<u32>(arg(0)));
+ result = std::isnan(value) ? 0x87654321U : static_cast<u32>(static_cast<s32>(value));
+ break;
+ }
+ case IR::Opcode::GetRegister: result = inst.Arg(0).Reg() == IR::Reg::RZ ? 0 : regs[static_cast<u32>(inst.Arg(0).Reg())]; break;
+ case IR::Opcode::SetRegister: if (inst.Arg(0).Reg() != IR::Reg::RZ) regs[static_cast<u32>(inst.Arg(0).Reg())] = static_cast<u32>(arg(1)); break;
+ case IR::Opcode::GetPred: result = predicates[static_cast<u32>(inst.Arg(0).Pred())]; break;
+ case IR::Opcode::SetPred: predicates[static_cast<u32>(inst.Arg(0).Pred())] = arg(1) != 0; break;
+ case IR::Opcode::IAdd32: result = static_cast<u32>(arg(0) + arg(1)); break;
+ case IR::Opcode::ISub32: result = static_cast<u32>(arg(0) - arg(1)); break;
+ case IR::Opcode::IMul32: result = static_cast<u32>(arg(0) * arg(1)); break;
+ case IR::Opcode::IAdd64: result = arg(0) + arg(1); break;
+ case IR::Opcode::UMax32: result = std::max(arg(0), arg(1)); break;
+ case IR::Opcode::UMin32: result = std::min(arg(0), arg(1)); break;
+ case IR::Opcode::FindUMsb32: result = arg(0) ? 31U - std::countl_zero(static_cast<u32>(arg(0))) : ~0U; break;
+ case IR::Opcode::BitReverse32: {
+ const u32 source = static_cast<u32>(arg(0));
+ for (u32 bit = 0; bit < 32; ++bit) result |= u64{(source >> bit) & 1U} << (31 - bit);
+ break;
+ }
+ case IR::Opcode::ShiftLeftLogical64: result = arg(1) < 64 ? arg(0) << arg(1) : 0; break;
+ case IR::Opcode::ShiftRightLogical64: result = arg(1) < 64 ? arg(0) >> arg(1) : 0; break;
+ case IR::Opcode::INeg32: result = 0U - static_cast<u32>(arg(0)); break;
+ case IR::Opcode::ShiftLeftLogical32: result = arg(1) < 32 ? static_cast<u32>(arg(0) << arg(1)) : 0; break;
+ case IR::Opcode::ShiftRightArithmetic32: result = static_cast<u32>(static_cast<s32>(arg(0)) >> std::min<u64>(arg(1), 31)); break;
+ case IR::Opcode::ShiftRightLogical32: result = arg(1) < 32 ? static_cast<u32>(arg(0)) >> arg(1) : 0; break;
+ case IR::Opcode::BitwiseAnd32: result = arg(0) & arg(1); break;
+ case IR::Opcode::BitwiseOr32: result = arg(0) | arg(1); break;
+ case IR::Opcode::BitwiseXor32: result = arg(0) ^ arg(1); break;
+ case IR::Opcode::BitwiseNot32: result = arg(0) ^ 0xffffffffULL; break;
+ case IR::Opcode::BitFieldUExtract: result = (arg(0) >> arg(1)) & ((u64{1} << arg(2)) - 1); break;
+ case IR::Opcode::BitFieldInsert: { const u64 mask = ((u64{1} << arg(3)) - 1) << arg(2); result = (arg(0) & ~mask) | ((arg(1) << arg(2)) & mask); break; }
+ case IR::Opcode::SelectF32: case IR::Opcode::SelectU32: case IR::Opcode::SelectU1: result = arg(0) ? arg(1) : arg(2); break;
+ case IR::Opcode::LogicalAnd: result = arg(0) && arg(1); break;
+ case IR::Opcode::LogicalOr: result = arg(0) || arg(1); break;
+ case IR::Opcode::LogicalNot: result = !arg(0); break;
+ case IR::Opcode::SMin32: result = static_cast<u32>(std::min(static_cast<s32>(arg(0)), static_cast<s32>(arg(1)))); break;
+ case IR::Opcode::SMax32: result = static_cast<u32>(std::max(static_cast<s32>(arg(0)), static_cast<s32>(arg(1)))); break;
+ case IR::Opcode::SLessThan: result = static_cast<s32>(arg(0)) < static_cast<s32>(arg(1)); break;
+ case IR::Opcode::ULessThan: result = arg(0) < arg(1); break;
+ case IR::Opcode::IEqual: result = arg(0) == arg(1); break;
+ case IR::Opcode::GetCFlag: result = carry; break;
+ case IR::Opcode::SetCFlag: carry = arg(0) != 0; break;
+ case IR::Opcode::GetZFlag: result = zero; break;
+ case IR::Opcode::SetZFlag: zero = arg(0) != 0; break;
+ case IR::Opcode::GetSFlag: result = sign; break;
+ case IR::Opcode::SetSFlag: sign = arg(0) != 0; break;
+ case IR::Opcode::GetOFlag: result = overflow; break;
+ case IR::Opcode::SetOFlag: overflow = arg(0) != 0; break;
+ case IR::Opcode::GetCarryFromOp: { const auto* sum = inst.Arg(0).Inst(); result = get(sum->Arg(0)) + get(sum->Arg(1)) > 0xffffffffULL; break; }
+ case IR::Opcode::GetZeroFromOp: result = arg(0) == 0; break;
+ case IR::Opcode::GetSignFromOp: result = (arg(0) & 0x80000000U) != 0; break;
+ case IR::Opcode::GetOverflowFromOp: { const auto* sum = inst.Arg(0).Inst(); result = ((get(sum->Arg(0)) ^ arg(0)) & (get(sum->Arg(1)) ^ arg(0)) & 0x80000000U) != 0; break; }
+ case IR::Opcode::GetCbufU32: result = constants.at((arg(0) << 32) | arg(1)); break;
+ case IR::Opcode::CompositeConstructU32x2: result = arg(0) | (arg(1) << 32); break;
+ case IR::Opcode::PackUint2x32: case IR::Opcode::UnpackUint2x32: result = arg(0); break;
+ case IR::Opcode::CompositeExtractU32x2: result = static_cast<u32>(arg(0) >> (arg(1) * 32)); break;
+ case IR::Opcode::WriteSharedU64: shared[arg(0)] = static_cast<u32>(arg(1)); shared[arg(0) + 4] = static_cast<u32>(arg(1) >> 32); break;
+ case IR::Opcode::LoadSharedU64: result = shared.at(arg(0)) | (u64{shared.at(arg(0) + 4)} << 32); break;
+ case IR::Opcode::WriteGlobal32: global[arg(0)] = static_cast<u32>(arg(1)); break;
+ case IR::Opcode::WriteSharedU32: shared[arg(0)] = static_cast<u32>(arg(1)); break;
+ case IR::Opcode::LoadSharedU32: result = shared.at(arg(0)); break;
+ case IR::Opcode::GlobalAtomicIAdd32: result = global.at(arg(0)); global[arg(0)] += static_cast<u32>(arg(1)); break;
+ case IR::Opcode::SharedAtomicIAdd32: result = shared.at(arg(0)); shared[arg(0)] += static_cast<u32>(arg(1)); break;
+ case IR::Opcode::Barrier: ++workgroup_barriers; break;
+ case IR::Opcode::DeviceMemoryBarrier: ++device_barriers; break;
+ default: FAIL("Unexpected scalar compute IR " << IR::NameOf(inst.GetOpcode()));
+ }
+ values[&inst] = result;
+ }
+ }
+};
+void TranslateScalarNative(std::span<const Native> input, IR::Block& block) {
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ INFO("offset=" << lowered.error_offset << " error=" << lowered.error);
+ REQUIRE(lowered);
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (u64 word : lowered.instructions) {
+ const auto opcode = Shader::Maxwell::Decode(word);
+ if (opcode == Shader::Maxwell::Opcode::EXIT) continue;
+ switch (opcode) {
+#define INST(name, cute, encode) case Shader::Maxwell::Opcode::name: visitor.name(word); break;
+#include "shader_recompiler/frontend/maxwell/maxwell.inc"
+#undef INST
+ }
+ }
+}
+}
+
+TEST_CASE("Native uniform comparisons and selects preserve independent predicate banks", "[native-shader]") {
+ const std::array<Native, 7> input{{
+ {{0x09057891, 0x40, 0x0f8e303f, 0}},
+ {{0x0500728c, 7, 0x0bf21070, 0}},
+ {{0x05057287, 7, 0x08800000, 0}},
+ {{0x05057290, 7, 0x0fffe13f, 0}},
+ {{0x0400728c, 6, 0x0bf01070, 0}},
+ {{0x04047287, 6, 0x08000000, 0}},
+ VertexTransform.back(),
+ }};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (u32 width : {100U, 256U}) {
+ ScalarNativeMachine machine;
+ machine.regs[225] = 2;
+ machine.regs[223] = width;
+ machine.regs[220] = 40;
+ machine.regs[222] = 32;
+ for (u32 p = 0; p < 7; ++p) machine.predicates[p] = (p & 1) != 0;
+ const auto original = machine.predicates;
+ machine.Execute(block);
+ REQUIRE(machine.regs[221] == width - std::min(192U, width));
+ REQUIRE(machine.regs[220] == 32);
+ REQUIRE(((machine.regs[239] & 0x20) != 0) == (192U < width));
+ REQUIRE((machine.regs[239] & 0x40) == 0);
+ REQUIRE(machine.predicates == original);
+ }
+}
+
+TEST_CASE("Native global stores carry a register offset into the high address word", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x02007986, 9, 0x0810a906, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ ScalarNativeMachine machine;
+ machine.regs[222] = 0xfffffff0;
+ machine.regs[223] = 0x21;
+ machine.regs[2] = 0x30;
+ machine.regs[9] = 0xdeadbeef;
+ machine.Execute(block);
+ REQUIRE(machine.global == std::unordered_map<u64, u32>{{0x2200000020ULL, 0xdeadbeef}});
+}
+
+TEST_CASE("Native LEA predicate carry reaches the high pointer word", "[native-shader]") {
+ const std::array<Native, 4> input{{
+ {{0x00007a11, 0x800, 0x078010ff, 0}},
+ {{0xff017a10, 0x900, 0x007fe4ff, 0}},
+ {{0x00007386, 7, 0x0010a900, 0}},
+ VertexTransform.back(),
+ }};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (u32 base : {0x100U, 0xfffffff0U}) {
+ ScalarNativeMachine machine;
+ machine.regs[0] = 8;
+ machine.regs[7] = 0x12345678;
+ machine.constants[0x20] = base;
+ machine.constants[0x24] = 0x21;
+ machine.Execute(block);
+ REQUIRE(machine.global == std::unordered_map<u64, u32>{{(u64{0x21} << 32) + base + 32, 0x12345678}});
+ }
+}
+
+TEST_CASE("Native shared atomics and barriers retain real shared-memory effects", "[native-shader]") {
+ const std::array<Native, 6> input{{
+ {{0x03007388, 0x407, 0x4800, 0}},
+ {{0x03097984, 0x400, 0x4800, 0}},
+ {{0xff0a738c, 0x7fc09, 0, 0}},
+ {{0x00007992, 0, 0x5000, 0}},
+ {{0x00007b1d, 0, 0, 0}},
+ VertexTransform.back(),
+ }};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ ScalarNativeMachine machine;
+ machine.regs[3] = 3;
+ machine.regs[7] = 5;
+ machine.shared[0x7fc] = 100;
+ machine.Execute(block);
+ REQUIRE(machine.shared.at(16) == 5);
+ REQUIRE(machine.shared.at(0x7fc) == 105);
+ REQUIRE(machine.regs[10] == 100);
+ REQUIRE(machine.workgroup_barriers == 1);
+ REQUIRE(machine.device_barriers == 1);
+}
+
+TEST_CASE("Native uniform high shifts and multiply operands retain numeric values", "[native-shader]") {
+ std::array<Native, 7> input{{
+ {{0x00057882, 123, 0, 0}},
+ {{0x3f047899, 5, 0x08010605, 0}},
+ {{0x05057899, 5, 0x0800063f, 0}},
+ {{0x01027c24, 5, 0x0f8e0202, 0}},
+ {{0x00007c17, 4, 0x0b800200, 0}},
+ {{0x000a7424, 4, 0x078e0205, 0}},
+ VertexTransform.back(),
+ }};
+ for (u32 value : {0U, 1U, 123U, 0x07ffffffU, 0xffffffffU}) {
+ input[0].words[1] = value;
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ ScalarNativeMachine machine;
+ machine.regs[0] = 5000;
+ machine.regs[1] = 11;
+ machine.regs[2] = 7;
+ machine.regs[5] = 3;
+ machine.Execute(block);
+ const u32 shifted = value << 5;
+ const u32 minimum = static_cast<u32>(std::min(5000, static_cast<s32>(shifted)));
+ REQUIRE(machine.regs[220] == shifted);
+ REQUIRE(machine.regs[221] == shifted);
+ REQUIRE(machine.regs[2] == 11U * shifted + 7);
+ REQUIRE(machine.regs[0] == minimum);
+ REQUIRE(machine.regs[10] == minimum * 3U + 4);
+ }
+}
+
+TEST_CASE("Native bit reversal and register shifts preserve FFT indexing", "[native-shader]") {
+ const std::array<Native, 6> input{{
+ {{0x00037301, 19, 0, 0}},
+ {{0x03017211, 255, 0x078f50ff, 0}},
+ {{0xff087219, 4, 0x00011613, 0}},
+ {{0x0f0f7219, 12, 0x00000eff, 0}},
+ {{0xff047224, 255, 0x078e0a0c, 0}},
+ VertexTransform.back(),
+ }};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (u32 shift : {0U, 1U, 10U, 31U, 32U, 33U, 63U}) {
+ ScalarNativeMachine machine;
+ machine.regs[19] = 0x12345678;
+ machine.regs[4] = shift;
+ machine.regs[12] = shift;
+ machine.regs[15] = 0x9abcdef0;
+ machine.Execute(block);
+ REQUIRE(machine.regs[3] == 0x1e6a2c48);
+ REQUIRE(machine.regs[1] == (0x1e6a2c48U >> 22));
+ REQUIRE(machine.regs[8] == (shift < 32 ? 0x12345678U >> shift : 0));
+ REQUIRE(machine.regs[15] == (0x9abcdef0U << (shift & 31)));
+ REQUIRE(machine.regs[4] == 0U - shift);
+ }
+}
+
+TEST_CASE("Native paired shared values keep both words under scaled addressing", "[native-shader]") {
+ const std::array<Native, 3> input{{
+ {{0x13007388, 4, 0x8a00, 0}},
+ {{0x13027984, 0, 0x8a00, 0}},
+ VertexTransform.back(),
+ }};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ ScalarNativeMachine machine;
+ machine.regs[19] = 17;
+ machine.regs[4] = 0x12345678;
+ machine.regs[5] = 0x9abcdef0;
+ machine.Execute(block);
+ REQUIRE(machine.shared.at(17 * 8) == 0x12345678);
+ REQUIRE(machine.shared.at(17 * 8 + 4) == 0x9abcdef0);
+ REQUIRE(machine.regs[2] == 0x12345678);
+ REQUIRE(machine.regs[3] == 0x9abcdef0);
+}
+
+TEST_CASE("Native formatted surface stores retain coordinates channels and descriptor", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x00007b9b, 0x60000a04, 0x00100f00, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ u32 writes{};
+ for (const auto& inst : block.Instructions()) if (inst.GetOpcode() == IR::Opcode::BoundImageWrite) {
+ REQUIRE(inst.Arg(0).U32() == 40);
+ const auto info = inst.Flags<IR::TextureInstInfo>();
+ REQUIRE(info.type.Value() == Shader::TextureType::Color2D);
+ const auto* coordinates = inst.Arg(1).Inst();
+ REQUIRE(coordinates->Arg(0).Inst()->Arg(0).Reg() == IR::Reg::R0);
+ REQUIRE(coordinates->Arg(1).Inst()->Arg(0).Reg() == IR::Reg::R1);
+ const auto* color = inst.Arg(2).Inst();
+ for (u32 component = 0; component < 4; ++component)
+ REQUIRE(static_cast<u32>(color->Arg(component).Inst()->Arg(0).Reg()) == 4 + component);
+ ++writes;
+ }
+ REQUIRE(writes == 1);
+}
+
+TEST_CASE("Native predicated predicate copies retain their guard and source", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x0000281c, 0, 0x01f0f070, 0}}, VertexTransform.back()}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ const auto predicate = Shader::Maxwell::Instruction{lowered.instructions.front()}.Pred();
+ REQUIRE(predicate.index == 2);
+ REQUIRE_FALSE(predicate.negated);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (bool source : {false, true}) {
+ ScalarNativeMachine machine;
+ machine.predicates[3] = source;
+ machine.Execute(block);
+ REQUIRE(machine.predicates[0] == source);
+ }
+}
+
+TEST_CASE("Native NTZ nearest conversion rounds ties to even and clears NaNs", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x00117305, 2, 0x00203100, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ u32 rounds{};
+ for (const auto& inst : block.Instructions()) if (inst.GetOpcode() == IR::Opcode::FPRoundEven32) {
+ REQUIRE(inst.Flags<IR::FpControl>().fmz_mode == IR::FmzMode::None);
+ ++rounds;
+ }
+ REQUIRE(rounds == 1);
+ for (const auto [value, expected] : std::array<std::pair<float, s32>, 9>{{
+ {0.5f, 0}, {1.5f, 2}, {2.5f, 2}, {-0.5f, 0}, {-1.5f, -2}, {-2.5f, -2},
+ {0.9f, 1}, {-0.9f, -1}, {std::numeric_limits<float>::quiet_NaN(), 0},
+ }}) {
+ ScalarNativeMachine machine;
+ machine.regs[2] = std::bit_cast<u32>(value);
+ machine.Execute(block);
+ REQUIRE(machine.regs[17] == static_cast<u32>(expected));
+ }
+ auto unsupported = input;
+ unsupported[0].words[2] ^= 1U << 13;
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(unsupported));
+}
+
+TEST_CASE("Native predicate packing merges only selected predicate bits", "[native-shader]") {
+ for (u32 mask : {0U, 8U, 16U, 0x55U, 0x7fU}) {
+ const std::array<Native, 2> input{{{{0x02137803, mask, 0, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (u32 bits = 0; bits < 128; ++bits) {
+ ScalarNativeMachine machine;
+ machine.regs[2] = 0xa5a5a5a5U;
+ for (u32 i = 0; i < 7; ++i) machine.predicates[i] = (bits >> i) & 1;
+ const auto saved = machine.predicates;
+ machine.Execute(block);
+ REQUIRE(machine.regs[19] == ((0xa5a5a5a5U & ~mask) | (bits & mask)));
+ REQUIRE(machine.predicates == saved);
+ }
+ }
+}
+
+TEST_CASE("Native signed NTZ floor conversion rounds negative fractions down", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x00237305, 0x24, 0x00207100, 0x000e2400}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (const auto [value, expected] : std::array<std::pair<float, s32>, 8>{{
+ {1.9f, 1}, {-1.1f, -2}, {-0.1f, -1}, {0.9f, 0},
+ {0.0f, 0}, {-2.0f, -2}, {12345.5f, 12345},
+ {std::numeric_limits<float>::quiet_NaN(), 0},
+ }}) {
+ ScalarNativeMachine machine;
+ machine.regs[36] = std::bit_cast<u32>(value);
+ machine.Execute(block);
+ REQUIRE(machine.regs[35] == static_cast<u32>(expected));
+ }
+}
+
+TEST_CASE("Native dependency waits retain the full six-bit pending-operation count", "[native-shader]") {
+ for (u32 count : {0U, 1U, 15U, 22U, 63U}) {
+ const std::array<Native, 2> input{{{{0x0000791a, 0xd000U | (count << 6), 0, 0}}, VertexTransform.back()}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ REQUIRE(lowered.instructions.size() == 1);
+ }
+ const std::array<Native, 2> unsupported{{{{0x0000791a, 0xc000, 0, 0}}, VertexTransform.back()}};
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(unsupported));
+}
+
+TEST_CASE("Native histogram warp match compares both address words and preserves predicates", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x00ff73a1, 0, 0x00000200, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (u32 active : {1U, 0x80000000U, 0x80000001U, 0x55555555U, 0xffffffffU}) {
+ for (u32 mismatch : {0U, 1U, 2U}) {
+ std::array<std::array<u32, 256>, 32> registers{};
+ std::array<std::array<bool, 7>, 32> predicates{};
+ const u32 first = std::countr_zero(active);
+ for (u32 lane = 0; lane < 32; ++lane) {
+ registers[lane][0] = 0xfffffff0U + (mismatch == 1 && lane != first ? 4 : 0);
+ registers[lane][1] = 0x21U + (mismatch == 2 && lane != first ? 1 : 0);
+ for (u32 p = 0; p < 7; ++p) predicates[lane][p] = ((lane + p) & 1) != 0;
+ }
+ const auto original = predicates;
+ ExecuteNativeWarp(block, active, registers, predicates);
+ for (u32 lane = 0; lane < 32; ++lane) {
+ auto expected = original[lane];
+ if (active & (1U << lane)) expected[0] = mismatch == 0 || std::popcount(active) == 1;
+ REQUIRE(predicates[lane] == expected);
+ }
+ }
+ }
+}
+
+TEST_CASE("Native histogram reduction sums only active lanes with unsigned wrap", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x030573c4, 0, 0x0000c000, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (u32 active : {1U, 0x80000000U, 0x80000001U, 0x55555555U, 0xffffffffU}) {
+ std::array<std::array<u32, 256>, 32> registers{};
+ std::array<std::array<bool, 7>, 32> predicates{};
+ u32 sum = 0;
+ for (u32 lane = 0; lane < 32; ++lane) {
+ registers[lane][3] = 0xfffffff0U + lane * 17;
+ registers[lane][221] = 0xcafebeef;
+ if (active & (1U << lane)) sum += registers[lane][3];
+ for (u32 p = 0; p < 7; ++p) predicates[lane][p] = ((lane + p) & 1) != 0;
+ }
+ const auto original = predicates;
+ ExecuteNativeWarp(block, active, registers, predicates);
+ REQUIRE(predicates == original);
+ for (u32 lane = 0; lane < 32; ++lane)
+ REQUIRE(registers[lane][221] == ((active & (1U << lane)) ? sum : 0xcafebeef));
+ }
+}
+
+TEST_CASE("Native histogram pointer carry and global reduction retain the address and addend", "[native-shader]") {
+ const std::array<Native, 4> input{{
+ {{0x00007a10, 0x00001000, 0x07f1e0ff, 0}},
+ {{0xff017624, 0x00001100, 0x000e06ff, 0}},
+ {{0x0000798e, 3, 0x0010e100, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (u32 low : {0U, 0xfffffff0U, 0xffffffffU}) {
+ ScalarNativeMachine machine;
+ machine.regs[0] = 32;
+ machine.regs[3] = 17;
+ machine.constants[0x40] = low;
+ machine.constants[0x44] = 0x21;
+ const u64 address = 0x2100000000ULL + low + 32;
+ machine.global[address] = 0xfffffff8U;
+ machine.Execute(block);
+ REQUIRE(machine.regs[0] == static_cast<u32>(address));
+ REQUIRE(machine.regs[1] == address >> 32);
+ REQUIRE(machine.predicates[0] == (low > 0xffffffdfU));
+ REQUIRE(machine.global == std::unordered_map<u64, u32>{{address, 9}});
+ }
+}
+
+TEST_CASE("Native histogram shared increment scales its address and increments once", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x03ff7f8c, 0, 0x0d00403f, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ ScalarNativeMachine machine;
+ machine.regs[3] = 63;
+ machine.shared[252] = 0xffffffffU;
+ machine.Execute(block);
+ REQUIRE(machine.shared == std::unordered_map<u64, u32>{{252, 0}});
+}
+
+TEST_CASE("Native histogram uniform leading bit and comparison select the highest active lane", "[native-shader]") {
+ const std::array<Native, 3> input{{{{0x000472bd, 4, 0x080e0000, 0}},
+ {{0x02007c0c, 4, 0x0bf02070, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (u32 mask : {1U, 0x80000000U, 0x55555555U, 0xffffffffU}) {
+ for (u32 lane = 0; lane < 32; ++lane) {
+ ScalarNativeMachine machine;
+ machine.regs[220] = mask;
+ machine.regs[2] = lane;
+ machine.Execute(block);
+ REQUIRE(machine.regs[220] == 31U - std::countl_zero(mask));
+ REQUIRE(machine.predicates[0] == (lane == 31U - std::countl_zero(mask)));
+ }
+ }
+}
+
+TEST_CASE("Native formatted surface loads retain red data coordinates descriptor and guard", "[native-shader]") {
+ const Native instruction = GENERATE(Native{{0x400a0b97, 0x60001200, 0x001e0100, 0}},
+ Native{{0x12250b97, 0x60001200, 0x001e0100, 0}});
+ const std::array<Native, 2> input{{instruction, VertexTransform.back()}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ REQUIRE(lowered.required_texture_buffer == 0);
+ const auto guard = Shader::Maxwell::Instruction{lowered.instructions.front()}.Pred();
+ REQUIRE(guard.index == 0);
+ REQUIRE_FALSE(guard.negated);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ u32 loads{}, writes{};
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::BoundImageRead) {
+ REQUIRE(inst.Arg(0).U32() == 0x12 * 4);
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::Color2D);
+ const auto* coords = inst.Arg(1).Inst();
+ REQUIRE(static_cast<u32>(coords->Arg(0).Inst()->Arg(0).Reg()) == instruction.Bits(24, 8));
+ REQUIRE(static_cast<u32>(coords->Arg(1).Inst()->Arg(0).Reg()) == instruction.Bits(24, 8) + 1);
+ ++loads;
+ }
+ if (inst.GetOpcode() == IR::Opcode::SetRegister) {
+ REQUIRE(static_cast<u32>(inst.Arg(0).Reg()) == instruction.Bits(16, 8));
+ const auto* red = inst.Arg(1).Inst();
+ REQUIRE(red->GetOpcode() == IR::Opcode::CompositeExtractU32x4);
+ REQUIRE(red->Arg(1).U32() == 0);
+ REQUIRE(red->Arg(0).Inst()->GetOpcode() == IR::Opcode::BoundImageRead);
+ ++writes;
+ }
+ }
+ REQUIRE(loads == 1);
+ REQUIRE(writes == 1);
+ auto invalid = input;
+ invalid[0].words[2] ^= 0x200; // Reject other channel masks until implemented.
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(invalid));
+}
+
+TEST_CASE("Native volume texel fetch preserves XYZ LOD descriptor and split RGBA outputs", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x00087b66, 0x50000a02, 0x049e0f0a, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ u32 fetches{};
+ std::vector<u32> writes;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::BoundImageFetch) {
+ REQUIRE(inst.Arg(0).U32() == 40);
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::Color3D);
+ const auto* coords = inst.Arg(1).Inst();
+ for (u32 i = 0; i < 3; ++i)
+ REQUIRE(static_cast<u32>(coords->Arg(i).Inst()->Arg(0).Reg()) == i);
+ REQUIRE(inst.Arg(3).U32() == 0);
+ ++fetches;
+ }
+ if (inst.GetOpcode() == IR::Opcode::SetRegister) {
+ writes.push_back(static_cast<u32>(inst.Arg(0).Reg()));
+ const auto* extract = inst.Arg(1).Inst()->Arg(0).Inst();
+ REQUIRE(extract->GetOpcode() == IR::Opcode::CompositeExtractF32x4);
+ REQUIRE(extract->Arg(1).U32() == writes.size() - 1);
+ }
+ }
+ REQUIRE(fetches == 1);
+ REQUIRE(writes == std::vector<u32>{8, 9, 10, 11});
+ auto invalid = input;
+ invalid[0].words[0] |= 1U << 24;
+ REQUIRE_FALSE(Shader::Ampere::LowerToMaxwell(invalid));
+}
+
+TEST_CASE("Native volume surface store retains XYZ and all four data channels", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x00007b9b, 0xa0000c04, 0x00100f00, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ u32 stores{};
+ for (const auto& inst : block.Instructions()) if (inst.GetOpcode() == IR::Opcode::BoundImageWrite) {
+ REQUIRE(inst.Arg(0).U32() == 48);
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::Color3D);
+ for (u32 i = 0; i < 3; ++i)
+ REQUIRE(static_cast<u32>(inst.Arg(1).Inst()->Arg(i).Inst()->Arg(0).Reg()) == i);
+ for (u32 i = 0; i < 4; ++i)
+ REQUIRE(static_cast<u32>(inst.Arg(2).Inst()->Arg(i).Inst()->Arg(0).Reg()) == 4 + i);
+ ++stores;
+ }
+ REQUIRE(stores == 1);
+}
+
+TEST_CASE("Native half FMA ReLU preserves fusion values above one and canonical NaNs", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x0d127231, 0x3000000d, 0x03848d10, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ const std::array<std::array<u16, 3>, 9> cases{{
+ {{0x3c00, 0x4400, 0x4200}}, // -1*1+4 = 3, not saturated to one
+ {{0x4000, 0x3c00, 0x0000}}, // -2*2+1 < 0
+ {{0x3c00, 0x3c00, 0x0000}},
+ {{0x0000, 0x0001, 0x0001}}, // smallest positive subnormal survives
+ {{0x7c00, 0x0000, 0x0000}},
+ {{0x0000, 0x7c00, 0x7c00}},
+ {{0x7e01, 0x3c00, 0x7fff}},
+ {{0xfe01, 0x3c00, 0x7fff}},
+ {{0x8000, 0x8000, 0x0000}},
+ }};
+ for (const auto& values : cases) {
+ ScalarNativeMachine machine;
+ machine.regs[13] = (u32{values[0]} << 16) | 0x4400;
+ machine.regs[16] = 0xc0000000 | values[1];
+ machine.predicates = {true, false, true, false, true, false, true};
+ const auto predicates = machine.predicates;
+ machine.Execute(block);
+ REQUIRE(machine.regs[18] == (u32{values[2]} | (u32{values[2]} << 16)));
+ REQUIRE(machine.predicates == predicates);
+ }
+ // A separate register operand permits a fused product residual that would
+ // vanish if the multiplication rounded before adding its third operand.
+ auto fused = input;
+ fused[0].words[1] = 0x3000000e;
+ IR::Block fused_block{pool};
+ TranslateScalarNative(fused, fused_block);
+ ScalarNativeMachine machine;
+ machine.regs[13] = 0x3c013c01;
+ machine.regs[14] = 0x3bfe3bfe;
+ machine.regs[16] = 0x3c003c00;
+ machine.Execute(fused_block);
+ REQUIRE(machine.regs[18] == 0x00100010); // 2^-20 in each half
+}
+
+TEST_CASE("Native half FMA ReLU accepts packed immediates without losing input aliases", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x02027831, 0x3c003c00, 0x03968805, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (const auto [source, addend, expected] : std::array<std::array<u16, 3>, 4>{{
+ {{0x4000, 0x3c00, 0x3c00}}, {{0x3c00, 0x4400, 0x0000}},
+ {{0x4400, 0x3c00, 0x4200}}, {{0x7e01, 0x3c00, 0x7fff}},
+ }}) {
+ ScalarNativeMachine machine;
+ machine.regs[2] = source | 0xc4000000U;
+ machine.regs[5] = (u32{addend} << 16) | 0x4800;
+ machine.Execute(block);
+ REQUIRE(machine.regs[2] == (u32{expected} | (u32{expected} << 16)));
+ }
+}
+
+TEST_CASE("Native quad butterfly and swizzled add retain horizontal and vertical differences", "[native-shader]") {
+ const bool vertical = GENERATE(false, true);
+ const std::array<Native, 3> input{{
+ {{0x05107f89, vertical ? 0x0c5c0300U : 0x0c3c0300U, 0x000e0000, 0}},
+ {{0x100c7822, vertical ? 0x5aU : 0x66U, 5, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ std::array<std::array<u32, 256>, 32> registers{};
+ std::array<std::array<bool, 7>, 32> predicates{};
+ for (u32 lane = 0; lane < 32; ++lane)
+ registers[lane][5] = std::bit_cast<u32>(static_cast<float>(lane / 4 * 100 + (lane & 1) * 3 + ((lane >> 1) & 1) * 11));
+ ExecuteNativeWarp(block, 0xffffffffU, registers, predicates);
+ for (u32 lane = 0; lane < 32; ++lane) {
+ REQUIRE(registers[lane][16] == registers[lane ^ (vertical ? 2 : 1)][5]);
+ REQUIRE(std::bit_cast<float>(registers[lane][12]) == (vertical ? 11.0f : 3.0f));
+ }
+}
+
+TEST_CASE("Native array gradient fetch retains array index XY derivatives descriptor and RGBA", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x04047b6c, 0xa080080c, 0x001e0f06, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ u32 samples{};
+ std::vector<u32> writes;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::BoundImageGradient) {
+ REQUIRE(inst.Arg(0).U32() == 32);
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::ColorArray2D);
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().num_derivatives.Value() == 2);
+ const auto* coords = inst.Arg(1).Inst();
+ REQUIRE(coords->Arg(0).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{4});
+ REQUIRE(coords->Arg(1).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{5});
+ const auto* layer = coords->Arg(2).Inst()->Arg(0).Inst();
+ REQUIRE(layer->GetOpcode() == IR::Opcode::BitFieldUExtract);
+ REQUIRE(layer->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{6});
+ REQUIRE(layer->Arg(2).U32() == 16);
+ for (u32 component = 0; component < 4; ++component)
+ REQUIRE(inst.Arg(2).Inst()->Arg(component).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{12 + component});
+ ++samples;
+ }
+ if (inst.GetOpcode() == IR::Opcode::SetRegister) writes.push_back(static_cast<u32>(inst.Arg(0).Reg()));
+ }
+ REQUIRE(samples == 1);
+ REQUIRE(writes == std::vector<u32>{4, 5, 6, 7});
+}
+
+TEST_CASE("Native capture distinguishes forward branches from call targets beyond main EXIT", "[native-shader]") {
+ using Capture = Shader::Ampere::ProgramCaptureState;
+ Capture calls;
+ REQUIRE(calls.Observe(Native{{0x7944, 48, 0x03c00000, 0}}, 1) == Capture::Result::Continue);
+ REQUIRE(calls.Observe(VertexTransform.back(), 2) == Capture::Result::Continue);
+ REQUIRE(calls.Observe(Native{{0x08007950, 0, 0, 0}}, 6) == Capture::Result::Complete);
+ Capture branches;
+ REQUIRE(branches.Observe(Native{{0x7947, 48, 0x03800000, 0}}, 0) == Capture::Result::Continue);
+ REQUIRE(branches.Observe(VertexTransform.back(), 1) == Capture::Result::Continue);
+ REQUIRE(branches.Observe(VertexTransform.back(), 4) == Capture::Result::Complete);
+ Capture multiple;
+ REQUIRE(multiple.Observe(Native{{0x7944, 96, 0x03c00000, 0}}, 1) == Capture::Result::Continue);
+ REQUIRE(multiple.Observe(Native{{0x7944, 128, 0x03c00000, 0}}, 3) == Capture::Result::Continue);
+ REQUIRE(multiple.Observe(VertexTransform.back(), 4) == Capture::Result::Continue);
+ REQUIRE(multiple.Observe(Native{{0x08007950, 0, 0, 0}}, 9) == Capture::Result::Continue);
+ REQUIRE(multiple.Observe(Native{{0x08007950, 0, 0, 0}}, 13) == Capture::Result::Complete);
+ Capture loop_exit;
+ REQUIRE(loop_exit.Observe(Native{{0x0944, 16, 0x03c00000, 0}}, 0) == Capture::Result::Continue);
+ REQUIRE(loop_exit.Observe(Native{{0x7947, 0xfffffff0, 0x0383ffff, 0}}, 1) == Capture::Result::Continue);
+ REQUIRE(loop_exit.Observe(Native{{0x7941, 0, 0x03800000, 0}}, 2) == Capture::Result::Continue);
+ REQUIRE(loop_exit.Observe(VertexTransform.back(), 3) == Capture::Result::Complete);
+ Capture invalid;
+ REQUIRE(invalid.Observe(Native{{0x7947, 1, 0x03800000, 0}}, 0) == Capture::Result::InvalidTarget);
+}
+
+TEST_CASE("Native implicit volume sample preserves XYZ and all four split channels", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x0a0a7b60, 0x5080400e, 0x041e0f0c, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ u32 samples{};
+ std::vector<std::pair<u32, u32>> copies;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::BoundImageSampleImplicitLod) {
+ REQUIRE(inst.Arg(0).U32() == 256);
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::Color3D);
+ for (u32 i = 0; i < 3; ++i)
+ REQUIRE(inst.Arg(1).Inst()->Arg(i).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{250 + i});
+ ++samples;
+ }
+ if (inst.GetOpcode() == IR::Opcode::SetRegister && inst.Arg(1).Inst()->GetOpcode() == IR::Opcode::GetRegister)
+ copies.emplace_back(static_cast<u32>(inst.Arg(0).Reg()), static_cast<u32>(inst.Arg(1).Inst()->Arg(0).Reg()));
+ }
+ REQUIRE(samples == 1);
+ REQUIRE(copies == std::vector<std::pair<u32, u32>>{{250, 10}, {251, 11}, {252, 14}, {10, 250}, {11, 251}, {12, 252}, {13, 253}});
+}
+
+TEST_CASE("Native array shadow sampling retains layer XY depth reference and scalar destination", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x082e7b60, 0xb080140a, 0x001e41ff, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ u32 samples{};
+ std::vector<std::pair<u32, u32>> copies;
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::BoundImageSampleDrefImplicitLod) {
+ REQUIRE(inst.Arg(0).U32() == 80);
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().type.Value() == Shader::TextureType::ColorArray2D);
+ REQUIRE(inst.Flags<IR::TextureInstInfo>().is_depth.Value() == 1);
+ REQUIRE(inst.Arg(1).Inst()->Arg(0).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{251});
+ REQUIRE(inst.Arg(1).Inst()->Arg(1).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{252});
+ REQUIRE(inst.Arg(2).Inst()->Arg(0).Inst()->Arg(0).Reg() == IR::Reg{11});
+ ++samples;
+ }
+ if (inst.GetOpcode() == IR::Opcode::SetRegister && inst.Arg(1).Inst()->GetOpcode() == IR::Opcode::GetRegister)
+ copies.emplace_back(static_cast<u32>(inst.Arg(0).Reg()), static_cast<u32>(inst.Arg(1).Inst()->Arg(0).Reg()));
+ }
+ REQUIRE(samples == 1);
+ REQUIRE(copies == std::vector<std::pair<u32, u32>>{{250, 8}, {251, 9}, {252, 10}});
+ REQUIRE(block.Instructions().back().Arg(0).Reg() == IR::Reg{46});
+}
+
+TEST_CASE("Native immediate uniform comparison and select preserve predicates at unsigned boundaries", "[native-shader]") {
+ const std::array<Native, 3> input{{{{0x0400788c, 6, 0x0bf01070, 0}}, {{0x04057887, 6, 0x08000000, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (u32 value : {0U, 5U, 6U, 7U, 0x80000000U, 0xffffffffU}) {
+ ScalarNativeMachine machine;
+ machine.regs[220] = value;
+ for (u32 i = 0; i < 7; ++i) machine.predicates[i] = (i & 1) != 0;
+ const auto saved = machine.predicates;
+ machine.Execute(block);
+ REQUIRE(((machine.regs[239] >> 6) & 1) == (value < 6));
+ REQUIRE(machine.regs[221] == std::min(value, 6U));
+ REQUIRE(machine.predicates == saved);
+ }
+}
+
+TEST_CASE("Native uniform signed right shift sign extends the high word", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x3f057899, 31, 0x08011404, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (u32 value : {0U, 1U, 0x7fffffffU, 0x80000000U, 0xffffffffU}) {
+ ScalarNativeMachine machine;
+ machine.regs[220] = value;
+ machine.Execute(block);
+ REQUIRE(machine.regs[221] == ((value & 0x80000000U) ? 0xffffffffU : 0U));
+ }
+}
+
+TEST_CASE("Native indexed constant load adds a signed byte displacement", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x30077b82, 0x00fe0500, 0x00000800, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ ScalarNativeMachine machine;
+ machine.regs[48] = 0x800;
+ machine.constants[(u64{3} << 32) | 0x14] = 0x12345678;
+ machine.Execute(block);
+ REQUIRE(machine.regs[7] == 0x12345678);
+}
+
+TEST_CASE("Native local spill retains byte address data and execution predicate", "[native-shader]") {
+ const std::array<Native, 3> input{{{{0xff001387, 85, 0x00100800, 0}},
+ {{0xff2a7983, 0, 0x00300800, 0}}, VertexTransform.back()}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ REQUIRE(Shader::Maxwell::Decode(lowered.instructions[0]) == Shader::Maxwell::Opcode::STL);
+ REQUIRE(Shader::Maxwell::Instruction{lowered.instructions[0]}.Pred().index == 1);
+ REQUIRE((lowered.instructions[0] & 255) == 85);
+ REQUIRE((lowered.instructions[1] & 255) == 42);
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ class LocalEnvironment : public EmptyEnvironment {
+ public:
+ u32 LocalMemorySize() const override { return 4; }
+ } env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.STL(lowered.instructions[0]);
+ visitor.LDL(lowered.instructions[1]);
+ u32 loads{}, stores{};
+ for (const auto& inst : block.Instructions()) {
+ if (inst.GetOpcode() == IR::Opcode::LoadLocal) {
+ REQUIRE(inst.Arg(0).U32() == 0);
+ ++loads;
+ }
+ if (inst.GetOpcode() == IR::Opcode::WriteLocal) {
+ REQUIRE(inst.Arg(0).U32() == 0);
+ REQUIRE(inst.Arg(1).Inst()->Arg(0).Reg() == IR::Reg{85});
+ ++stores;
+ }
+ }
+ REQUIRE(loads == 1);
+ REQUIRE(stores == 1);
+}
+
+TEST_CASE("Native scalar half conversion applies absolute value before rounding", "[native-shader]") {
+ const bool negate = GENERATE(false, true);
+ const std::array<Native, 2> input{{{{0x00047304, negate ? 0x80000000U : 0x40000000U, 0x00200800, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (const auto [value, expected] : std::array<std::pair<float, u32>, 7>{{
+ {-1.0f, 0x3c00}, {-0.0f, 0}, {-2.5f, 0x4100},
+ {1.00048828125f, 0x3c00}, {1.00146484375f, 0x3c02},
+ {-0x1p-24f, 1}, {-std::numeric_limits<float>::infinity(), 0x7c00},
+ }}) {
+ ScalarNativeMachine machine;
+ machine.regs[0] = std::bit_cast<u32>(value);
+ machine.Execute(block);
+ REQUIRE(machine.regs[4] == (expected | (negate && !std::signbit(value) ? 0x8000U : 0U)));
+ }
+}
+
+TEST_CASE("Native predicated half ReLU guards its complete expansion including predicate scratch", "[native-shader]") {
+ for (u32 pred : {1U, 6U, 9U, 14U}) {
+ const std::array<Native, 2> input{{{{0x12140231U | (pred << 12), 0x20000012, 0x03848914, 0}}, VertexTransform.back()}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ REQUIRE(Shader::Maxwell::Decode(lowered.instructions.front()) == Shader::Maxwell::Opcode::BRA);
+ REQUIRE(((lowered.instructions.front() >> 16) & 15) == (pred ^ 8));
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ EmptyEnvironment env;
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ for (size_t i = 1; i + 1 < lowered.instructions.size(); ++i) {
+ const u64 word = lowered.instructions[i];
+ REQUIRE(((word >> 16) & 15) == 7);
+ switch (Shader::Maxwell::Decode(word)) {
+#define INST(name, cute, encode) case Shader::Maxwell::Opcode::name: visitor.name(word); break;
+#include "shader_recompiler/frontend/maxwell/maxwell.inc"
+#undef INST
+ }
+ }
+ for (bool value : {false, true}) {
+ ScalarNativeMachine machine;
+ machine.regs[18] = 0x3c003c00;
+ machine.regs[20] = 0x44004400;
+ machine.predicates.fill(true);
+ machine.predicates[pred & 7] = value;
+ const auto saved = machine.predicates;
+ if (value != ((pred & 8) != 0)) machine.Execute(block);
+ REQUIRE(machine.regs[20] == (value != ((pred & 8) != 0) ? 0x42004200U : 0x44004400U));
+ REQUIRE(machine.predicates == saved);
+ }
+ }
+}
+
+TEST_CASE("Native unsigned half conversion truncates and saturates NTZ edge cases", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x000e7305, 13, 0x0010f000, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (const auto [value, expected] : std::array<std::pair<u32, u32>, 8>{{
+ {0x3e00, 1}, {0x4100, 2}, {0xbe00, 0}, {0x0001, 0},
+ {0x7bff, 65504}, {0x7c00, 0xffffffffU}, {0xfc00, 0}, {0x7e00, 0},
+ }}) {
+ ScalarNativeMachine machine;
+ machine.regs[13] = 0xdead0000 | value;
+ machine.Execute(block);
+ REQUIRE(machine.regs[14] == expected);
+ }
+}
+
+TEST_CASE("Native predicate LUT OR combines three inputs while preserving other predicates", "[native-shader]") {
+ const std::array<Native, 3> input{{{{0x0000781c, 0, 0x01f25f16, 0}}, {{0x0000781c, 0, 0x00703570, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ for (u32 bits = 0; bits < 128; ++bits) {
+ ScalarNativeMachine machine;
+ for (u32 i = 0; i < 7; ++i) machine.predicates[i] = (bits >> i) & 1;
+ auto expected = machine.predicates;
+ expected[1] = expected[1] || expected[2] || expected[3];
+ expected[0] = expected[0] || expected[1];
+ machine.Execute(block);
+ REQUIRE(machine.predicates == expected);
+ }
+}
+
+TEST_CASE("Native half FMA absolute source clears each half sign before broadcast", "[native-shader]") {
+ for (bool broadcast : {false, true}) {
+ const std::array<Native, 2> input{{{{0x1a087231, 0x20000008,
+ broadcast ? 0x00040a0dU : 0x0004020dU, 0}}, VertexTransform.back()}};
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ TranslateScalarNative(input, block);
+ ScalarNativeMachine machine;
+ machine.regs[26] = 0xc400c000;
+ machine.regs[8] = 0x40003c00;
+ machine.regs[13] = 0x44004200;
+ machine.Execute(block);
+ REQUIRE(machine.regs[8] == (broadcast ? 0x45004500U : 0x47004500U));
+ }
+}
+
+TEST_CASE("Native constant interpolation retains attribute and flat input behavior", "[native-shader]") {
+ const std::array<Native, 2> input{{{{0x00047326, 255, 0x000e403c, 0}}, VertexTransform.back()}};
+ const auto lowered = Shader::Ampere::LowerToMaxwell(input);
+ REQUIRE(lowered);
+ REQUIRE(((lowered.instructions[0] >> 54) & 3) == 2);
+ class FlatEnvironment : public EmptyEnvironment {
+ public:
+ FlatEnvironment() { sph.ps.imap_generic_vector[7].raw = 0x55; }
+ } env;
+ Shader::ObjectPool<IR::Inst> pool;
+ IR::Block block{pool};
+ Shader::Maxwell::TranslatorVisitor visitor{env, block};
+ visitor.IPA(lowered.instructions[0]);
+ u32 reads{};
+ for (const auto& inst : block.Instructions()) {
+ REQUIRE(inst.GetOpcode() != IR::Opcode::FPMul32);
+ if (inst.GetOpcode() == IR::Opcode::GetAttribute) {
+ REQUIRE(inst.Arg(0).Attribute() == IR::Attribute{60});
+ ++reads;
+ }
+ }
+ REQUIRE(reads == 1);
+}
diff --git a/src/tests/video_core/native_swizzle.cpp b/src/tests/video_core/native_swizzle.cpp
new file mode 100644
index 0000000000..93c4b85dac
--- /dev/null
+++ b/src/tests/video_core/native_swizzle.cpp
@@ -0,0 +1,67 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <vector>
+#include <catch2/catch_test_macros.hpp>
+#include "video_core/textures/decoders.h"
+
+namespace {
+u64 Digest(const std::vector<u8>& bytes) {
+ u64 digest = 14695981039346656037ULL;
+ for (const u8 byte : bytes) digest = (digest ^ byte) * 1099511628211ULL;
+ return digest;
+}
+}
+
+TEST_CASE("Native Orin texture mips match independent upstream swizzle vectors", "[native-swizzle]") {
+ struct Fixture { u32 width, height, levels, bytes; std::array<u64, 4> hashes; };
+ const std::array fixtures{
+ Fixture{32, 64, 2, 8, {0x9fcd630e81008de4ULL, 0x0b316aa26800970aULL}},
+ Fixture{128, 256, 4, 16, {0x1d16743f55444bd6ULL, 0x628ec170d3dc24b6ULL, 0x9d5ce8671743eadcULL, 0x68c86243a7167475ULL}},
+ Fixture{128, 128, 3, 8, {0xa3694bb8d6333e7eULL, 0xe748d5f21bd06d5dULL, 0xabfa06266de4d322ULL}},
+ Fixture{256, 64, 2, 16, {0x9820bfdf5000d898ULL, 0x2177c49e446a9450ULL}},
+ Fixture{64, 256, 4, 16, {0x30e15cf8e750e118ULL, 0x286e806b76ac89e4ULL, 0xa0dc28aebfbe0862ULL, 0xdca4a8b9b78cafc2ULL}},
+ };
+ for (const auto& fixture : fixtures) {
+ u32 state = 0x12345678;
+ for (u32 level = 0; level < fixture.levels; ++level) {
+ const u32 width = (std::max(1U, fixture.width >> level) + 3) / 4;
+ const u32 height = (std::max(1U, fixture.height >> level) + 3) / 4;
+ u32 block_height = 16;
+ while (block_height > 1 && height <= block_height * 4) block_height >>= 1;
+ const u32 block_log2 = static_cast<u32>(std::countr_zero(block_height));
+ const size_t size = Tegra::Texture::CalculateSize(true, fixture.bytes, width, height, 1, block_log2, 0);
+ std::vector<u8> tiled(size);
+ for (auto& byte : tiled) { state = state * 1664525U + 1013904223U; byte = static_cast<u8>(state >> 24); }
+ std::vector<u8> linear(width * height * fixture.bytes);
+ Tegra::Texture::UnswizzleTexture(linear, tiled, fixture.bytes, width, height, 1, block_log2, 0, 1, true);
+ INFO("width=" << width << " height=" << height << " mip=" << level);
+ REQUIRE(Digest(linear) == fixture.hashes[level]);
+ auto restored = tiled;
+ Tegra::Texture::SwizzleTexture(restored, linear, fixture.bytes, width, height, 1, block_log2, 0, 1, true);
+ REQUIRE(restored == tiled);
+ }
+ }
+}
+
+TEST_CASE("Native and legacy GOB byte lanes remain distinct", "[native-swizzle]") {
+ std::vector<u8> tiled(512);
+ for (u32 i = 0; i < tiled.size(); ++i) tiled[i] = static_cast<u8>(i / 16);
+ std::vector<u8> native(512), legacy(512);
+ Tegra::Texture::UnswizzleTexture(native, tiled, 1, 64, 8, 1, 0, 0, 1, true);
+ Tegra::Texture::UnswizzleTexture(legacy, tiled, 1, 64, 8, 1, 0, 0);
+ REQUIRE(native[16] == 4);
+ REQUIRE(legacy[16] == 2);
+ REQUIRE(native[2 * 64] == 2);
+ REQUIRE(legacy[2 * 64] == 4);
+ REQUIRE(native != legacy);
+ std::vector<u8> crop(16 * 3);
+ Tegra::Texture::UnswizzleSubrect(crop, tiled, 1, 64, 8, 1, 16, 2, 16, 3, 0, 0, 16, true);
+ for (u32 y = 0; y < 3; ++y) {
+ REQUIRE(std::equal(crop.begin() + y * 16, crop.begin() + (y + 1) * 16,
+ native.begin() + (y + 2) * 64 + 16));
+ }
+}
diff --git a/src/tests/video_core/pending_download.cpp b/src/tests/video_core/pending_download.cpp
new file mode 100644
index 0000000000..4b2a218933
--- /dev/null
+++ b/src/tests/video_core/pending_download.cpp
@@ -0,0 +1,112 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <deque>
+#include <memory>
+#include <vector>
+#include <catch2/catch_test_macros.hpp>
+#include "common/slot_vector.h"
+#include "video_core/texture_cache/pending_download.h"
+
+namespace {
+struct DownloadImage {
+ DownloadImage(u32 value, std::shared_ptr<std::vector<u32>> destroyed_)
+ : handle{std::make_unique<u32>(value)}, destroyed{std::move(destroyed_)} {}
+ DownloadImage(DownloadImage&&) noexcept = default;
+ DownloadImage& operator=(DownloadImage&&) noexcept = default;
+ ~DownloadImage() { if (handle) destroyed->push_back(*handle); }
+ std::unique_ptr<u32> handle;
+ std::shared_ptr<std::vector<u32>> destroyed;
+};
+using Download = VideoCommon::PendingImageDownload<DownloadImage>;
+}
+
+TEST_CASE("Queued texture downloads survive deletion and reuse of their cache slot", "[texture-download]") {
+ auto destroyed = std::make_shared<std::vector<u32>>();
+ Common::SlotVector<DownloadImage> images;
+ const auto old_id = images.insert(42U, destroyed);
+ std::vector<Download> pending{{true, 0, old_id, {}}, {false, 0, old_id, {}}};
+ std::deque<std::vector<Download>> submitted{{{true, 0, old_id, {}}}};
+ REQUIRE(VideoCommon::RetainPendingImageDownloads(old_id, images[old_id], pending, submitted));
+ REQUIRE_FALSE(images[old_id].handle);
+ REQUIRE(pending[0].retired_image == submitted[0][0].retired_image);
+ REQUIRE_FALSE(pending[1].retired_image); // Buffer-download slot IDs are a separate namespace.
+ images.erase(old_id);
+ const auto replacement_id = images.insert(99U, destroyed);
+ REQUIRE(replacement_id == old_id);
+ REQUIRE(*pending[0].GetImage(images).handle == 42);
+ REQUIRE(*submitted[0][0].GetImage(images).handle == 42);
+ REQUIRE(*images[replacement_id].handle == 99);
+ // Retiring the replacement must not redirect an earlier generation's downloads.
+ pending.push_back({true, 1, replacement_id, {}});
+ REQUIRE(VideoCommon::RetainPendingImageDownloads(replacement_id, images[replacement_id], pending, submitted));
+ images.erase(replacement_id);
+ REQUIRE(*pending[0].GetImage(images).handle == 42);
+ REQUIRE(*pending[2].GetImage(images).handle == 99);
+ REQUIRE(destroyed->empty());
+ pending.clear();
+ REQUIRE(*destroyed == std::vector<u32>{99});
+ REQUIRE(*submitted[0][0].GetImage(images).handle == 42);
+ submitted.pop_front();
+ REQUIRE(*destroyed == std::vector<u32>{99, 42});
+}
+
+TEST_CASE("Textures without queued downloads retain ordinary cache retirement", "[texture-download]") {
+ auto destroyed = std::make_shared<std::vector<u32>>();
+ Common::SlotVector<DownloadImage> images;
+ const auto id = images.insert(7U, destroyed);
+ std::vector<Download> pending{{false, 0, id, {}}};
+ std::deque<std::vector<Download>> submitted;
+ REQUIRE_FALSE(VideoCommon::RetainPendingImageDownloads(id, images[id], pending, submitted));
+ REQUIRE(*images[id].handle == 7);
+ images.erase(id);
+ REQUIRE(*destroyed == std::vector<u32>{7});
+}
+
+TEST_CASE("Repeated whole-image downloads keep first occurrence and buffer requests", "[texture-download]") {
+ auto destroyed = std::make_shared<std::vector<u32>>();
+ Common::SlotVector<DownloadImage> images;
+ const auto a = images.insert(42U, destroyed);
+ const auto b = images.insert(99U, destroyed);
+ std::vector<Download> pending{{true, 0, a, {}}, {true, 0, b, {}},
+ {false, 3, Common::SlotId{10000}, {}}, {true, 0, a, {}}};
+ const auto writeback = [&](const auto& downloads) {
+ u32 value = 0;
+ for (auto it = downloads.rbegin(); it != downloads.rend(); ++it)
+ if (it->is_swizzle) value = *it->GetImage(images).handle;
+ return value;
+ };
+ const u32 expected = writeback(pending);
+ REQUIRE(VideoCommon::DeduplicatePendingImageDownloads(pending, images) == 1);
+ REQUIRE(pending.size() == 3);
+ REQUIRE(pending[0].object_id == a);
+ REQUIRE(pending[1].object_id == b);
+ REQUIRE_FALSE(pending[2].is_swizzle);
+ REQUIRE(pending[2].async_buffer_id == 3);
+ REQUIRE(writeback(pending) == expected);
+ REQUIRE(expected == 42);
+}
+
+TEST_CASE("Readback deduplication distinguishes recycled image generations", "[texture-download]") {
+ auto destroyed = std::make_shared<std::vector<u32>>();
+ Common::SlotVector<DownloadImage> images;
+ const auto id = images.insert(42U, destroyed);
+ std::vector<Download> pending{{true, 0, id, {}}};
+ std::deque<std::vector<Download>> submitted;
+ REQUIRE(VideoCommon::RetainPendingImageDownloads(id, images[id], pending, submitted));
+ auto old = pending.front();
+ images.erase(id);
+ const auto replacement = images.insert(99U, destroyed);
+ REQUIRE(replacement == id);
+ pending.push_back({true, 0, replacement, {}});
+ for (size_t i = 0; i < 1000; ++i) pending.push_back(old);
+ REQUIRE(VideoCommon::DeduplicatePendingImageDownloads(pending, images) == 1000);
+ REQUIRE(pending.size() == 2);
+ REQUIRE(*pending[0].GetImage(images).handle == 42);
+ REQUIRE(*pending[1].GetImage(images).handle == 99);
+ REQUIRE(destroyed->empty());
+ pending.clear();
+ REQUIRE(destroyed->empty());
+ old.retired_image.reset();
+ REQUIRE(*destroyed == std::vector<u32>{42});
+}
diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt
index 64d14ab2a6..79028f0baf 100644
--- a/src/video_core/CMakeLists.txt
+++ b/src/video_core/CMakeLists.txt
@@ -93,6 +93,8 @@ add_library(video_core STATIC
fence_manager.h
gpu.cpp
gpu.h
+ native_query_trace.h
+ native_semaphore.h
gpu_thread.cpp
gpu_thread.h
guest_memory.h
@@ -242,6 +244,7 @@ add_library(video_core STATIC
texture_cache/image_base.h
texture_cache/image_info.cpp
texture_cache/image_info.h
+ texture_cache/pending_download.h
texture_cache/image_view_base.cpp
texture_cache/image_view_base.h
texture_cache/image_view_info.cpp
diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index f30c85911f..126e73c9b6 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -9,6 +9,7 @@
#include <algorithm>
#include <memory>
#include <numeric>
+#include <stdexcept>
#include "common/range_sets.inc"
#include "video_core/buffer_cache/buffer_cache_base.h"
@@ -1372,7 +1373,12 @@ void BufferCache<P>::UpdateVertexBuffer(u32 index) {
const auto& array = maxwell3d->regs.vertex_streams[index];
const auto& limit = maxwell3d->regs.vertex_stream_limits[index];
const GPUVAddr gpu_addr_begin = array.Address();
- const GPUVAddr gpu_addr_end = limit.Address() + 1;
+ const bool native = maxwell3d->GetNativeMacroEngine().native_turing;
+ const u64 native_size = native ? maxwell3d->regs.NativeVertexStreamSize(index) : 0;
+ if (native && array.enable != 0 && native_size > (std::numeric_limits<u32>::max)()) {
+ throw std::runtime_error("Native vertex stream exceeds supported 32-bit buffer size");
+ }
+ const GPUVAddr gpu_addr_end = native ? gpu_addr_begin + native_size : limit.Address() + 1;
const std::optional<DAddr> device_addr = gpu_memory->GpuToCpuAddress(gpu_addr_begin);
const u32 address_size = static_cast<u32>(gpu_addr_end - gpu_addr_begin);
u32 size = address_size; // TODO: Analyze stride and number of vertices
diff --git a/src/video_core/control/channel_state.cpp b/src/video_core/control/channel_state.cpp
index 12a452dfc0..57113ce68e 100644
--- a/src/video_core/control/channel_state.cpp
+++ b/src/video_core/control/channel_state.cpp
@@ -55,6 +55,7 @@ void ChannelState::Init(Core::System& system, u64 program_id_) {
ASSERT(memory_manager);
program_id = program_id_;
payload.emplace(system, *memory_manager, *this);
+ payload->maxwell_3d.SetNativeMacroContext(native_macro_context);
BindNvkDefaultSubchannels(*payload);
initialized = true;
}
diff --git a/src/video_core/control/channel_state.h b/src/video_core/control/channel_state.h
index 80811c1458..9f64f4bfe3 100644
--- a/src/video_core/control/channel_state.h
+++ b/src/video_core/control/channel_state.h
@@ -56,12 +56,15 @@ struct ChannelState {
Engines::Nv01Timer nv01_timer;
DmaPusher dma_pusher;
};
+ std::shared_ptr<NativeMacroContext> native_macro_context;
+ std::shared_ptr<MemoryManager> memory_manager_owner;
std::optional<Payload> payload;
MemoryManager* memory_manager = nullptr;
s32 bind_id = -1;
u64 program_id = 0;
bool initialized = false;
+ bool native_abi = false;
};
} // namespace Control
diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp
index 8af13e415c..5f8a830c83 100644
--- a/src/video_core/control/scheduler.cpp
+++ b/src/video_core/control/scheduler.cpp
@@ -4,7 +4,9 @@
// SPDX-FileCopyrightText: 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
+#include <algorithm>
#include <memory>
+#include <vector>
#include "common/assert.h"
#include "video_core/control/channel_state.h"
@@ -26,6 +28,26 @@ void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) {
// Multiple channels can prepare their commands in parallel.
channel_state->payload->dma_pusher.Push(std::move(entries));
channel_state->payload->dma_pusher.DispatchCalls();
+ if (channel_state->payload->dma_pusher.HasPendingNativeAcquire() &&
+ std::find(pending_native_channels.begin(), pending_native_channels.end(), channel_state) == pending_native_channels.end()) {
+ pending_native_channels.push_back(channel_state);
+ }
+ if (!pending_native_channels.empty()) ResumePending(gpu);
+}
+
+void Scheduler::ResumePending(GPU& gpu) {
+ ResumeReadyNativeChannels(pending_native_channels,
+ [](const auto& state) { return state->payload->dma_pusher.HasPendingNativeAcquire() &&
+ state->payload->dma_pusher.CanResumeNativeAcquire(); },
+ [&](const auto& state) {
+ {
+ std::scoped_lock lock{scheduling_guard};
+ gpu.BindChannel(state->bind_id);
+ }
+ state->payload->dma_pusher.DispatchCalls();
+ });
+ std::erase_if(pending_native_channels,
+ [](const auto& state) { return !state->payload->dma_pusher.HasPendingNativeAcquire(); });
}
void Scheduler::DeclareChannel(std::shared_ptr<ChannelState> new_channel) {
diff --git a/src/video_core/control/scheduler.h b/src/video_core/control/scheduler.h
index 342bea5025..b6e9544ca0 100644
--- a/src/video_core/control/scheduler.h
+++ b/src/video_core/control/scheduler.h
@@ -8,6 +8,7 @@
#include <memory>
#include <mutex>
+#include <vector>
#include "common/container/unordered_map.h"
#include "video_core/dma_pusher.h"
@@ -24,11 +25,15 @@ class Scheduler {
public:
void Push(GPU& gpu, s32 channel, CommandList&& entries);
+ void ResumePending(GPU& gpu);
+ [[nodiscard]] bool HasPendingNativeChannels() const { return !pending_native_channels.empty(); }
+
void DeclareChannel(std::shared_ptr<ChannelState> new_channel);
private:
::Common::unordered_map<s32, std::shared_ptr<ChannelState>> channels;
std::mutex scheduling_guard;
+ std::vector<std::shared_ptr<ChannelState>> pending_native_channels;
};
} // namespace Control
diff --git a/src/video_core/dirty_flags.cpp b/src/video_core/dirty_flags.cpp
index c2ecc12f56..e6856aaca3 100644
--- a/src/video_core/dirty_flags.cpp
+++ b/src/video_core/dirty_flags.cpp
@@ -22,6 +22,7 @@ void SetupDirtyVertexBuffers(Maxwell3D::DirtyState::Tables& tables) {
OFF(vertex_stream_limits) + i * NUM(vertex_stream_limits[0]);
FillBlock(tables, array_offset, num_array, VertexBuffer0 + i, VertexBuffers);
+ FillBlock(tables, 0x600 / sizeof(u32) + i * 2, 2, VertexBuffer0 + i, VertexBuffers);
FillBlock(tables, limit_offset, NUM(vertex_stream_limits), VertexBuffer0 + i,
VertexBuffers);
}
diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp
index 2c0ba007ef..3e9359ea3f 100644
--- a/src/video_core/dma_pusher.cpp
+++ b/src/video_core/dma_pusher.cpp
@@ -4,9 +4,11 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <cstdlib>
#include "common/settings.h"
#include "core/core.h"
#include "video_core/dma_pusher.h"
+#include "video_core/control/channel_state.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/gpu.h"
#include "video_core/guest_memory.h"
@@ -29,22 +31,38 @@ DmaPusher::DmaPusher(Core::System& system_, MemoryManager& memory_manager_, Cont
DmaPusher::~DmaPusher() = default;
void DmaPusher::DispatchCalls() {
- dma_pushbuffer_subindex = 0;
dma_state.is_last_call = true;
while (system.IsPoweredOn()) {
if (!Step()) {
break;
}
}
+ if (channel_state.native_abi && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) {
+ static u32 dispatch_traces{};
+ if (dispatch_traces++ < 512) {
+ LOG_INFO(HW_GPU, "Native dispatch end: channel={}, queued={}, failed={}, acquire={}, method={:#x}, remaining={}, resume={}",
+ channel_state.bind_id, dma_pushbuffer.size(), HasFailedNativeWork(),
+ HasPendingNativeAcquire(), dma_state.method, dma_state.method_count, command_resume_offset);
+ }
+ }
system.GPU().FlushCommands();
system.GPU().OnCommandListEnd();
}
+bool DmaPusher::HasFailedNativeWork() const {
+ const auto& context = channel_state.native_macro_context;
+ return channel_state.native_abi && context &&
+ (context->engine.native_execution_failed || context->compute_engine.native_execution_failed);
+}
+
bool DmaPusher::Step() {
+ if (HasFailedNativeWork()) return false;
if (!ib_enable || dma_pushbuffer.empty()) {
return false;
}
+ if (!puller.ResumeNativeAcquire(*this)) return false;
+
CommandList& command_list = dma_pushbuffer.front();
const size_t prefetch_size = command_list.prefetch_command_list.size();
@@ -57,7 +75,10 @@ bool DmaPusher::Step() {
}
if (prefetch_size > 0) {
- ProcessCommands(command_list.prefetch_command_list);
+ const std::span<const CommandHeader> commands{command_list.prefetch_command_list};
+ command_resume_offset += ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset);
+ if (puller.HasPendingNativeAcquire() || HasFailedNativeWork()) return false;
+ command_resume_offset = 0;
dma_pushbuffer.pop();
return true;
}
@@ -86,13 +107,17 @@ bool DmaPusher::Step() {
const bool use_safe = Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe();
if (use_safe) {
Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::SafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers);
- ProcessCommands(headers);
+ const std::span<const CommandHeader> commands{headers.data(), headers.size()};
+ command_resume_offset += ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset);
} else {
Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::UnsafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers);
- ProcessCommands(headers);
+ const std::span<const CommandHeader> commands{headers.data(), headers.size()};
+ command_resume_offset += ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset);
}
}
+ if (puller.HasPendingNativeAcquire() || HasFailedNativeWork()) return false;
+ command_resume_offset = 0;
if (++dma_pushbuffer_subindex >= command_list_size) {
dma_pushbuffer.pop();
dma_pushbuffer_subindex = 0;
@@ -111,20 +136,21 @@ bool DmaPusher::Step() {
return true;
}
-void DmaPusher::ProcessCommands(std::span<const CommandHeader> commands) {
+size_t DmaPusher::ProcessCommands(std::span<const CommandHeader> commands, size_t word_offset) {
for (size_t index = 0; index < commands.size();) {
// Data word of methods command
if (dma_state.method_count && dma_state.non_incrementing) {
auto const& command_header = commands[index]; //must ref (MUltiMethod re)
- dma_state.dma_word_offset = u32(index * sizeof(u32));
- const u32 max_write = u32(std::min<std::size_t>(index + dma_state.method_count, commands.size()) - index);
+ dma_state.dma_word_offset = u64((word_offset + index) * sizeof(u32));
+ u32 max_write = u32(std::min<std::size_t>(index + dma_state.method_count, commands.size()) - index);
+ if (channel_state.native_abi && dma_state.method < non_puller_methods) max_write = 1;
CallMultiMethod(&command_header.argument, max_write);
dma_state.method_count -= max_write;
dma_state.is_last_call = true;
index += max_write;
} else if (dma_state.method_count) {
auto const command_header = commands[index]; //can copy
- dma_state.dma_word_offset = u32(index * sizeof(u32));
+ dma_state.dma_word_offset = u64((word_offset + index) * sizeof(u32));
dma_state.is_last_call = dma_state.method_count <= 1;
CallMethod(command_header.argument);
dma_state.method += !dma_state.non_incrementing ? 1 : 0;
@@ -163,7 +189,9 @@ void DmaPusher::ProcessCommands(std::span<const CommandHeader> commands) {
}
index++;
}
+ if (puller.HasPendingNativeAcquire() || HasFailedNativeWork()) return index;
}
+ return commands.size();
}
void DmaPusher::SetState(const CommandHeader& command_header) {
diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h
index 62af628511..51c735baea 100644
--- a/src/video_core/dma_pusher.h
+++ b/src/video_core/dma_pusher.h
@@ -131,6 +131,8 @@ public:
}
void DispatchCalls();
+ [[nodiscard]] bool HasPendingNativeAcquire() const { return puller.HasPendingNativeAcquire(); }
+ [[nodiscard]] bool CanResumeNativeAcquire() { return !HasFailedNativeWork() && puller.CanResumeNativeAcquire(*this); }
void BindSubchannel(Engines::EngineInterface* engine, u32 subchannel_id, Engines::EngineTypes engine_type) {
subchannels[subchannel_id] = engine;
@@ -142,8 +144,10 @@ public:
private:
static constexpr u32 non_puller_methods = 0x40;
static constexpr u32 max_subchannels = 8;
+ bool HasFailedNativeWork() const;
bool Step();
- void ProcessCommands(std::span<const CommandHeader> commands);
+ size_t ProcessCommands(std::span<const CommandHeader> commands, size_t word_offset = 0);
+ size_t command_resume_offset{};
void SetState(const CommandHeader& command_header);
diff --git a/src/video_core/engines/draw_manager.cpp b/src/video_core/engines/draw_manager.cpp
index 5891c25622..a610fb1661 100644
--- a/src/video_core/engines/draw_manager.cpp
+++ b/src/video_core/engines/draw_manager.cpp
@@ -5,11 +5,15 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <cstring>
+#include <fstream>
+#include <cstdlib>
+#include <set>
#include "common/settings.h"
#include "video_core/dirty_flags.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/rasterizer_interface.h"
+#include "video_core/memory_manager.h"
namespace Tegra::Engines {
@@ -277,6 +281,45 @@ void Maxwell3D::DrawManager::ProcessDraw(Maxwell3D& maxwell3d, bool draw_indexed
LOG_TRACE(HW_GPU, "called, topology={}, count={}", draw_state.topology, draw_indexed ? draw_state.index_buffer.count : draw_state.vertex_buffer.count);
UpdateTopology(maxwell3d);
if (maxwell3d.ShouldExecute()) {
+ auto& native = maxwell3d.GetNativeMacroEngine();
+ if (native.native_turing) {
+ const auto count = draw_indexed ? draw_state.index_buffer.count : draw_state.vertex_buffer.count;
+ if (!count || !instance_count) return;
+ if (native.native_execution_failed) return;
+ if (Settings::values.renderer_backend.GetValue() != Settings::RendererBackend::Vulkan) {
+ native.native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Native graphics shaders require the experimental Vulkan path");
+ return;
+ }
+ if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) {
+ static thread_local std::set<std::array<GPUVAddr, Maxwell3D::Regs::MaxShaderProgram>> captured;
+ std::array<GPUVAddr, Maxwell3D::Regs::MaxShaderProgram> programs{};
+ for (size_t stage = 0; stage < programs.size(); ++stage) {
+ if (!maxwell3d.regs.IsShaderConfigEnabled(stage)) continue;
+ const auto& pipeline = maxwell3d.regs.pipelines[stage];
+ programs[stage] = (GPUVAddr{pipeline.reserved[0] & 0xff} << 32) | pipeline.reserved[1];
+ }
+ if (captured.size() < 4096 && captured.insert(programs).second) {
+ std::ofstream registers("native-graphics-registers.bin", std::ios::binary);
+ registers.write(reinterpret_cast<const char*>(&maxwell3d.regs), sizeof(maxwell3d.regs));
+ for (size_t stage = 0; stage < Maxwell3D::Regs::MaxShaderProgram; ++stage) {
+ if (!maxwell3d.regs.IsShaderConfigEnabled(stage)) continue;
+ const auto& pipeline = maxwell3d.regs.pipelines[stage];
+ // C797 uses per-stage program addresses at 0x2014/0x2018,
+ // unlike Maxwell's program-region plus 0x2004 offset.
+ const GPUVAddr address = (GPUVAddr{pipeline.reserved[0] & 0xff} << 32) | pipeline.reserved[1];
+ LOG_INFO(HW_GPU, "Native graphics shader stage={}, address={:#x}", stage, address);
+ if (!maxwell3d.memory_manager.IsFullyMappedRange(address, 0x1000)) continue;
+ std::array<u8, 0x1000> bytes{};
+ maxwell3d.memory_manager.ReadBlockUnsafe(address, bytes.data(), bytes.size());
+ std::ofstream shader("native-graphics-shader-" + std::to_string(stage) + ".bin", std::ios::binary);
+ shader.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+ }
+ LOG_DEBUG(HW_GPU, "Translating native graphics draw: indexed={}, vertices={}, instances={}",
+ draw_indexed, count, instance_count);
+ }
+ }
+ }
maxwell3d.rasterizer->Draw(draw_indexed, instance_count);
}
}
@@ -285,6 +328,12 @@ void Maxwell3D::DrawManager::ProcessDrawIndirect(Maxwell3D& maxwell3d) {
LOG_TRACE(HW_GPU, "called, topology={}, is_indexed={}, includes_count={}, buffer_size={}, max_draw_count={}", draw_state.topology, indirect_state.is_indexed, indirect_state.include_count, indirect_state.buffer_size, indirect_state.max_draw_counts);
UpdateTopology(maxwell3d);
if (maxwell3d.ShouldExecute()) {
+ auto& native = maxwell3d.GetNativeMacroEngine();
+ if (native.native_turing) {
+ native.native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Native graphics shader translation required: indirect draw");
+ return;
+ }
maxwell3d.rasterizer->DrawIndirect();
}
}
diff --git a/src/video_core/engines/engine_upload.cpp b/src/video_core/engines/engine_upload.cpp
index ba21eec4c4..0ffdae8036 100644
--- a/src/video_core/engines/engine_upload.cpp
+++ b/src/video_core/engines/engine_upload.cpp
@@ -27,6 +27,7 @@ void State::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) {
void State::ProcessExec(const bool is_linear_) {
write_offset = 0;
+ flushed_offset = 0;
copy_size = regs.line_length_in * regs.line_count;
inner_buffer.resize_destructive(copy_size);
is_linear = is_linear_;
@@ -39,10 +40,29 @@ void State::ProcessData(const u32 data, const bool is_last_call) {
if (!is_last_call) {
return;
}
+ if (native_partial && is_linear && regs.line_count == 1) {
+ const u32 count = write_offset - flushed_offset;
+ if (count) {
+ rasterizer->AccelerateInlineToMemory(regs.dest.Address() + flushed_offset, count,
+ std::span<const u8>{inner_buffer.data() + flushed_offset, count});
+ flushed_offset = write_offset;
+ }
+ return;
+ }
ProcessData(inner_buffer);
}
void State::ProcessData(const u32* data, size_t num_data) {
+ if (native_partial && is_linear && regs.line_count == 1) {
+ const auto count = std::min<size_t>(num_data * sizeof(u32), copy_size - write_offset);
+ if (count) {
+ rasterizer->AccelerateInlineToMemory(regs.dest.Address() + write_offset, count,
+ std::span<const u8>{reinterpret_cast<const u8*>(data), count});
+ write_offset += static_cast<u32>(count);
+ flushed_offset = write_offset;
+ }
+ return;
+ }
std::span<const u8> read_buffer(reinterpret_cast<const u8*>(data), num_data * sizeof(u32));
ProcessData(read_buffer);
}
diff --git a/src/video_core/engines/engine_upload.h b/src/video_core/engines/engine_upload.h
index 21bf8aeb46..55a82f938b 100644
--- a/src/video_core/engines/engine_upload.h
+++ b/src/video_core/engines/engine_upload.h
@@ -62,6 +62,7 @@ public:
explicit State(MemoryManager& memory_manager_, Registers& regs_);
~State();
+ void SetNativePartialTransfers(bool enabled) { native_partial = enabled; }
void ProcessExec(bool is_linear_);
void ProcessData(u32 data, bool is_last_call);
void ProcessData(const u32* data, size_t num_data);
@@ -80,6 +81,8 @@ public:
private:
void ProcessData(std::span<const u8> read_buffer);
+ bool native_partial{};
+ u32 flushed_offset{};
u32 write_offset = 0;
u32 copy_size = 0;
Common::ScratchBuffer<u8> inner_buffer;
diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp
index 96a12539a2..1de51fd7aa 100644
--- a/src/video_core/engines/kepler_compute.cpp
+++ b/src/video_core/engines/kepler_compute.cpp
@@ -5,12 +5,15 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <bitset>
+#include <fstream>
#include "common/assert.h"
#include "common/logging.h"
+#include "shader_recompiler/exception.h"
#include "core/core.h"
#include "video_core/engines/kepler_compute.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/memory_manager.h"
+#include "video_core/native_query_trace.h"
#include "video_core/rasterizer_interface.h"
#include "video_core/textures/decoders.h"
@@ -40,8 +43,173 @@ void KeplerCompute::ConsumeSinkImpl(Core::System& system) {
method_sink.clear();
}
+bool KeplerCompute::ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call) {
+ if (native_context) {
+ if (method == 0x45 || method == 0x47) native_compute_uploaded = true;
+ native_macro = native_compute_uploaded ? &native_context->compute_engine : &native_context->engine;
+ native_macro_positions = native_compute_uploaded ? std::span<u32>{native_context->compute_positions} : std::span<u32>{native_context->positions};
+ }
+ if (method >= native_registers.size()) {
+ LOG_ERROR(HW_GPU, "Native compute method out of range: {:#x}", method);
+ return true;
+ }
+ if (method >= 0xe00) {
+ if (!native_executing_macro) {
+ if (method & 1) { LOG_ERROR(HW_GPU, "Native compute macro starts on argument method"); return true; }
+ native_executing_macro = method;
+ }
+ native_parameters.push_back(argument);
+ if (!is_last_call) return true;
+ const u32 entry = ((*native_executing_macro - 0xe00) / 2) % native_macro_positions.size();
+ native_executing_macro.reset();
+ const auto result = native_macro->native_interpreter.Execute(native_macro->native_code,
+ native_macro_positions[entry], native_parameters,
+ [&](u32 address) { return address < native_registers.size() ? native_registers[address] : 0U; },
+ [&](u32 address, u32 value) {
+ static u32 diagnostic_writes{};
+ if (diagnostic_writes++ < 4096) LOG_DEBUG(HW_GPU, "Native compute MME emit method={:#x}, value={:#x}", address, value);
+ if (address >= 0xe00) return false;
+ CallMethod(system, address, value, true);
+ return true;
+ });
+ if (!result.success) {
+ native_macro->native_execution_failed = true;
+ std::ofstream dump("native-compute-mme.bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native_macro->native_code.data()), sizeof(native_macro->native_code));
+ LOG_ERROR(HW_GPU, "Native MME failed: compute start={:#x}, pc={:#x}, consumed={}/{}, error={}, parameters={:08x}",
+ native_macro_positions[entry], result.pc, result.consumed, native_parameters.size(), result.error,
+ fmt::join(native_parameters, ","));
+ }
+ native_parameters.clear();
+ return true;
+ }
+ native_registers[method] = argument;
+ if (method == 0x5a || (method >= 0xc6 && method <= 0x108)) {
+ VideoCommon::TraceNativeQuery(12, memory_manager.GetID(), method, 0, 0, argument, 0);
+ }
+ if (method == 0x107) {
+ const GPUVAddr address = ((GPUVAddr{native_registers[0xc6]} << 32) |
+ native_registers[0xc7]) << 8;
+ const GPUVAddr program = (GPUVAddr{native_registers[0xc8 + 49] & 0x1ffff} << 32) |
+ native_registers[0xc8 + 48];
+ const u32 control = native_registers[0xa7];
+ LOG_WARNING(HW_GPU, "Native inline QMD: address={:#x}, program={:#x}, control={:#x}, grid={}/{}/{}",
+ address, program, control, native_registers[0xc8 + 12],
+ native_registers[0xc8 + 13] & 0xffff, native_registers[0xc8 + 14] & 0xffff);
+ if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) {
+ std::ofstream dump("native-inline-qmd.bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native_registers.data() + 0xc8), 256);
+ std::ofstream registers("native-inline-compute-registers.bin", std::ios::binary);
+ registers.write(reinterpret_cast<const char*>(native_registers.data()), sizeof(native_registers));
+ if (memory_manager.IsFullyMappedRange(program, 0x10000)) {
+ std::vector<u8> code(0x10000);
+ memory_manager.ReadBlockUnsafe(program, code.data(), code.size());
+ std::ofstream shader("native-inline-compute-shader.bin", std::ios::binary);
+ shader.write(reinterpret_cast<const char*>(code.data()), code.size());
+ }
+ }
+ if (!memory_manager.IsFullyMappedRange(address, 256)) {
+ native_macro->native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Unmapped native inline QMD destination");
+ return true;
+ }
+ rasterizer->AccelerateInlineToMemory(address, 256,
+ std::span<const u8>{reinterpret_cast<const u8*>(native_registers.data() + 0xc8), 256});
+ if ((control & (1U << 24)) || !(native_registers[0xcc] & (1U << 12))) {
+ const u32 previous = native_registers[0xad];
+ native_registers[0xad] = static_cast<u32>(address >> 8);
+ ProcessNativeMethod(system, 0xb0, 3, true);
+ native_registers[0xad] = previous;
+ }
+ return true;
+ }
+ switch (method) {
+ case 0x45: native_macro->ClearCode(argument); return true;
+ case 0x46: native_macro->AddCode(0, argument); return true;
+ case 0x47: native_bind_pointer = argument; return true;
+ case 0x48:
+ if (native_bind_pointer < native_macro_positions.size()) native_macro_positions[native_bind_pointer++] = argument;
+ else LOG_ERROR(HW_GPU, "Native compute macro bind out of bounds");
+ return true;
+ case 0xae: case 0xaf: case 0xb0: {
+ const GPUVAddr qmd_address = GPUVAddr{native_registers[0xad]} << 8;
+ VideoCommon::TraceNativeQuery(9, memory_manager.GetID(), qmd_address,
+ memory_manager.GpuToCpuAddress(qmd_address).value_or(0), method, argument, 0);
+ if (method == 0xae || (method == 0xaf && !(argument & 2)) ||
+ (method == 0xb0 && ((argument & 15) == 0 || (argument & 15) == 1))) return true;
+ if ((argument & 15) == 12 && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) {
+ static u32 captured{};
+ if (captured++ == 0 && memory_manager.IsFullyMappedRange(qmd_address, 256)) {
+ std::array<u32, 64> descriptor{};
+ memory_manager.ReadBlockUnsafe(qmd_address, descriptor.data(), sizeof(descriptor));
+ std::ofstream dump("native-increment-qmd.bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(descriptor.data()), sizeof(descriptor));
+ LOG_WARNING(HW_GPU, "Native increment dependency descriptor: address={:#x}, words={:08x}",
+ qmd_address, fmt::join(descriptor, ","));
+ }
+ }
+ const auto error = native_qmd.Signal(qmd_address, argument,
+ [&](u64 address, NativeQmdScheduler::Descriptor& descriptor) {
+ if (!memory_manager.IsFullyMappedRange(address, sizeof(descriptor))) return false;
+ memory_manager.ReadBlockUnsafe(address, descriptor.data(), sizeof(descriptor));
+ for (u32 slot = 0; slot < 2; ++slot) {
+ const u32 flags = descriptor[25 + slot * 4];
+ if (!(flags & (1U << 23))) continue;
+ const GPUVAddr destination = (GPUVAddr{flags & 0xff} << 32) | descriptor[24 + slot * 4];
+ VideoCommon::TraceNativeQuery(7 + slot, memory_manager.GetID(), destination,
+ memory_manager.GpuToCpuAddress(destination).value_or(0),
+ descriptor[4], descriptor[26 + slot * 4], flags);
+ }
+ return true;
+ },
+ [&](u64 address, u32 payload, u32 flags) {
+ if (!memory_manager.IsFullyMappedRange(address, sizeof(payload))) return false;
+ if (flags & (1U << 26)) {
+ rasterizer->SignalFence([this, address, payload] {
+ const u32 value = memory_manager.Read<u32>(address) + payload;
+ memory_manager.Write<u32>(address, value);
+ VideoCommon::TraceNativeQuery(1, memory_manager.GetID(), address,
+ memory_manager.GpuToCpuAddress(address).value_or(0), 0, value, 1U << 26);
+ });
+ } else {
+ rasterizer->Query(address, VideoCommon::QueryType::Payload,
+ VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
+ }
+ return true;
+ }, [&](const auto& descriptor) { return DispatchNativeCompute(descriptor); });
+ if (error.empty()) {
+ LOG_DEBUG(HW_GPU, "Native QMD control action={:#x}, QMD={:#x}", argument, qmd_address);
+ return true;
+ }
+ native_macro->native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Native QMD failed: {}", error);
+ std::array<u32, 128> qmd{};
+ if (memory_manager.IsFullyMappedRange(qmd_address, sizeof(qmd))) {
+ memory_manager.ReadBlockUnsafe(qmd_address, qmd.data(), sizeof(qmd));
+ std::ofstream dump("native-qmd.bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(qmd.data()), sizeof(qmd));
+ std::ofstream macros("native-dispatch-mme.bin", std::ios::binary);
+ macros.write(reinterpret_cast<const char*>(native_macro->native_code.data()), sizeof(native_macro->native_code));
+ std::ofstream registers("native-dispatch-registers.bin", std::ios::binary);
+ registers.write(reinterpret_cast<const char*>(native_registers.data()), sizeof(native_registers));
+ }
+ LOG_ERROR(HW_GPU, "Native compute dispatch unsupported: method={:#x}, value={:#x}, QMD={:#x}, words={:08x}", method, argument,
+ qmd_address, fmt::join(qmd, ","));
+ return true;
+ }
+ default:
+ // The inline upload register block is shared with the older class.
+ if (method < Regs::NUM_REGS) regs.reg_array[method] = argument;
+ return method != KEPLER_COMPUTE_REG_INDEX(exec_upload) && method != KEPLER_COMPUTE_REG_INDEX(data_upload);
+ }
+}
+
void KeplerCompute::CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) {
- ASSERT_MSG(method < Regs::NUM_REGS, "Invalid KeplerCompute register, increase the size of the Regs structure");
+ if (native_mode && ProcessNativeMethod(system, method, method_argument, is_last_call)) return;
+ if (method >= Regs::NUM_REGS) {
+ LOG_ERROR(HW_GPU, "Compute register out of range: {:#x}", method);
+ return;
+ }
regs.reg_array[method] = method_argument;
@@ -85,6 +253,10 @@ void KeplerCompute::CallMethod(Core::System& system, u32 method, u32 method_argu
}
void KeplerCompute::CallMultiMethod(Core::System& system, u32 method, const u32* base_start, u32 amount, u32 methods_pending) {
+ if (native_mode) {
+ for (u32 i = 0; i < amount; ++i) CallMethod(system, method, base_start[i], methods_pending - i <= 1);
+ return;
+ }
switch (method) {
case KEPLER_COMPUTE_REG_INDEX(data_upload):
upload_address = current_dma_segment;
@@ -100,6 +272,44 @@ void KeplerCompute::CallMultiMethod(Core::System& system, u32 method, const u32*
}
}
+bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& qmd) {
+ launch_description = {};
+ launch_description.grid_dim_x.Assign(qmd[12]);
+ launch_description.grid_dim_y.Assign(qmd[13] & 0xffff);
+ launch_description.grid_dim_z.Assign(qmd[14] & 0xffff);
+ launch_description.shared_alloc.Assign(qmd[17] & 0x3ffff);
+ launch_description.block_dim_x.Assign(qmd[18] >> 16);
+ launch_description.block_dim_y.Assign(qmd[19] & 0xffff);
+ launch_description.block_dim_z.Assign(qmd[19] >> 16);
+ launch_description.const_buffer_enable_mask.Assign(qmd[20] & 0xff);
+ launch_description.local_pos_alloc.Assign(qmd[23] & 0xffffff);
+ launch_description.local_crs_alloc.Assign(qmd[50] & 0xffffff);
+ regs.code_loc.address_low = qmd[48];
+ regs.code_loc.address_high = qmd[49] & 0x1ffff;
+ for (u32 slot = 0; slot < NumConstBuffers; ++slot) {
+ const u32 high = qmd[33 + slot * 2];
+ if (high & 0x1ff00) {
+ LOG_ERROR(HW_GPU, "Native compute constant address exceeds supported GPU width");
+ return false;
+ }
+ auto& buffer = launch_description.const_buffer_config[slot];
+ buffer.address_low = qmd[32 + slot * 2];
+ buffer.address_high.Assign(high & 0xff);
+ buffer.size.Assign(((high >> 19) & 0x1fff) << 4);
+ }
+ if (!launch_description.block_dim_x || !launch_description.block_dim_y || !launch_description.block_dim_z)
+ return false;
+ try {
+ LOG_INFO(HW_GPU, "Native compute begin: program={:#x}", regs.code_loc.Address());
+ rasterizer->DispatchCompute();
+ LOG_INFO(HW_GPU, "Native compute complete");
+ } catch (const Shader::Exception& error) {
+ LOG_ERROR(HW_GPU, "Native compute shader rejected: {}", error.what());
+ return false;
+ }
+ return true;
+}
+
void KeplerCompute::ProcessLaunch() {
const GPUVAddr launch_desc_loc = regs.launch_desc_loc.Address();
memory_manager.ReadBlockUnsafe(launch_desc_loc, &launch_description,
diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h
index 078d3d45be..99cf74d1e8 100644
--- a/src/video_core/engines/kepler_compute.h
+++ b/src/video_core/engines/kepler_compute.h
@@ -15,6 +15,8 @@
#include "common/common_types.h"
#include "video_core/engines/engine_interface.h"
#include "video_core/engines/engine_upload.h"
+#include "video_core/macro.h"
+#include "video_core/native_qmd.h"
#include "video_core/textures/texture.h"
namespace Core {
@@ -51,6 +53,22 @@ public:
/// Binds a rasterizer to this engine.
void BindRasterizer(VideoCore::RasterizerInterface* rasterizer);
+ void SetNativeMacroMode(MacroEngine* shared_macro, std::span<u32> positions, NativeMacroContext* context = nullptr) {
+ if (native_context != context) native_compute_uploaded = false;
+ native_context = context;
+ native_mode = shared_macro != nullptr;
+ upload_state.SetNativePartialTransfers(native_mode);
+ native_macro = shared_macro;
+ native_macro_positions = positions;
+ if (native_mode) execution_mask.set();
+ else {
+ execution_mask.reset();
+ execution_mask[KEPLER_COMPUTE_REG_INDEX(exec_upload)] = true;
+ execution_mask[KEPLER_COMPUTE_REG_INDEX(data_upload)] = true;
+ execution_mask[KEPLER_COMPUTE_REG_INDEX(launch)] = true;
+ }
+ }
+
static constexpr std::size_t NumConstBuffers = 8;
struct Regs {
@@ -211,7 +229,22 @@ public:
return indirect_compute;
}
+ bool IsNativeMode() const { return native_mode; }
+
private:
+ bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor);
+ bool ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call);
+ NativeQmdScheduler native_qmd;
+ bool native_mode{};
+ NativeMacroContext* native_context{};
+ bool native_compute_uploaded{};
+ MacroEngine* native_macro{};
+ std::array<u32, 0x1000> native_registers{};
+ std::span<u32> native_macro_positions;
+ std::vector<u32> native_parameters;
+ std::optional<u32> native_executing_macro;
+ u32 native_bind_pointer{};
+
void ProcessLaunch();
void ConsumeSinkImpl(Core::System& system) override;
diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp
index e7da9d9138..f28a042d19 100644
--- a/src/video_core/engines/maxwell_3d.cpp
+++ b/src/video_core/engines/maxwell_3d.cpp
@@ -5,6 +5,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <cstring>
+#include <limits>
#include <optional>
#include "common/assert.h"
#include "common/bit_util.h"
@@ -222,6 +223,13 @@ void Maxwell3D::ProcessMacro(Core::System& system, u32 method, const u32* base_s
}
}
+bool Maxwell3D::ReadMacroMemory(GPUVAddr address, std::span<u32> destination) {
+ if (!memory_manager.IsFullyMappedRange(address, destination.size_bytes())) return false;
+ // ReadBlock flushes renderer caches before reading data written by GPU work.
+ memory_manager.ReadBlock(address, destination.data(), destination.size_bytes());
+ return true;
+}
+
void Maxwell3D::RefreshParametersImpl() {
if (!Settings::IsGPULevelHigh()) {
return;
@@ -253,7 +261,10 @@ u32 Maxwell3D::GetMaxCurrentVertices() {
const auto& limit = regs.vertex_stream_limits[index];
const GPUVAddr gpu_addr_begin = array.Address();
const GPUVAddr gpu_addr_end = limit.Address() + 1;
- const u32 address_size = static_cast<u32>(gpu_addr_end - gpu_addr_begin);
+ const u64 stream_size = GetNativeMacroEngine().native_turing
+ ? regs.NativeVertexStreamSize(index)
+ : gpu_addr_end - gpu_addr_begin;
+ const u32 address_size = static_cast<u32>(std::min<u64>(stream_size, std::numeric_limits<u32>::max()));
num_vertices = (std::max)(
num_vertices, address_size / (std::max)(attribute.SizeInBytes(), array.stride.Value()));
break;
@@ -305,6 +316,9 @@ void Maxwell3D::ProcessDirtyRegisters(u32 method, u32 argument) {
}
void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argument, bool is_last_call) {
+ if (GetNativeMacroEngine().native_turing && method == MAXWELL3D_REG_INDEX(clear_surface)) {
+ LOG_INFO(HW_GPU, "Native GPU surface clear: value={:#x}", argument);
+ }
switch (method) {
case MAXWELL3D_REG_INDEX(wait_for_idle):
return rasterizer->WaitForIdle();
@@ -312,9 +326,9 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume
shadow_state.shadow_ram_control = static_cast<Regs::ShadowRamControl>(nonshadow_argument);
return;
case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
- return macro_engine.ClearCode(regs.load_mme.instruction_ptr);
+ return GetNativeMacroEngine().ClearCode(regs.load_mme.instruction_ptr);
case MAXWELL3D_REG_INDEX(load_mme.instruction):
- return macro_engine.AddCode(regs.load_mme.instruction_ptr, argument);
+ return GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr, argument);
case MAXWELL3D_REG_INDEX(load_mme.start_address):
return ProcessMacroBind(argument);
case MAXWELL3D_REG_INDEX(falcon[4]):
@@ -378,10 +392,10 @@ void Maxwell3D::CallMacroMethod(Core::System& system, u32 method, const std::vec
// Lookup the macro offset
const u32 entry =
- ((method - MacroRegistersStart) >> 1) % static_cast<u32>(macro_positions.size());
+ ((method - MacroRegistersStart) >> 1) % static_cast<u32>(GetNativeMacroPositions().size());
// Execute the current macro.
- macro_engine.Execute(system, *this, macro_positions[entry], parameters);
+ GetNativeMacroEngine().Execute(system, *this, GetNativeMacroPositions()[entry], parameters);
draw_manager.DrawDeferred(*this);
}
@@ -453,11 +467,11 @@ void Maxwell3D::CallMultiMethod(Core::System& system, u32 method, const u32* bas
}
void Maxwell3D::ProcessMacroUpload(u32 data) {
- macro_engine.AddCode(regs.load_mme.instruction_ptr++, data);
+ GetNativeMacroEngine().AddCode(regs.load_mme.instruction_ptr++, data);
}
void Maxwell3D::ProcessMacroBind(u32 data) {
- macro_positions[regs.load_mme.start_address_ptr++] = data;
+ GetNativeMacroPositions()[regs.load_mme.start_address_ptr++] = data;
}
void Maxwell3D::ProcessFirmwareCall4() {
@@ -586,6 +600,9 @@ void Maxwell3D::ProcessCounterReset() {
}
void Maxwell3D::ProcessSyncPoint() {
+ if (GetNativeMacroEngine().native_turing) {
+ LOG_DEBUG(HW_GPU, "Native 3D syncpoint increment: id={}, raw={:#x}", regs.sync_info.sync_point.Value(), regs.reg_array[MAXWELL3D_REG_INDEX(sync_info)]);
+ }
const u32 sync_point = regs.sync_info.sync_point.Value();
[[maybe_unused]] const u32 cache_flush = regs.sync_info.clean_l2.Value();
rasterizer->SignalSyncPoint(sync_point);
diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h
index 0cab2ae986..9533626ecb 100644
--- a/src/video_core/engines/maxwell_3d.h
+++ b/src/video_core/engines/maxwell_3d.h
@@ -2605,6 +2605,12 @@ public:
u32 stop_shader_mask;
};
+ // Native C597/C797 use byte counts at 0x600 rather than legacy end addresses.
+ u64 NativeVertexStreamSize(std::size_t index) const {
+ const std::size_t method = 0x600 / sizeof(u32) + index * 2;
+ return (u64{reg_array[method] & 0xffU} << 32) | reg_array[method + 1];
+ }
+
// clang-format off
union {
struct {
@@ -3129,6 +3135,17 @@ public:
/// Reads a register value located at the input method address
u32 GetRegisterValue(u32 method) const;
+ void SetNativeTuringMacroMode(bool enabled) {
+ GetNativeMacroEngine().native_turing = enabled;
+ upload_state.SetNativePartialTransfers(enabled);
+ }
+ void SetNativeMacroContext(std::shared_ptr<NativeMacroContext> context) {
+ native_macro_context = std::move(context);
+ }
+ MacroEngine& GetNativeMacroEngine() { return native_macro_context ? native_macro_context->engine : macro_engine; }
+ std::span<u32> GetNativeMacroPositions() { return native_macro_context ? std::span<u32>{native_macro_context->positions} : std::span<u32>{macro_positions}; }
+
+
/// Write the value to the register identified by method.
void CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) override;
@@ -3180,6 +3197,8 @@ public:
return current_macro_dirty;
}
+ bool ReadMacroMemory(GPUVAddr address, std::span<u32> destination);
+
u32 GetMaxCurrentVertices();
size_t EstimateIndexBufferSize();
@@ -3264,6 +3283,7 @@ private:
std::vector<u32> macro_params;
/// Interpreter for the macro codes uploaded to the GPU.
+ std::shared_ptr<NativeMacroContext> native_macro_context;
MacroEngine macro_engine;
Upload::State upload_state;
diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp
index ed4675ed12..7c8872e209 100644
--- a/src/video_core/engines/maxwell_dma.cpp
+++ b/src/video_core/engines/maxwell_dma.cpp
@@ -14,6 +14,7 @@
#include "video_core/engines/maxwell_dma.h"
#include "video_core/guest_memory.h"
#include "video_core/memory_manager.h"
+#include "video_core/native_query_trace.h"
#include "video_core/renderer_base.h"
#include "video_core/textures/decoders.h"
@@ -58,6 +59,12 @@ void MaxwellDMA::CallMultiMethod(Core::System& system, u32 method, const u32* ba
}
void MaxwellDMA::Launch() {
+ if (memory_manager.UsesNativeTiling()) {
+ const GPUVAddr destination = regs.offset_out;
+ VideoCommon::TraceNativeQuery(10, memory_manager.GetID(), destination,
+ memory_manager.GpuToCpuAddress(destination).value_or(0),
+ regs.line_count, static_cast<GPUVAddr>(regs.offset_in), regs.line_length_in);
+ }
LOG_TRACE(Render_OpenGL, "DMA copy {:#x} -> {:#x}", static_cast<GPUVAddr>(regs.offset_in), GPUVAddr(regs.offset_out));
// TODO(Subv): Perform more research and implement all features of this engine.
@@ -219,7 +226,7 @@ void MaxwellDMA::CopyBlockLinearToPitch() {
UnswizzleSubrect(tmp_write_buffer, tmp_read_buffer, bytes_per_pixel, width, height, depth,
x_offset, src_params.origin.y, x_elements, regs.line_count, block_height,
- block_depth, dst_operand.pitch);
+ block_depth, dst_operand.pitch, memory_manager.UsesNativeTiling());
}
void MaxwellDMA::CopyPitchToBlockLinear() {
@@ -284,7 +291,7 @@ void MaxwellDMA::CopyPitchToBlockLinear() {
// If the input is linear and the output is tiled, swizzle the input and copy it over.
SwizzleSubrect(tmp_write_buffer, tmp_read_buffer, bytes_per_pixel, width, height, depth,
x_offset, dst_params.origin.y, x_elements, regs.line_count, block_height,
- block_depth, regs.pitch_in);
+ block_depth, regs.pitch_in, memory_manager.UsesNativeTiling());
}
void MaxwellDMA::CopyBlockLinearToBlockLinear() {
@@ -337,11 +344,11 @@ void MaxwellDMA::CopyBlockLinearToBlockLinear() {
UnswizzleSubrect(intermediate_buffer, tmp_read_buffer, bytes_per_pixel, src_width, src.height,
src.depth, src_x_offset, src.origin.y, x_elements, regs.line_count,
- src.block_size.height, src.block_size.depth, pitch);
+ src.block_size.height, src.block_size.depth, pitch, memory_manager.UsesNativeTiling());
SwizzleSubrect(tmp_write_buffer, intermediate_buffer, bytes_per_pixel, dst_width, dst.height,
dst.depth, dst_x_offset, dst.origin.y, x_elements, regs.line_count,
- dst.block_size.height, dst.block_size.depth, pitch);
+ dst.block_size.height, dst.block_size.depth, pitch, memory_manager.UsesNativeTiling());
}
void MaxwellDMA::ReleaseSemaphore() {
diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp
index 0b05d31718..9031ea75df 100644
--- a/src/video_core/engines/puller.cpp
+++ b/src/video_core/engines/puller.cpp
@@ -4,6 +4,8 @@
// SPDX-FileCopyrightText: 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
+#include <cstdlib>
+
#include "common/assert.h"
#include "common/logging.h"
#include "common/settings.h"
@@ -18,6 +20,7 @@
#include "video_core/engines/puller.h"
#include "video_core/gpu.h"
#include "video_core/memory_manager.h"
+#include "video_core/native_query_trace.h"
#include "video_core/rasterizer_interface.h"
namespace Tegra::Engines {
@@ -27,6 +30,19 @@ namespace {
constexpr u32 Gf100BindClassMask = 0xffff;
constexpr u32 Gf100BindValidMask = 0x1f0000 | Gf100BindClassMask;
+u32 ReadNativeSemaphoreWord(DmaPusher& dma_pusher, GPUVAddr address) {
+ const auto cpu_address = dma_pusher.memory_manager.GpuToCpuAddress(address);
+ if (cpu_address && dma_pusher.rasterizer->MustFlushRegion(*cpu_address, sizeof(u32), VideoCommon::CacheType::BufferCache)) {
+ dma_pusher.rasterizer->FlushRegion(*cpu_address, sizeof(u32), VideoCommon::CacheType::BufferCache);
+ const u32 value = dma_pusher.memory_manager.Read<u32>(address);
+ LOG_WARNING(HW_GPU, "Native semaphore buffer readback: address={:#x}, current={}", address, value);
+ VideoCommon::TraceNativeQuery(6, dma_pusher.memory_manager.GetID(), address, *cpu_address, 0, value, 0);
+ return value;
+ }
+ return dma_pusher.memory_manager.Read<u32>(address);
+}
+
+
} // Anonymous namespace
void Puller::ProcessBindMethod(DmaPusher& dma_pusher, const MethodCall& method_call) {
@@ -36,18 +52,39 @@ void Puller::ProcessBindMethod(DmaPusher& dma_pusher, const MethodCall& method_c
engine &= Gf100BindClassMask;
}
+ if (dma_pusher.channel_state.native_abi) dma_pusher.memory_manager.EnableNativeTiling();
const auto engine_id = static_cast<EngineID>(engine);
bound_engines[method_call.subchannel] = engine_id;
switch (engine_id) {
case EngineID::FERMI_TWOD_A:
dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->fermi_2d, method_call.subchannel, EngineTypes::Fermi2D);
break;
+ case EngineID::AMPERE_B:
+ if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; }
+ dma_pusher.channel_state.payload->maxwell_3d.SetNativeTuringMacroMode(true);
+ dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->maxwell_3d, method_call.subchannel, EngineTypes::Maxwell3D);
+ break;
case EngineID::MAXWELL_B:
+ dma_pusher.channel_state.payload->maxwell_3d.SetNativeTuringMacroMode(false);
dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->maxwell_3d, method_call.subchannel, EngineTypes::Maxwell3D);
break;
+ case EngineID::AMPERE_COMPUTE_B:
+ if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; }
+ [[fallthrough]];
case EngineID::KEPLER_COMPUTE_B:
+ if (engine_id == EngineID::AMPERE_COMPUTE_B && dma_pusher.channel_state.native_macro_context) {
+ auto& context = *dma_pusher.channel_state.native_macro_context;
+ dma_pusher.channel_state.payload->kepler_compute.SetNativeMacroMode(&context.engine, context.positions, &context);
+ } else {
+ dma_pusher.channel_state.payload->kepler_compute.SetNativeMacroMode(
+ engine_id == EngineID::AMPERE_COMPUTE_B ? &dma_pusher.channel_state.payload->maxwell_3d.GetNativeMacroEngine() : nullptr,
+ dma_pusher.channel_state.payload->maxwell_3d.GetNativeMacroPositions());
+ }
dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->kepler_compute, method_call.subchannel, EngineTypes::KeplerCompute);
break;
+ case EngineID::AMPERE_DMA_COPY_B:
+ if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; }
+ [[fallthrough]];
case EngineID::MAXWELL_DMA_COPY_A:
dma_pusher.BindSubchannel(&dma_pusher.channel_state.payload->maxwell_dma, method_call.subchannel, EngineTypes::MaxwellDMA);
break;
@@ -64,6 +101,14 @@ void Puller::ProcessBindMethod(DmaPusher& dma_pusher, const MethodCall& method_c
}
void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) {
+ if (dma_pusher.channel_state.native_abi && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) {
+ static u32 fence_traces{};
+ if (fence_traces++ < 512) {
+ LOG_INFO(HW_GPU, "Native fence action: channel={}, id={}, op={}, value={}",
+ dma_pusher.channel_state.bind_id, regs.fence_action.syncpoint_id.Value(),
+ static_cast<u32>(regs.fence_action.op.Value()), regs.fence_value);
+ }
+ }
switch (regs.fence_action.op) {
case Puller::FenceOperation::Acquire:
// UNIMPLEMENTED_MSG("Channel Scheduling pending.");
@@ -71,6 +116,13 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) {
dma_pusher.rasterizer->ReleaseFences();
break;
case Puller::FenceOperation::Increment:
+ if (dma_pusher.channel_state.native_abi) {
+ const auto& context = dma_pusher.channel_state.native_macro_context;
+ if (context && (context->engine.native_execution_failed || context->compute_engine.native_execution_failed)) {
+ LOG_ERROR(HW_GPU, "Native submission completion blocked by failed GPU work: syncpoint={}", regs.fence_action.syncpoint_id.Value());
+ break;
+ }
+ }
dma_pusher.rasterizer->SignalSyncPoint(regs.fence_action.syncpoint_id);
break;
default:
@@ -79,16 +131,68 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) {
}
}
+bool Puller::CanResumeNativeAcquire(DmaPusher& dma_pusher) const {
+ return native_acquire && !dma_pusher.memory_manager.IsShuttingDown() &&
+ native_acquire->Matches(ReadNativeSemaphoreWord(dma_pusher, native_acquire->address));
+}
+
+bool Puller::ResumeNativeAcquire(DmaPusher& dma_pusher) {
+ if (!native_acquire) return true;
+ if (!CanResumeNativeAcquire(dma_pusher)) return false;
+ LOG_WARNING(HW_GPU, "Native channel resumed after semaphore: channel={}, address={:#x}, current={}, required={}",
+ dma_pusher.channel_state.bind_id, native_acquire->address,
+ dma_pusher.memory_manager.Read<u32>(native_acquire->address), native_acquire->value);
+ VideoCommon::TraceNativeQuery(5, dma_pusher.memory_manager.GetID(), native_acquire->address,
+ dma_pusher.memory_manager.GpuToCpuAddress(native_acquire->address).value_or(0),
+ 0, native_acquire->value, native_acquire->operation);
+ native_acquire.reset();
+ return true;
+}
+
void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) {
const auto semaphoreOperationMask = 0xF;
const auto op = GpuSemaphoreOperation(regs.semaphore_trigger & semaphoreOperationMask);
if (op == GpuSemaphoreOperation::WriteLong) {
const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()};
const u32 payload = regs.semaphore_sequence;
- dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::HasTimeout, payload, 0);
+ const auto properties = dma_pusher.channel_state.native_abi && (regs.semaphore_trigger & (1U << 24))
+ ? VideoCommon::QueryPropertiesFlags::IsAFence : VideoCommon::QueryPropertiesFlags::HasTimeout;
+ dma_pusher.rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, properties, payload, 0);
} else {
do {
- const u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress());
+ u32 word = dma_pusher.memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress());
+ if (dma_pusher.channel_state.native_abi) {
+ word = ReadNativeSemaphoreWord(dma_pusher, regs.semaphore_address.SemaphoreAddress());
+ const auto ready = [&] {
+ if (op == GpuSemaphoreOperation::AcquireEqual) return word == regs.semaphore_sequence;
+ if (op == GpuSemaphoreOperation::AcquireGequal) return word >= regs.semaphore_sequence;
+ if (op == GpuSemaphoreOperation::AcquireMask) return (word & regs.semaphore_sequence) != 0;
+ return false;
+ };
+ if (!ready()) {
+ dma_pusher.rasterizer->ReleaseFences(true);
+ word = ReadNativeSemaphoreWord(dma_pusher, regs.semaphore_address.SemaphoreAddress());
+ static u32 native_wait_traces{};
+ if (ready() && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && native_wait_traces++ < 8) {
+ LOG_WARNING(HW_GPU, "Native semaphore completed after pending GPU fences: address={:#x}, current={}, required={}",
+ regs.semaphore_address.SemaphoreAddress(), word, regs.semaphore_sequence);
+ }
+ }
+ if (!ready()) {
+ if (!NativeSemaphoreWait::Supported(static_cast<u32>(op))) {
+ if (auto context = dma_pusher.channel_state.native_macro_context) context->engine.native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Unsupported native semaphore operation {}", static_cast<u32>(op));
+ } else {
+ native_acquire = NativeSemaphoreWait{regs.semaphore_address.SemaphoreAddress(), regs.semaphore_sequence, static_cast<u32>(op)};
+ VideoCommon::TraceNativeQuery(4, dma_pusher.memory_manager.GetID(), native_acquire->address,
+ dma_pusher.memory_manager.GpuToCpuAddress(native_acquire->address).value_or(0),
+ 0, native_acquire->value, native_acquire->operation);
+ LOG_WARNING(HW_GPU, "Native channel suspended for semaphore: channel={}, address={:#x}, current={}, required={}",
+ dma_pusher.channel_state.bind_id, native_acquire->address, word, native_acquire->value);
+ }
+ }
+ break;
+ }
regs.acquire_source = true;
regs.acquire_value = regs.semaphore_sequence;
if (op == GpuSemaphoreOperation::AcquireEqual) {
@@ -211,12 +315,19 @@ void Puller::CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_ca
case EngineID::FERMI_TWOD_A:
dma_pusher.channel_state.payload->fermi_2d.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall());
break;
+ case EngineID::AMPERE_B:
case EngineID::MAXWELL_B:
dma_pusher.channel_state.payload->maxwell_3d.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall());
break;
+ case EngineID::AMPERE_COMPUTE_B:
+ if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; }
+ [[fallthrough]];
case EngineID::KEPLER_COMPUTE_B:
dma_pusher.channel_state.payload->kepler_compute.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall());
break;
+ case EngineID::AMPERE_DMA_COPY_B:
+ if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; }
+ [[fallthrough]];
case EngineID::MAXWELL_DMA_COPY_A:
dma_pusher.channel_state.payload->maxwell_dma.CallMethod(dma_pusher.system, method_call.method, method_call.argument, method_call.IsLastCall());
break;
@@ -239,12 +350,19 @@ void Puller::CallEngineMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subcha
case EngineID::FERMI_TWOD_A:
dma_pusher.channel_state.payload->fermi_2d.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending);
break;
+ case EngineID::AMPERE_B:
case EngineID::MAXWELL_B:
dma_pusher.channel_state.payload->maxwell_3d.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending);
break;
+ case EngineID::AMPERE_COMPUTE_B:
+ if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; }
+ [[fallthrough]];
case EngineID::KEPLER_COMPUTE_B:
dma_pusher.channel_state.payload->kepler_compute.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending);
break;
+ case EngineID::AMPERE_DMA_COPY_B:
+ if (!dma_pusher.channel_state.native_abi) { UNIMPLEMENTED_MSG("Native GPU class on legacy channel"); return; }
+ [[fallthrough]];
case EngineID::MAXWELL_DMA_COPY_A:
dma_pusher.channel_state.payload->maxwell_dma.CallMultiMethod(dma_pusher.system, method, base_start, amount, methods_pending);
break;
diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h
index e8994f5640..8dde8440bd 100644
--- a/src/video_core/engines/puller.h
+++ b/src/video_core/engines/puller.h
@@ -8,11 +8,13 @@
#include <array>
#include <cstddef>
+#include <optional>
#include <vector>
#include "common/bit_field.h"
#include "common/common_funcs.h"
#include "common/common_types.h"
#include "video_core/engines/engine_interface.h"
+#include "video_core/native_semaphore.h"
namespace Core {
class System;
@@ -23,6 +25,9 @@ class MemoryManager;
class DmaPusher;
enum class EngineID {
+ AMPERE_B = 0xc797,
+ AMPERE_COMPUTE_B = 0xc7c0,
+ AMPERE_DMA_COPY_B = 0xc7b5,
NV01_TIMER = 0x0004,
FERMI_TWOD_A = 0x902D, // 2D Engine
MAXWELL_B = 0xB197, // 3D Engine
@@ -70,6 +75,10 @@ public:
BitField<8, 24, u32> syncpoint_id;
};
+ [[nodiscard]] bool HasPendingNativeAcquire() const { return native_acquire.has_value(); }
+ [[nodiscard]] bool CanResumeNativeAcquire(DmaPusher& dma_pusher) const;
+ bool ResumeNativeAcquire(DmaPusher& dma_pusher);
+
void CallMethod(DmaPusher& dma_pusher, const MethodCall& method_call);
void CallMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending);
void BindRasterizer(DmaPusher& dma_pusher, VideoCore::RasterizerInterface* rasterizer);
@@ -127,6 +136,8 @@ private:
void ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher);
[[nodiscard]] bool ExecuteMethodOnEngine(DmaPusher& dma_pusher, u32 method);
+ std::optional<NativeSemaphoreWait> native_acquire;
+
/// Mapping of command subchannels to their bound engine ids
std::array<EngineID, 8> bound_engines{};
diff --git a/src/video_core/framebuffer_config.h b/src/video_core/framebuffer_config.h
index 1fc70583f9..bb041cf12f 100644
--- a/src/video_core/framebuffer_config.h
+++ b/src/video_core/framebuffer_config.h
@@ -37,6 +37,7 @@ struct FramebufferConfig {
Service::android::BufferTransformFlags transform_flags{};
Common::Rectangle<int> crop_rect{};
BlendMode blending{};
+ bool native_tiling{};
u32 layer_stack_mask{Service::Nvnflinger::DefaultLayerStackMask};
};
diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp
index 1621b24125..e8a0685e72 100644
--- a/src/video_core/gpu.cpp
+++ b/src/video_core/gpu.cpp
@@ -252,6 +252,8 @@ struct GPU::Impl {
}
void RequestComposite(std::vector<Tegra::FramebufferConfig>&& layers, std::vector<Service::Nvidia::NvFence>&& fences) {
+ static u32 trace_composites{};
+ if (trace_composites++ < 16) LOG_INFO(HW_GPU, "Composite requested: layers={}, fences={}", layers.size(), fences.size());
const size_t num_fences{fences.size()};
size_t current_request_counter{};
if (num_fences != 0) {
@@ -281,6 +283,7 @@ struct GPU::Impl {
}
free_swap_counters.push_back(current_request_counter);
}
+ LOG_DEBUG(HW_GPU, "Composite fences complete");
renderer->Composite(composite_layers);
};
for (size_t i = 0; i < num_fences; i++) {
diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp
index a01375e7be..a8bf6cc848 100644
--- a/src/video_core/gpu_thread.cpp
+++ b/src/video_core/gpu_thread.cpp
@@ -4,6 +4,9 @@
// SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <chrono>
+#include <thread>
+
#include "common/assert.h"
#include "common/scope_exit.h"
#include "common/settings.h"
@@ -36,7 +39,15 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten
auto current_context = context.Acquire();
CommandDataContainer next;
while (!stop_token.stop_requested()) {
- state.queue.PopWait(next, stop_token);
+ if (scheduler.HasPendingNativeChannels()) {
+ scheduler.ResumePending(system.GPU());
+ if (!state.queue.TryPop(next)) {
+ std::this_thread::sleep_for(std::chrono::milliseconds{1});
+ continue;
+ }
+ } else {
+ state.queue.PopWait(next, stop_token);
+ }
if (stop_token.stop_requested()) {
break;
}
diff --git a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp
index 2a3446ca66..bbe73acd1a 100644
--- a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp
+++ b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp
@@ -39,6 +39,7 @@ UNIFORM(4) uint block_size;
UNIFORM(5) uint x_shift;
UNIFORM(6) uint block_height;
UNIFORM(7) uint block_height_mask;
+UNIFORM(8) uint native_tiling;
END_PUSH_CONSTANTS
#if HAS_EXTENDED_TYPES
@@ -77,7 +78,12 @@ uint SwizzleTable(uint pos) {
uint SwizzleOffset(uvec2 pos) {
pos = pos & SWIZZLE_MASK;
- return SwizzleTable(pos.y * 64 + pos.x);
+ uint offset = SwizzleTable(pos.y * 64 + pos.x);
+ if (native_tiling != 0) {
+ uint difference = ((offset >> 5) ^ (offset >> 6)) & 1;
+ offset ^= (difference << 5) | (difference << 6);
+ }
+ return offset;
}
uvec4 ReadTexel(uint offset) {
diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp
index 34e4f93307..8dfaf0cd1a 100644
--- a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp
+++ b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp
@@ -41,6 +41,7 @@ UNIFORM(6) uint block_height;
UNIFORM(7) uint block_height_mask;
UNIFORM(8) uint block_depth;
UNIFORM(9) uint block_depth_mask;
+UNIFORM(10) uint native_tiling;
END_PUSH_CONSTANTS
#if HAS_EXTENDED_TYPES
@@ -79,7 +80,12 @@ uint SwizzleTable(uint pos) {
uint SwizzleOffset(uvec2 pos) {
pos = pos & SWIZZLE_MASK;
- return SwizzleTable(pos.y * 64 + pos.x);
+ uint offset = SwizzleTable(pos.y * 64 + pos.x);
+ if (native_tiling != 0) {
+ uint difference = ((offset >> 5) ^ (offset >> 6)) & 1;
+ offset ^= (difference << 5) | (difference << 6);
+ }
+ return offset;
}
uvec4 ReadTexel(uint offset) {
diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp
index e084837f25..28b1dcf599 100644
--- a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp
+++ b/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp
@@ -42,7 +42,7 @@ layout(push_constant) uniform PushConstants {
uint block_depth; // Offset 48
uint block_depth_mask; // Offset 52
- int _pad; // Offset 56
+ uint native_tiling; // Offset 56
ivec3 destination; // Offset 60
} pc;
@@ -59,6 +59,7 @@ BEGIN_PUSH_CONSTANTS
UNIFORM(8) uint block_depth;
UNIFORM(9) uint block_depth_mask;
UNIFORM(10) uvec3 blocks_dim;
+ UNIFORM(11) uint native_tiling;
END_PUSH_CONSTANTS
#define pc // Map pc prefix to nothing for OpenGL compatibility
#endif
@@ -103,7 +104,12 @@ uint SwizzleTable(uint pos) {
// --- Helpers ---
uint SwizzleOffset(uvec2 pos) {
pos &= SWIZZLE_MASK;
- return SwizzleTable(pos.y * 64u + pos.x);
+ uint offset = SwizzleTable(pos.y * 64 + pos.x);
+ if (pc.native_tiling != 0) {
+ uint difference = ((offset >> 5) ^ (offset >> 6)) & 1;
+ offset ^= (difference << 5) | (difference << 6);
+ }
+ return offset;
}
uvec4 ReadTexel(uint offset) {
diff --git a/src/video_core/macro.cpp b/src/video_core/macro.cpp
index 86f528af15..4364204740 100644
--- a/src/video_core/macro.cpp
+++ b/src/video_core/macro.cpp
@@ -4,6 +4,8 @@
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <cstdlib>
+#include <unordered_set>
#include <cstring>
#include <fstream>
#include <optional>
@@ -1339,6 +1341,42 @@ static void Dump(u64 hash, std::span<const u32> code, bool decompiled = false) {
void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u32 method,
std::span<const u32> parameters) {
+ if (native_turing) {
+ maxwell3d.RefreshParameters();
+ const auto context = reinterpret_cast<std::uintptr_t>(this);
+ static std::unordered_set<std::uintptr_t> captured_contexts;
+ if (std::getenv("EDEN_NATIVE_FRAME_CAPTURE") && captured_contexts.insert(context).second) {
+ std::ofstream dump("native-graphics-mme-" + std::to_string(context) + ".bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code));
+ }
+ LOG_DEBUG(HW_GPU, "Native MME context={:#x}", context);
+ LOG_DEBUG(HW_GPU, "Native MME start={:#x}, parameters={}, data={:08x}", method, parameters.size(), fmt::join(parameters, ","));
+ u32 dma_count{};
+ const auto result = native_interpreter.Execute(native_code, method, parameters,
+ [&](u32 address) { return address < Maxwell3D::Regs::NUM_REGS ? maxwell3d.GetRegisterValue(address) : 0U; },
+ [&](u32 address, u32 value) {
+ if (address >= 0xe00) return false;
+ if (address == 0x158) dma_count = value; // MME_DMA_READ_FIFOED
+ maxwell3d.CallMethod(system, address, value, true);
+ return true;
+ }, [&]() -> std::optional<std::vector<u32>> {
+ if (dma_count > 0x10000) return std::nullopt;
+ std::vector<u32> data(dma_count);
+ const GPUVAddr address = (u64{maxwell3d.GetRegisterValue(0x154) & 0xff} << 32) |
+ maxwell3d.GetRegisterValue(0x155);
+ if (dma_count && !maxwell3d.ReadMacroMemory(address, data)) return std::nullopt;
+ dma_count = 0;
+ return data;
+ });
+ if (!result.success) {
+ native_execution_failed = true;
+ std::ofstream dump("native-mme-" + std::to_string(method) + ".bin", std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native_code.data()), sizeof(native_code));
+ LOG_ERROR(HW_GPU, "Native MME failed: start={:#x}, pc={:#x}, consumed={}/{}, error={}",
+ method, result.pc, result.consumed, parameters.size(), result.error);
+ }
+ return;
+ }
const auto execute_variant = [&system, &maxwell3d, &parameters,
method](AnyCachedMacro& cached) {
if (std::holds_alternative<MacroInterpreterImpl>(cached) ||
diff --git a/src/video_core/macro.h b/src/video_core/macro.h
index 90519a5429..52dfe7c82c 100644
--- a/src/video_core/macro.h
+++ b/src/video_core/macro.h
@@ -7,6 +7,7 @@
#pragma once
#include <memory>
+#include "video_core/macro_turing.h"
#include <span>
#include <variant>
#include <vector>
@@ -223,10 +224,15 @@ struct MacroEngine {
MacroEngine(bool is_interpreted_) noexcept : is_interpreted{is_interpreted_} {}
// Store the uploaded macro code to compile them when they're called.
inline void AddCode(u32 method, u32 data) noexcept {
+ if (native_turing) {
+ if (native_write_offset < native_code.size()) native_code[native_write_offset++] = data;
+ return;
+ }
uploaded_macro_code[method].push_back(data);
}
// Clear the code associated with a method.
inline void ClearCode(u32 method) noexcept {
+ if (native_turing) { native_write_offset = method < 0x1000 ? method * 3 : static_cast<u32>(native_code.size()); return; }
macro_cache.erase(method);
uploaded_macro_code.erase(method);
}
@@ -240,6 +246,19 @@ struct MacroEngine {
::Common::unordered_map<u32, CacheInfo> macro_cache;
::Common::unordered_map<u32, std::vector<u32>> uploaded_macro_code;
bool is_interpreted;
+ bool native_turing{};
+ bool native_execution_failed{};
+ u32 native_write_offset{};
+ std::array<u32, 0x3000> native_code{};
+ TuringMacroInterpreter native_interpreter;
+
+};
+
+struct NativeMacroContext {
+ MacroEngine engine{true};
+ std::array<u32, 0x80> positions{};
+ MacroEngine compute_engine{true};
+ std::array<u32, 0x80> compute_positions{};
};
} // namespace Tegra
diff --git a/src/video_core/macro_turing.h b/src/video_core/macro_turing.h
new file mode 100644
index 0000000000..18966fc083
--- /dev/null
+++ b/src/video_core/macro_turing.h
@@ -0,0 +1,237 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+// Instruction semantics adapted from Mesa mme_tu104_sim.c and mme_tu104.c:
+// Copyright © 2022 Collabora Ltd. SPDX-License-Identifier: MIT
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <span>
+#include <optional>
+#include <vector>
+#include <string_view>
+
+namespace Tegra {
+
+// TU104/GA10x MME has two ALUs and two outputs per 96-bit instruction.
+class TuringMacroInterpreter {
+public:
+ using Word = std::uint32_t;
+ struct Result {
+ bool success;
+ Word pc;
+ std::size_t consumed;
+ std::string_view error;
+ };
+
+ template <typename Read, typename Emit>
+ Result Execute(std::span<const Word> code, Word start,
+ std::span<const Word> parameters, Read read, Emit emit) {
+ return Execute(code, start, parameters, read, emit,
+ []() -> std::optional<std::vector<Word>> { return std::nullopt; });
+ }
+
+ template <typename Read, typename Emit, typename Barrier>
+ Result Execute(std::span<const Word> code, Word start,
+ std::span<const Word> parameters, Read read, Emit emit, Barrier barrier) {
+ std::vector<Word> dma_parameters;
+ std::span<const Word> fifo = parameters;
+ std::size_t fifo_index{};
+ bool dma_fifo{};
+ std::array<Word, 24> regs{};
+ Word initialized{}, carry{}, method{}, increment{};
+ Word pc = start, loop_count{}, loop_start{}, loop_end{};
+ std::size_t consumed{};
+ bool end_next{}, stopped{};
+ std::string_view error;
+ const auto fail = [&](std::string_view message) { if (error.empty()) error = message; };
+ for (Word steps = 0; steps < 1000000; ++steps) {
+ if (static_cast<std::size_t>(pc) * 3 + 2 >= code.size()) {
+ fail("instruction outside uploaded RAM");
+ break;
+ }
+ const std::array<Word, 3> bits{code[pc * 3 + 2], code[pc * 3 + 1], code[pc * 3]};
+ const auto field = [&](Word low, Word count) {
+ std::uint64_t value = bits[low / 32];
+ if (low / 32 < 2) value |= std::uint64_t{bits[low / 32 + 1]} << 32;
+ return Word((value >> (low % 32)) & ((std::uint64_t{1} << count) - 1));
+ };
+ const std::array<Word, 2> imm{field(30, 16), field(66, 16)};
+ const std::array<Word, 2> op{field(10, 5), field(46, 5)};
+ const std::array<Word, 2> dst{field(15, 5), field(51, 5)};
+ const std::array<Word, 2> src_a{field(20, 5), field(56, 5)};
+ const std::array<Word, 2> src_b{field(25, 5), field(61, 5)};
+ const std::array<Word, 2> out_method{field(82, 3), field(89, 3)};
+ const std::array<Word, 2> out_emit{field(85, 4), field(92, 4)};
+ const Word pred_reg = field(5, 5), pred_mode = field(1, 4);
+ std::array<Word, 2> load{}, alu{};
+ for (Word i = 0; i < 2; ++i) {
+ const Word reg = 28 + i, out = 3 + i;
+ const bool needed = pred_reg == reg || src_a[0] == reg || src_a[1] == reg ||
+ src_b[0] == reg || src_b[1] == reg || out_method[0] == out ||
+ out_method[1] == out || out_emit[0] == out || out_emit[1] == out;
+ if (needed) {
+ if (fifo_index >= fifo.size()) { fail("parameter FIFO underflow"); break; }
+ load[i] = fifo[fifo_index++];
+ if (!dma_fifo) ++consumed;
+ }
+ }
+ // Both ALUs read the register file before either destination is
+ // committed. Carry chaining remains explicit between the lanes.
+ const auto input_regs = regs;
+ const Word input_initialized = initialized;
+ const auto get = [&](Word reg, Word lane) -> Word {
+ if (reg < 24) {
+ if (!(input_initialized & (1U << reg))) fail("uninitialized register read");
+ return input_regs[reg];
+ }
+ switch (reg) {
+ case 24: return 0;
+ case 25: return Word(std::int32_t(std::int16_t(imm[lane])));
+ case 26: return Word(std::int32_t(std::int16_t(imm[1 - lane])));
+ case 27: return (imm[0] << 16) | imm[1];
+ case 28: return load[0];
+ case 29: return load[1];
+ default: fail("invalid source register"); return 0;
+ }
+ };
+ constexpr std::array<std::string_view, 16> predicates{
+ "UUUU", "TTTT", "FFFF", "TTUU", "FFUU", "TFUU", "TUUU", "FUUU",
+ "UUTT", "UUTF", "UUTU", "UUFT", "UUFF", "UUFU", "UUUT", "UUUF"};
+ Word mask = 15;
+ if (pred_mode) {
+ const char skipped = get(pred_reg, 0) ? 'T' : 'F';
+ mask = 0;
+ for (Word i = 0; i < 4; ++i)
+ if (predicates[pred_mode][i] != skipped) mask |= 1U << i;
+ }
+ Word next_pc = pc + 1;
+ bool branch_cancels_end{};
+ for (Word lane = 0; lane < 2 && error.empty(); ++lane) {
+ if (!(mask & (1U << lane))) continue;
+ const Word x = get(src_a[lane], lane), y = get(src_b[lane], lane);
+ Word value{};
+ const auto condition = [&](Word comparison) {
+ switch (comparison) {
+ case 17: case 25: return std::int32_t(x) < std::int32_t(y);
+ case 18: case 26: return x < y;
+ case 19: case 27: return std::int32_t(x) <= std::int32_t(y);
+ case 20: case 28: return x <= y;
+ default: return x == y;
+ }
+ };
+ switch (op[lane]) {
+ case 0: value = x + y; carry = value < x; break;
+ case 1: value = x + y + carry; break;
+ case 2: value = x - y; carry = value > x; break;
+ case 3: value = x - y - carry; break;
+ case 4: case 6: {
+ const std::uint64_t a = op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(x))) : x;
+ const std::uint64_t b = op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(y))) : y;
+ const auto product = a * b;
+ value = Word(product); carry = Word(product >> 32); break;
+ }
+ case 5: value = carry; break;
+ case 7: {
+ if (x != 0x1000 || y != 1) {
+ fail("unsupported extended operation");
+ break;
+ }
+ auto data = barrier();
+ if (!data) { fail("macro DMA barrier failed"); break; }
+ if (!data->empty()) {
+ dma_parameters = std::move(*data);
+ fifo = dma_parameters;
+ fifo_index = 0;
+ dma_fifo = true;
+ }
+ break;
+ }
+ case 8: value = Word(std::countl_zero(x)); break;
+ case 9: value = x << (y & 31); break;
+ case 10: value = x >> (y & 31); break;
+ case 11: value = Word(std::int32_t(x) >> (y & 31)); break;
+ case 12: value = x & y; break;
+ case 13: value = ~(x & y); break;
+ case 14: value = x | y; break;
+ case 15: value = x ^ y; break;
+ case 16: {
+ const Word dest = (imm[lane] >> 10) & 63, count = (imm[lane] >> 5) & 31;
+ const Word source = imm[lane] & 31;
+ if (dest >= 32) { fail("invalid merge destination"); break; }
+ const Word bits_mask = (1U << count) - 1;
+ value = (x & ~(bits_mask << dest)) | (((y >> source) & bits_mask) << dest);
+ break;
+ }
+ case 17: case 18: case 19: case 20: case 21:
+ value = condition(op[lane]) ? ~0U : 0; break;
+ case 22: value = read((x + y) & 0xffff); break;
+ case 23:
+ if (loop_count || imm[lane] < 2) { fail("invalid loop"); break; }
+ // The native SDK emits zero-count loops around FIFO reads.
+ // Unlike Mesa's test simulator, these must skip the body.
+ if (x == 0) { next_pc = pc + imm[lane]; break; }
+ loop_count = x - 1; loop_start = pc; loop_end = pc + imm[lane] - 1;
+ break;
+ case 24: next_pc = pc + (imm[lane] & 0x7fff); break;
+ case 25: case 26: case 27: case 28: case 29:
+ if (condition(op[lane]) == bool(imm[lane] & 0x8000)) {
+ const auto offset = std::int32_t(imm[lane] << 19) >> 19;
+ if (offset == -4096) stopped = true;
+ else {
+ next_pc = Word(std::int32_t(pc) + offset);
+ // Native back edges use bit 14 to suppress the
+ // fallthrough termination when the branch is taken.
+ branch_cancels_end = (imm[lane] & 0x4000) != 0;
+ }
+ }
+ break;
+ case 30:
+ if (x >= dram.size()) { fail("data RAM read out of bounds"); break; }
+ value = dram[x]; break;
+ case 31:
+ if (x >= dram.size()) { fail("data RAM write out of bounds"); break; }
+ dram[x] = y; break;
+ }
+ alu[lane] = value;
+ if (dst[lane] < 24) { regs[dst[lane]] = value; initialized |= 1U << dst[lane]; }
+ else if (dst[lane] != 24) fail("invalid destination register");
+ }
+ const auto output = [&](Word selector) -> Word {
+ switch (selector) {
+ case 1: return alu[0]; case 2: return alu[1];
+ case 3: return load[0]; case 4: return load[1];
+ case 5: return imm[0]; case 6: return imm[1];
+ case 8: return imm[0] >> 12; case 9: return imm[1] >> 12;
+ case 10: return (imm[0] << 16) | imm[1];
+ default: fail("invalid output selector"); return 0;
+ }
+ };
+ for (Word lane = 0; lane < 2 && error.empty(); ++lane) {
+ if (!(mask & (4U << lane))) continue;
+ if (out_method[lane]) {
+ const Word address = output(out_method[lane]);
+ method = address & 0xfff; increment = (address >> 12) & 15;
+ }
+ if (out_emit[lane]) {
+ const Word value = output(out_emit[lane]);
+ if (error.empty() && !emit(method, value)) fail("unsupported emitted method");
+ method += increment;
+ }
+ }
+ if (!error.empty()) break;
+ if (end_next || stopped) return {true, pc, consumed, {}};
+ end_next = field(0, 1) != 0 && !branch_cancels_end;
+ if (loop_count && pc == loop_end) { --loop_count; next_pc = loop_start + 1; }
+ pc = next_pc;
+ }
+ if (error.empty()) error = "instruction limit exceeded";
+ return {false, pc, consumed, error};
+ }
+
+ std::array<Word, 0xc00> dram{};
+};
+
+} // namespace Tegra
diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp
index 8ffdcb6276..9fd3ca62e3 100644
--- a/src/video_core/memory_manager.cpp
+++ b/src/video_core/memory_manager.cpp
@@ -22,6 +22,11 @@
#include "video_core/texture_cache/util.h"
namespace Tegra {
+
+bool MemoryManager::IsShuttingDown() const {
+ return system.IsShuttingDown();
+}
+
using Tegra::Memory::GuestMemoryFlags;
std::atomic<size_t> MemoryManager::unique_identifier_generator{};
@@ -56,6 +61,10 @@ MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, GPU
: MemoryManager(system_, system_.Host1x().MemoryManager(), address_space_bits_, split_address_, big_page_bits_, page_bits_)
{}
+void MemoryManager::SetNativeSyncpointShim(GPUVAddr address) {
+ native_syncpoint_shim.store(address, std::memory_order_release);
+}
+
MemoryManager::~MemoryManager() = default;
MemoryManager::EntryType MemoryManager::GetEntry(size_t position, bool is_big_page) const {
@@ -231,6 +240,15 @@ std::optional<DAddr> MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t s
template <typename T>
T MemoryManager::Read(GPUVAddr addr) const {
+ if constexpr (sizeof(T) == sizeof(u32)) {
+ const auto shim = native_syncpoint_shim.load(std::memory_order_acquire);
+ if (shim && addr >= shim && addr - shim < 192 * sizeof(u32) &&
+ Common::IsAligned(addr - shim, sizeof(u32))) {
+ return static_cast<T>(system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(
+ static_cast<u32>((addr - shim) / sizeof(u32))));
+ }
+ }
+
if (auto page_pointer{GetPointer(addr)}; page_pointer) {
// NOTE: Avoid adding any extra logic to this fast-path block
T value;
diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h
index 08978567d2..79cffae1e4 100644
--- a/src/video_core/memory_manager.h
+++ b/src/video_core/memory_manager.h
@@ -43,6 +43,11 @@ public:
u64 big_page_bits_ = 16, u64 page_bits_ = 12);
~MemoryManager();
+ void SetNativeSyncpointShim(GPUVAddr address);
+ void EnableNativeTiling() { native_tiling = true; }
+ bool UsesNativeTiling() const { return native_tiling; }
+ bool IsShuttingDown() const;
+
static constexpr bool HAS_FLUSH_INVALIDATION = true;
inline size_t GetID() const noexcept {
@@ -178,6 +183,7 @@ private:
GPUVAddr gpu_addr, std::size_t size,
boost::container::small_vector<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result) const;
+ std::atomic<GPUVAddr> native_syncpoint_shim{};
Core::System& system;
MaxwellDeviceMemoryManager& memory;
@@ -220,6 +226,7 @@ private:
boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash{};
boost::container::small_vector<std::pair<DAddr, std::size_t>, 32> page_stash2{};
+ bool native_tiling{};
mutable std::mutex guard;
static constexpr size_t continuous_bits = 64;
diff --git a/src/video_core/native_qmd.h b/src/video_core/native_qmd.h
new file mode 100644
index 0000000000..c9ae3b8cc0
--- /dev/null
+++ b/src/video_core/native_qmd.h
@@ -0,0 +1,112 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+#include <array>
+#include <cstdint>
+#include <string_view>
+#include <type_traits>
+#include <unordered_map>
+
+namespace Tegra {
+
+// Native QMD 3.0 dependency scheduling; nonempty grids require a dispatch backend.
+class NativeQmdScheduler {
+public:
+ using Word = std::uint32_t;
+ using Address = std::uint64_t;
+ using Descriptor = std::array<Word, 64>;
+ struct RejectShaderDispatch { bool operator()(const Descriptor&) const { return false; } };
+ template <typename Read, typename Release>
+ std::string_view Signal(Address address, Word argument, Read read, Release release) {
+ return Signal(address, argument, read, release, RejectShaderDispatch{});
+ }
+ template <typename Read, typename Release, typename Dispatch>
+ std::string_view Signal(Address address, Word argument, Read read, Release release, Dispatch dispatch) {
+ struct Pending { Address address; Word action; Word count; };
+ std::array<Pending, 256> pending{};
+ std::size_t count = 1;
+ pending[0] = {address, argument & 15, ((argument >> 14) & 0x3ff) + 1};
+ for (std::size_t i = 0; i < count; ++i) {
+ const auto request = pending[i];
+ if (states.size() >= 4096 && !states.contains(request.address)) return "QMD cache limit";
+ auto& state = states[request.address];
+ if (request.action == 3 || request.action == 2) {
+ if (request.action == 3 || !state.loaded) {
+ if (!read(request.address, state.descriptor)) return "unmapped QMD";
+ const auto& qmd = state.descriptor;
+ if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue";
+ if constexpr (std::is_same_v<Dispatch, RejectShaderDispatch>) {
+ if (qmd[12] && (qmd[13] & 0xffff) && (qmd[14] & 0xffff))
+ return "native shader dispatch required";
+ }
+ state.dependencies = std::int32_t((qmd[4] >> 14) & 0xffff) + state.deferred;
+ state.deferred = 0;
+ state.loaded = true;
+ state.completed = false;
+ }
+ state.scheduled = true;
+ } else if (request.action == 7 || request.action == 12) {
+ const auto delta = request.action == 7 ? -std::int32_t(request.count) : std::int32_t(request.count);
+ if (!state.loaded || state.completed) { state.deferred += delta; continue; }
+ state.dependencies += delta;
+ } else if (request.action == 0 || request.action == 1) {
+ continue;
+ } else {
+ return "unsupported PCAS action";
+ }
+ if (!state.loaded || !state.scheduled || state.completed || state.dependencies > 0) continue;
+ if (state.dependencies < 0) return "QMD dependency underflow";
+ // The SDK patches release payloads while a scheduled QMD is
+ // waiting for its remaining dependencies. Fetch them at execution.
+ if (!read(request.address, state.descriptor)) return "unmapped QMD";
+ const auto qmd = state.descriptor;
+ if ((qmd[18] & 0xff) != 0x30 || (qmd[4] & 0x100)) return "unsupported QMD version or queue";
+ // Validate both release slots before making either visible.
+ for (Word slot = 0; slot < 2; ++slot) {
+ const Word flags = qmd[25 + slot * 4];
+ if (!(flags & (1U << 23))) continue;
+ const bool add = (flags & (1U << 26)) != 0;
+ if ((flags & (0x3bU << 24)) || (flags >> 30) != 1 ||
+ (add && (flags & (7U << 20)))) return "unsupported QMD release format";
+ if (add) {
+ if constexpr (!std::is_invocable_r_v<bool, Release, Address, Word, Word>)
+ return "unsupported QMD reduction callback";
+ }
+ }
+ if (qmd[12] && (qmd[13] & 0xffff) && (qmd[14] & 0xffff) && !dispatch(qmd))
+ return "native shader dispatch required";
+ state.completed = true;
+ for (Word slot = 0; slot < 2; ++slot) {
+ const Word flags = qmd[25 + slot * 4];
+ if (!(flags & (1U << 23))) continue;
+ const Address destination = (Address{flags & 0xff} << 32) | qmd[24 + slot * 4];
+ bool released{};
+ if constexpr (std::is_invocable_r_v<bool, Release, Address, Word, Word>) {
+ released = release(destination, qmd[26 + slot * 4], flags);
+ } else {
+ released = release(destination, qmd[26 + slot * 4]);
+ }
+ if (!released) return "unmapped QMD release";
+ }
+ if (qmd[16] & 1) {
+ const Word action = (qmd[16] >> 1) & 7;
+ if (action != 4 && action != 3 && action != 1) return "unsupported dependent QMD action";
+ if (count == pending.size()) return "QMD chain limit";
+ pending[count++] = {Address{qmd[15]} << 8, action == 4 ? 7U : action == 3 ? 3U : 2U, 1};
+ }
+ if (qmd[16] & (1U << 5)) return "second dependent QMD unsupported";
+ }
+ return {};
+ }
+private:
+ struct State {
+ Descriptor descriptor{};
+ std::int32_t dependencies{};
+ std::int32_t deferred{};
+ bool loaded{};
+ bool scheduled{};
+ bool completed{};
+ };
+ std::unordered_map<Address, State> states;
+};
+} // namespace Tegra
diff --git a/src/video_core/native_query_trace.h b/src/video_core/native_query_trace.h
new file mode 100644
index 0000000000..eae258dc04
--- /dev/null
+++ b/src/video_core/native_query_trace.h
@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <cstdlib>
+#include <fstream>
+#include <iomanip>
+#include <mutex>
+#include "common/common_types.h"
+
+namespace VideoCommon {
+namespace NativeQueryDiagnostics {
+struct State {
+ State() : file{"native-query-trace.csv", std::ios::trunc} {
+ file << "event,asid,gpu,cpu,type,value,flags\n";
+ }
+ std::mutex mutex;
+ std::ofstream file;
+ size_t records{};
+};
+inline State& GetState() {
+ static State state;
+ return state;
+}
+} // namespace NativeQueryDiagnostics
+
+// Events: 0=request, 1=written, 2=invalidated, 3=unsynchronized,
+// 4=channel suspended, 5=channel resumed, 6=buffer readback,
+// 7/8=QMD release slots observed, 9=PCAS request,
+// 10=DMA destination (value=source, flags=line bytes), 11=inline write.
+// Keep this opt-in and bounded.
+inline void TraceNativeQuery(u32 event, u64 asid, u64 gpu, u64 cpu, u32 type, u64 value, u32 flags) {
+ if (!std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) return;
+ auto& state = NativeQueryDiagnostics::GetState();
+ const std::scoped_lock lock{state.mutex};
+ if (state.records++ < 1000000 || event == 4 || event == 5) {
+ state.file << event << ',' << asid << ',' << std::hex << gpu << ',' << cpu << ','
+ << type << ',' << value << ',' << flags << std::dec << '\n';
+ }
+ if (event == 4 || event == 5) state.file.flush();
+}
+} // namespace VideoCommon
diff --git a/src/video_core/native_semaphore.h b/src/video_core/native_semaphore.h
new file mode 100644
index 0000000000..1ba7451c51
--- /dev/null
+++ b/src/video_core/native_semaphore.h
@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace Tegra {
+struct NativeSemaphoreWait {
+ std::uint64_t address;
+ std::uint32_t value;
+ std::uint32_t operation;
+
+ static constexpr bool Supported(std::uint32_t operation) {
+ return operation == 1 || operation == 4 || operation == 8;
+ }
+ constexpr bool Matches(std::uint32_t current) const {
+ switch (operation) {
+ case 1: return current == value;
+ case 4: return current >= value;
+ case 8: return (current & value) != 0;
+ default: return false;
+ }
+ }
+};
+
+// Revisit dependencies after another channel makes progress. A blocked
+// channel keeps its FIFO cursor until its real semaphore predicate is true.
+template <class Channels, class Ready, class Dispatch>
+std::size_t ResumeReadyNativeChannels(const Channels& channels, Ready ready, Dispatch dispatch) {
+ std::size_t resumed = 0;
+ for (std::size_t pass = 0; pass < channels.size(); ++pass) {
+ bool progress = false;
+ for (const auto& channel : channels) {
+ if (!ready(channel)) continue;
+ dispatch(channel);
+ ++resumed;
+ progress = true;
+ }
+ if (!progress) break;
+ }
+ return resumed;
+}
+} // namespace Tegra
diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h
index b8c1bc4f45..c2fefa6695 100644
--- a/src/video_core/query_cache/query_cache.h
+++ b/src/video_core/query_cache/query_cache.h
@@ -20,6 +20,7 @@
#include "common/settings.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/gpu.h"
+#include "video_core/native_query_trace.h"
#include "video_core/host1x/gpu_device_memory_manager.h"
#include "video_core/memory_manager.h"
#include "video_core/query_cache/bank_base.h"
@@ -246,6 +247,9 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type
return;
}
DAddr cpu_addr = *cpu_addr_opt;
+ const bool native_trace = gpu_memory->UsesNativeTiling();
+ const u64 trace_asid = gpu_memory->GetID();
+ if (native_trace) TraceNativeQuery(0, trace_asid, addr, cpu_addr, static_cast<u32>(counter_type), payload, static_cast<u32>(flags));
const size_t new_query_id = streamer->WriteCounter(cpu_addr, has_timestamp, payload, subreport);
auto* query = streamer->GetQuery(new_query_id);
if (is_fence) {
@@ -262,14 +266,16 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type
u8* pointer_timestamp = impl->device_memory.template GetPointer<u8>(cpu_addr + 8);
bool is_synced = (Settings::IsGPUFenceBehaviorDefault() ? !Settings::IsGPULevelHigh() : !Settings::IsGPUFenceBehaviorBalanced() && !Settings::IsGPUFenceBehaviorAccurate()) && is_fence;
std::function<void()> operation([this, is_synced, streamer, query_base = query, query_location,
- pointer, pointer_timestamp] {
+ pointer, pointer_timestamp, native_trace, trace_asid, addr, cpu_addr, counter_type] {
if (True(query_base->flags & QueryFlagBits::IsInvalidated)) {
+ if (native_trace) TraceNativeQuery(2, trace_asid, addr, cpu_addr, static_cast<u32>(counter_type), query_base->value, static_cast<u32>(query_base->flags));
if (!is_synced) [[likely]] {
impl->pending_unregister.push_back(query_location);
}
return;
}
if (False(query_base->flags & QueryFlagBits::IsFinalValueSynced)) [[unlikely]] {
+ if (native_trace) TraceNativeQuery(3, trace_asid, addr, cpu_addr, static_cast<u32>(counter_type), query_base->value, static_cast<u32>(query_base->flags));
LOG_ERROR(HW_GPU,
"Query report value not synchronized. Consider increasing GPU accuracy.");
if (!is_synced) [[likely]] {
@@ -287,6 +293,7 @@ void QueryCacheBase<Traits>::CounterReport(GPUVAddr addr, QueryType counter_type
u32 value = static_cast<u32>(query_base->value);
std::memcpy(pointer, &value, sizeof(value));
}
+ if (native_trace) TraceNativeQuery(1, trace_asid, addr, cpu_addr, static_cast<u32>(counter_type), query_base->value, static_cast<u32>(query_base->flags));
if (!is_synced) [[likely]] {
impl->pending_unregister.push_back(query_location);
}
diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp
index 9f4aaf390f..8a11c356ff 100644
--- a/src/video_core/renderer_vulkan/present/layer.cpp
+++ b/src/video_core/renderer_vulkan/present/layer.cpp
@@ -349,7 +349,7 @@ void Layer::UpdateRawImage(const Tegra::FramebufferConfig& framebuffer, size_t i
if (host_ptr) {
Tegra::Texture::UnswizzleTexture(
mapped_span.subspan(image_offset, linear_size), std::span(host_ptr, tiled_size),
- bytes_per_pixel, framebuffer.width, framebuffer.height, 1, block_height_log2, 0);
+ bytes_per_pixel, framebuffer.width, framebuffer.height, 1, block_height_log2, 0, 1, framebuffer.native_tiling);
buffer.Flush(); // Ensure host writes are visible before the GPU copy.
}
diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp
index ab58560170..bd092271e4 100644
--- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp
+++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp
@@ -633,7 +633,7 @@ struct alignas(16) BlockLinearUnswizzle3DPushConstants {
u32 block_depth; // Offset 48
u32 block_depth_mask; // Offset 52
- s32 _pad; // Offset 56
+ u32 native_tiling; // Offset 56
s32 destination[3]; // Offset 60
s32 _pad_end; // Offset 72
@@ -753,6 +753,7 @@ void BlockLinearUnswizzle3DPass::UnswizzleChunk(
pc.block_height_mask = params.block_height_mask;
pc.block_depth = params.block_depth;
pc.block_depth_mask = params.block_depth_mask;
+ pc.native_tiling = params.native_tiling;
pc.blocks_dim[0] = blocks_x;
pc.blocks_dim[1] = blocks_y;
diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
index d4ffcd8317..5e0975a231 100644
--- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
@@ -786,7 +786,7 @@ GraphicsPipeline* PipelineCache::BuiltPipeline(GraphicsPipeline* pipeline) const
if (pipeline->IsBuilt()) {
return pipeline;
}
- if (!use_asynchronous_shaders) {
+ if (maxwell3d->GetNativeMacroEngine().native_turing || !use_asynchronous_shaders) {
return pipeline;
}
// If something is using depth, we can assume that games are not rendering anything which
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index 264ffb28b5..740c988550 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -7,6 +7,8 @@
#include <algorithm>
#include <array>
#include <memory>
+#include <cstdlib>
+#include <fstream>
#include <mutex>
#include <fmt/format.h>
@@ -14,12 +16,14 @@
#include "video_core/renderer_vulkan/renderer_vulkan.h"
#include "common/assert.h"
+#include "shader_recompiler/exception.h"
#include "common/logging.h"
#include "common/scope_exit.h"
#include "common/settings.h"
#include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/gpu_logging/gpu_logging.h"
#include "video_core/control/channel_state.h"
+#include "video_core/native_query_trace.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/engines/kepler_compute.h"
#include "video_core/engines/maxwell_3d.h"
@@ -238,6 +242,14 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) {
SCOPE_EXIT {
gpu.TickWork();
};
+ bool submitted = false;
+ SCOPE_EXIT {
+ auto& native = maxwell3d->GetNativeMacroEngine();
+ if (native.native_turing && !submitted) {
+ native.native_execution_failed = true;
+ LOG_ERROR(HW_GPU, "Native graphics draw failed; refusing submission completion");
+ }
+ };
FlushWork();
gpu_memory->FlushCaching();
@@ -257,6 +269,45 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) {
HandleTransformFeedback();
query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable);
draw_func();
+ submitted = true;
+ static u32 native_capture_count{};
+ if (maxwell3d->GetNativeMacroEngine().native_turing &&
+ std::getenv("EDEN_NATIVE_DRAW_CAPTURE") && native_capture_count < 16) {
+ const u32 capture = native_capture_count++;
+ const auto prefix = fmt::format("native-draw-{}", capture);
+ const auto dump = [&](const std::string& name, GPUVAddr address, size_t size) {
+ if (!size || !gpu_memory->IsFullyMappedRange(address, size)) return;
+ std::vector<u8> bytes(size);
+ gpu_memory->ReadBlockUnsafe(address, bytes.data(), bytes.size());
+ std::ofstream file(prefix + name + ".bin", std::ios::binary);
+ file.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+ LOG_INFO(HW_GPU, "Native draw capture {}{}: address={:#x}, bytes={}, nonzero={}",
+ capture, name, address, size,
+ std::count_if(bytes.begin(), bytes.end(), [](u8 byte) { return byte != 0; }));
+ };
+ const auto& regs = maxwell3d->regs;
+ const auto& rt = regs.rt[0];
+ const size_t size = static_cast<size_t>(rt.array_pitch) * 4;
+ if (size && size <= 16 * 1024 * 1024) {
+ if (const auto address = gpu_memory->GpuToCpuAddress(rt.Address(), size)) {
+ LOG_INFO(HW_GPU, "Native draw {} render target physical address={:#x}", capture, *address);
+ texture_cache.DownloadMemory(*address, size);
+ dump("-rt0", rt.Address(), size);
+ }
+ }
+ std::ofstream registers(prefix + "-registers.bin", std::ios::binary);
+ registers.write(reinterpret_cast<const char*>(&regs), sizeof(regs));
+ for (size_t stream = 0; stream < 4; ++stream) {
+ if (regs.vertex_streams[stream].IsEnabled())
+ dump(fmt::format("-vertex-{}", stream), regs.vertex_streams[stream].Address(), 128);
+ }
+ for (size_t stage : {0U, 4U}) {
+ for (size_t bank : {2U, 3U}) {
+ const auto& cb = maxwell3d->state.shader_stages[stage].const_buffers[bank];
+ if (cb.enabled) dump(fmt::format("-cbuf-{}-{}", stage, bank), cb.address, std::min<size_t>(cb.size, 128));
+ }
+ }
+ }
}
void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) {
@@ -588,11 +639,13 @@ void RasterizerVulkan::DispatchCompute() {
ComputePipeline* const pipeline{pipeline_cache.CurrentComputePipeline()};
if (!pipeline) {
+ if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute pipeline unavailable");
return;
}
std::scoped_lock lock{texture_cache.mutex, buffer_cache.mutex};
if (!pipeline->Configure(*kepler_compute, *gpu_memory, scheduler, buffer_cache,
texture_cache)) {
+ if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute resources unavailable");
return;
}
@@ -942,6 +995,12 @@ Tegra::Engines::AccelerateDMAInterface& RasterizerVulkan::AccessAccelerateDMA()
void RasterizerVulkan::AccelerateInlineToMemory(GPUVAddr address, size_t copy_size,
std::span<const u8> memory) {
auto cpu_addr = gpu_memory->GpuToCpuAddress(address);
+ if (gpu_memory->UsesNativeTiling()) {
+ u32 first_word{};
+ if (memory.size() >= sizeof(first_word)) std::memcpy(&first_word, memory.data(), sizeof(first_word));
+ VideoCommon::TraceNativeQuery(11, gpu_memory->GetID(), address, cpu_addr.value_or(0),
+ 0, first_word, static_cast<u32>(copy_size));
+ }
if (!cpu_addr) [[unlikely]] {
gpu_memory->WriteBlock(address, memory.data(), copy_size);
return;
@@ -969,6 +1028,12 @@ std::optional<FramebufferTextureInfo> RasterizerVulkan::AccelerateDisplay(
std::scoped_lock lock{texture_cache.mutex};
const auto [image_view, scaled] =
texture_cache.TryFindFramebufferImageView(config, framebuffer_addr);
+ static u32 display_trace_count{};
+ if ((std::getenv("EDEN_NATIVE_FRAME_CAPTURE") || std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) && display_trace_count++ < 64) {
+ LOG_INFO(HW_GPU, "Native display source: address={:#x}, offset={:#x}, dimensions={}x{}, stride={}, accelerated={}, mask={:#x}",
+ framebuffer_addr, config.offset, config.width, config.height, pixel_stride,
+ image_view != nullptr, config.layer_stack_mask);
+ }
if (!image_view) {
return {};
}
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index 583cb6edaf..29afaac01b 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -2085,6 +2085,11 @@ void Image::DownloadMemory(VkBuffer buffer, size_t offset,
void Image::DownloadMemory(std::span<VkBuffer> buffers_span, std::span<size_t> offsets_span,
std::span<const VideoCommon::BufferImageCopy> copies) {
+ if (!original_image) {
+ LOG_CRITICAL(Render_Vulkan, "Missing image at download: gpu={:#x}, cpu={:#x}, type={}, format={}, flags={:#x}, size={}, dimensions={}x{}",
+ gpu_addr, cpu_addr, static_cast<u32>(info.type), static_cast<u32>(info.format),
+ static_cast<u32>(flags), unswizzled_size_bytes, info.size.width, info.size.height);
+ }
const bool is_rescaled = True(flags & ImageFlagBits::Rescaled);
if (is_rescaled) {
ScaleDown();
diff --git a/src/video_core/shader_cache.cpp b/src/video_core/shader_cache.cpp
index f1aff2810f..75e5432ab2 100644
--- a/src/video_core/shader_cache.cpp
+++ b/src/video_core/shader_cache.cpp
@@ -11,6 +11,7 @@
#include "common/assert.h"
#include "shader_recompiler/frontend/maxwell/control_flow.h"
+#include "shader_recompiler/exception.h"
#include "shader_recompiler/object_pool.h"
#include "video_core/control/channel_state.h"
#include "video_core/dirty_flags.h"
@@ -43,7 +44,7 @@ void ShaderCache::SyncGuestHost() {
ShaderCache::ShaderCache(Tegra::MaxwellDeviceMemoryManager& device_memory_)
: device_memory{device_memory_} {}
-bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) {
+bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try {
auto& dirty{maxwell3d->dirty.flags};
if (!dirty[VideoCommon::Dirty::Shaders]) {
return last_shaders_valid;
@@ -51,6 +52,7 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) {
dirty[VideoCommon::Dirty::Shaders] = false;
const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()};
+ const bool native = maxwell3d->GetNativeMacroEngine().native_turing;
for (size_t index = 0; index < Tegra::Engines::Maxwell3D::Regs::MaxShaderProgram; ++index) {
if (!maxwell3d->regs.IsShaderConfigEnabled(index)) {
unique_hashes[index] = 0;
@@ -63,7 +65,9 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) {
unique_hashes[index] = 0;
continue;
}
- const GPUVAddr shader_addr{base_addr + shader_config.offset};
+ const GPUVAddr shader_addr{native
+ ? (static_cast<u64>(shader_config.reserved[0] & 0xff) << 32) | shader_config.reserved[1]
+ : base_addr + shader_config.offset};
const std::optional<VAddr> cpu_shader_addr{gpu_memory->GpuToCpuAddress(shader_addr)};
if (!cpu_shader_addr) {
LOG_ERROR(HW_GPU, "Invalid GPU address for shader {:#016x}", shader_addr);
@@ -72,8 +76,8 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) {
}
const ShaderInfo* shader_info{TryGet(*cpu_shader_addr)};
if (!shader_info) {
- const u32 start_address{shader_config.offset};
- GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, base_addr, start_address};
+ const u32 start_address{native ? 0 : shader_config.offset};
+ GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, native ? shader_addr : base_addr, start_address};
shader_info = MakeShaderInfo(env, *cpu_shader_addr);
}
shader_infos[index] = shader_info;
@@ -81,6 +85,13 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) {
}
last_shaders_valid = true;
return true;
+} catch (const Shader::Exception& error) {
+ auto& native = maxwell3d->GetNativeMacroEngine();
+ if (!native.native_turing) throw;
+ native.native_execution_failed = true;
+ last_shaders_valid = false;
+ LOG_ERROR(HW_GPU, "Native graphics shader rejected: {}", error.what());
+ return false;
}
const ShaderInfo* ShaderCache::ComputeShader() {
@@ -103,14 +114,19 @@ void ShaderCache::GetGraphicsEnvironments(GraphicsEnvironments& result,
const std::array<u64, NUM_PROGRAMS>& unique_hashes) {
size_t env_index{};
const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()};
+ const bool native = maxwell3d->GetNativeMacroEngine().native_turing;
for (size_t index = 0; index < NUM_PROGRAMS; ++index) {
if (unique_hashes[index] == 0) {
continue;
}
const auto program{static_cast<Tegra::Engines::Maxwell3D::Regs::ShaderType>(index)};
auto& env{result.envs[index]};
- const u32 start_address{maxwell3d->regs.pipelines[index].offset};
- env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, base_addr, start_address};
+ const auto& config = maxwell3d->regs.pipelines[index];
+ const u32 start_address{native ? 0 : config.offset};
+ const GPUVAddr address = native
+ ? (static_cast<u64>(config.reserved[0] & 0xff) << 32) | config.reserved[1]
+ : base_addr;
+ env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, address, start_address};
env.SetCachedSize(shader_infos[index]->size_bytes);
result.env_ptrs[env_index++] = &env;
}
@@ -241,7 +257,7 @@ const ShaderInfo* ShaderCache::MakeShaderInfo(GenericEnvironment& env, VAddr cpu
info->unique_hash = env.CalculateHash();
info->size_bytes = env.ReadSizeBytes();
}
- const size_t size_bytes{info->size_bytes};
+ const size_t size_bytes{env.SourceSizeBytes() ? env.SourceSizeBytes() : info->size_bytes};
const ShaderInfo* const result{info.get()};
Register(std::move(info), cpu_addr, size_bytes);
return result;
diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp
index 4d5db7e511..4a7eff3008 100644
--- a/src/video_core/shader_environment.cpp
+++ b/src/video_core/shader_environment.cpp
@@ -5,7 +5,11 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
+#include <chrono>
+#include <cstdlib>
+#include <thread>
#include <filesystem>
+#include <cstring>
#include <fstream>
#include <memory>
#include <optional>
@@ -21,6 +25,11 @@
#include "common/settings.h"
#include <ranges>
#include "shader_recompiler/environment.h"
+#include "shader_recompiler/exception.h"
+#include "shader_recompiler/frontend/ampere/lower_to_maxwell.h"
+#include "shader_recompiler/frontend/ampere/program_override.h"
+#include "shader_recompiler/frontend/ampere/program_capture.h"
+#include "shader_recompiler/frontend/ampere/program_address_map.h"
#include "video_core/engines/kepler_compute.h"
#include "video_core/memory_manager.h"
#include "video_core/shader_environment.h"
@@ -149,11 +158,26 @@ u64 GenericEnvironment::ReadInstruction(u32 address) {
if (address >= cached_lowest && address < cached_highest) {
return code[(address - cached_lowest) / INST_SIZE];
}
+ if (native_source_size) {
+ throw Shader::RuntimeError("Lowered native instruction address out of range: {:#x}", address);
+ }
has_unbound_instructions = true;
return gpu_memory->Read<u64>(program_base + address);
}
+u32 GenericEnvironment::TranslateProgramAddress(u32 address) const {
+ if (const auto translated = Shader::Ampere::NativeProgramAddress(code, address)) return *translated;
+ if (native_source_size || Shader::Ampere::HasNativeAddressMap(code))
+ throw Shader::RuntimeError("Invalid native indirect branch address {:#x}", address);
+ return address;
+}
+
std::optional<u64> GenericEnvironment::Analyze() {
+ if (native_source_size) {
+ // Include the converted header and a version tag to isolate this experimental ABI.
+ return Common::CityHash64(reinterpret_cast<const char*>(code.data()),
+ code.size() * sizeof(u64)) ^ 0x4f554e4345000002ULL;
+ }
const std::optional<u64> size{TryFindSize()};
if (!size) {
return std::nullopt;
@@ -164,6 +188,12 @@ std::optional<u64> GenericEnvironment::Analyze() {
}
void GenericEnvironment::SetCachedSize(size_t size_bytes) {
+ if (native_source_size) {
+ if (size_bytes != CachedSizeBytes()) {
+ throw Shader::RuntimeError("Native shader changed while constructing pipeline");
+ }
+ return;
+ }
cached_lowest = start_address;
cached_highest = start_address + static_cast<u32>(size_bytes);
code.resize(CachedSizeWords());
@@ -291,6 +321,119 @@ Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr,
return entry;
}
+void GenericEnvironment::InitializeNativeCode(u32 source_header_size) {
+ using Shader::Ampere::Instruction;
+ const u32 native_header_size = source_header_size;
+ constexpr u32 maximum_instructions = Shader::Ampere::ProgramCaptureState::MaximumInstructions;
+ std::vector<Instruction> native;
+ Shader::Ampere::ProgramCaptureState capture;
+ for (u32 index = 0; index < maximum_instructions; ++index) {
+ const GPUVAddr address = program_base + native_header_size + index * sizeof(Instruction);
+ if (!gpu_memory->IsFullyMappedRange(address, sizeof(Instruction))) {
+ if (const char* directory = std::getenv("EDEN_NATIVE_SHADER_CAPTURE_DIR")) {
+ std::filesystem::create_directories(directory);
+ std::ofstream dump(std::filesystem::path{directory} /
+ fmt::format("{:016x}.capture.bin", program_base), std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native.data()), native.size() * sizeof(Instruction));
+ }
+ throw Shader::RuntimeError("Unmapped native shader instruction at {:#x} (program={:#x}, index={})",
+ address, program_base, index);
+ }
+ auto& instruction = native.emplace_back();
+ gpu_memory->ReadBlock(address, &instruction, sizeof(instruction));
+ const auto status = capture.Observe(instruction, index);
+ if (status == Shader::Ampere::ProgramCaptureState::Result::InvalidTarget)
+ throw Shader::RuntimeError("Native shader branch outside capture bounds");
+ if (status == Shader::Ampere::ProgramCaptureState::Result::Complete) break;
+ }
+ auto lowered = Shader::Ampere::LowerToMaxwell(native);
+ if (!lowered) {
+ if (const char* directory = std::getenv("EDEN_NATIVE_SHADER_CAPTURE_DIR")) {
+ std::filesystem::create_directories(directory);
+ std::ofstream dump(std::filesystem::path{directory} /
+ fmt::format("{:016x}.capture.bin", program_base), std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native.data()), native.size() * sizeof(Instruction));
+ }
+ }
+ if (!lowered) {
+ if (const char* directory = std::getenv("EDEN_NATIVE_SHADER_OVERRIDE_DIR")) {
+ const auto key = fmt::format("{:016x}", Shader::Ampere::NativeProgramKey(native));
+ const auto folder = std::filesystem::path{directory};
+ std::filesystem::create_directories(folder);
+ const auto source_path = folder / (key + ".source.bin");
+ const auto override_path = folder / (key + ".override.bin");
+ {
+ std::ofstream dump(source_path, std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(native.data()), native.size() * sizeof(Shader::Ampere::Instruction));
+ }
+ {
+ // Retain mapped bytes beyond the first EXIT for investigating
+ // native CALL targets in compiler library subroutines.
+ std::ofstream mapped(folder / (key + ".mapped.bin"), std::ios::binary);
+ std::array<u8, 0x1000> block{};
+ for (u32 offset = 0; offset < 0x10000; offset += 0x1000) {
+ if (!gpu_memory->IsFullyMappedRange(program_base + offset, block.size())) break;
+ gpu_memory->ReadBlock(program_base + offset, block.data(), block.size());
+ mapped.write(reinterpret_cast<const char*>(block.data()), block.size());
+ }
+ }
+ LOG_WARNING(Shader, "Native shader paused: key={}, offset={:#x}, reason={}, source={}",
+ key, lowered.error_offset, lowered.error, source_path.string());
+ const auto deadline = std::chrono::steady_clock::now() + std::chrono::minutes{30};
+ while (!gpu_memory->IsShuttingDown() && std::chrono::steady_clock::now() < deadline) {
+ if (std::filesystem::exists(override_path)) {
+ std::ifstream file(override_path, std::ios::binary | std::ios::ate);
+ const auto length = file.tellg();
+ if (length <= 0 || length > 16 * 1024 * 1024) {
+ throw Shader::RuntimeError("Invalid native shader override size");
+ }
+ std::vector<u8> bytes(static_cast<size_t>(length));
+ file.seekg(0);
+ file.read(reinterpret_cast<char*>(bytes.data()), bytes.size());
+ if (!file) throw Shader::RuntimeError("Native shader override read failed");
+ const u32 source_bytes = Shader::Ampere::ProgramOverrideSourceSize(bytes);
+ if (source_bytes > native.size() * sizeof(Shader::Ampere::Instruction)) {
+ if (!gpu_memory->IsFullyMappedRange(program_base + native_header_size, source_bytes))
+ throw Shader::RuntimeError("Unmapped extended native shader source");
+ native.resize(source_bytes / sizeof(Shader::Ampere::Instruction));
+ gpu_memory->ReadBlock(program_base + native_header_size, native.data(), source_bytes);
+ }
+ lowered = Shader::Ampere::DecodeProgramOverride(bytes, native);
+ if (!lowered) throw Shader::RuntimeError("Native shader override rejected: {}", lowered.error);
+ LOG_WARNING(Shader, "Native shader resumed: key={}, instructions={}", key, lowered.instructions.size());
+ break;
+ }
+ std::this_thread::sleep_for(std::chrono::milliseconds{100});
+ }
+ }
+ }
+ if (!lowered) {
+ throw Shader::RuntimeError("Native shader at {:#x}, instruction {:#x}: {}",
+ program_base, lowered.error_offset, lowered.error);
+ }
+ if (lowered.required_texture_buffer != ~0U) {
+ // Native TEX carries its cbuf index explicitly; TEXS obtains it from the environment.
+ texture_bound = lowered.required_texture_buffer;
+ }
+ native_source_size = native_header_size + lowered.source_size;
+ code.resize(sizeof(sph) / sizeof(u64));
+ std::memcpy(code.data(), &sph, sizeof(sph));
+ for (const u64 instruction : lowered.instructions) {
+ if ((code.size() * sizeof(u64)) % 32 == 0) {
+ code.push_back(0); // Maxwell scheduling word, absent from native instructions.
+ }
+ code.push_back(instruction);
+ }
+ // Keep the original padding ABI, then retain the native address map as
+ // unreachable data. Existing disk-cache serialization carries it with code.
+ code.push_back(0);
+ Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets);
+ cached_lowest = 0;
+ cached_highest = static_cast<u32>((code.size() - 1) * sizeof(u64));
+ LOG_INFO(Shader, "Lowered native shader at {:#x}: {} source bytes, {} instructions",
+ program_base, native_source_size, lowered.instructions.size());
+}
+
GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_,
Tegra::MemoryManager& gpu_memory_,
Maxwell::ShaderType program, GPUVAddr program_base_,
@@ -298,6 +441,23 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_,
: GenericEnvironment{gpu_memory_, program_base_, start_address_}, maxwell3d{&maxwell3d_} {
gpu_memory->ReadBlock(program_base + start_address, &sph, sizeof(sph));
initial_offset = sizeof(sph);
+ texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot;
+ if (maxwell3d->GetNativeMacroEngine().native_turing) {
+ if (program != Maxwell::ShaderType::VertexB && program != Maxwell::ShaderType::Pixel) {
+ throw Shader::RuntimeError("Unsupported native graphics shader stage {}", static_cast<u32>(program));
+ }
+ if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) {
+ const u32 native_stage = program == Maxwell::ShaderType::Pixel ? 4 : 0;
+ const auto& table = maxwell3d->state.shader_stages[native_stage].const_buffers[1];
+ if (table.enabled && table.size && table.size <= Tegra::Engines::Maxwell3D::Regs::MaxConstBufferSize) {
+ std::vector<u8> bytes(table.size);
+ gpu_memory->ReadBlock(table.address, bytes.data(), bytes.size());
+ std::ofstream dump(fmt::format("native-graphics-cbuf-{}-1.bin", static_cast<u32>(program)), std::ios::binary);
+ dump.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+ }
+ }
+ InitializeNativeCode();
+ }
gp_passthrough_mask = maxwell3d->regs.post_vtg_shader_attrib_skip_mask;
switch (program) {
case Maxwell::ShaderType::VertexA:
@@ -331,8 +491,7 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_,
const u64 local_size{sph.LocalMemorySize()};
ASSERT(local_size <= (std::numeric_limits<u32>::max)());
local_memory_size = static_cast<u32>(local_size) + sph.common3.shader_local_memory_crs_size;
- texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot;
- is_proprietary_driver = texture_bound == 2;
+ is_proprietary_driver = maxwell3d->GetNativeMacroEngine().native_turing || texture_bound == 2;
has_hle_engine_state =
maxwell3d->engine_state == Tegra::Engines::Maxwell3D::EngineHint::OnHLEMacro;
}
@@ -417,6 +576,12 @@ ComputeEnvironment::ComputeEnvironment(Tegra::Engines::KeplerCompute& kepler_com
is_proprietary_driver = texture_bound == 2;
shared_memory_size = qmd.shared_alloc;
workgroup_size = {qmd.block_dim_x, qmd.block_dim_y, qmd.block_dim_z};
+ if (kepler_compute->IsNativeMode()) {
+ initial_offset = sizeof(sph);
+ is_proprietary_driver = true;
+ InitializeNativeCode(0);
+ start_address = sizeof(sph);
+ }
}
u32 ComputeEnvironment::ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) {
@@ -511,7 +676,7 @@ void FileEnvironment::Deserialize(std::ifstream& file) {
if (stage == Shader::Stage::Compute) {
file.read(reinterpret_cast<char*>(&workgroup_size), sizeof(workgroup_size))
.read(reinterpret_cast<char*>(&shared_memory_size), sizeof(shared_memory_size));
- initial_offset = 0;
+ initial_offset = Shader::Ampere::HasNativeAddressMap(code) ? sizeof(sph) : 0;
} else {
file.read(reinterpret_cast<char*>(&sph), sizeof(sph));
initial_offset = sizeof(sph);
@@ -526,6 +691,13 @@ void FileEnvironment::Dump(u64 pipeline_hash, u64 shader_hash) {
DumpImpl(pipeline_hash, shader_hash, code, read_highest, read_lowest, initial_offset, stage);
}
+u32 FileEnvironment::TranslateProgramAddress(u32 address) const {
+ if (const auto translated = Shader::Ampere::NativeProgramAddress(code, address)) return *translated;
+ if (Shader::Ampere::HasNativeAddressMap(code))
+ throw Shader::RuntimeError("Invalid cached native indirect branch address {:#x}", address);
+ return address;
+}
+
u64 FileEnvironment::ReadInstruction(u32 address) {
if (address < read_lowest || address > read_highest) {
throw Shader::LogicError("Out of bounds address {}", address);
diff --git a/src/video_core/shader_environment.h b/src/video_core/shader_environment.h
index 924e61ef46..e49f01d95f 100644
--- a/src/video_core/shader_environment.h
+++ b/src/video_core/shader_environment.h
@@ -47,6 +47,8 @@ public:
[[nodiscard]] u64 ReadInstruction(u32 address) final;
+ [[nodiscard]] u32 TranslateProgramAddress(u32 address) const override;
+
[[nodiscard]] std::optional<u64> Analyze();
void SetCachedSize(size_t size_bytes);
@@ -57,6 +59,10 @@ public:
[[nodiscard]] size_t ReadSizeBytes() const noexcept;
+ [[nodiscard]] size_t SourceSizeBytes() const noexcept {
+ return native_source_size;
+ }
+
[[nodiscard]] bool CanBeSerialized() const noexcept;
[[nodiscard]] u64 CalculateHash() const;
@@ -72,6 +78,9 @@ public:
protected:
std::optional<u64> TryFindSize();
+ void InitializeNativeCode(u32 source_header_size = 128);
+ size_t native_source_size{};
+
Tegra::Texture::TICEntry ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit,
bool via_header_index, u32 raw);
@@ -172,6 +181,8 @@ public:
[[nodiscard]] u64 ReadInstruction(u32 address) override;
+ [[nodiscard]] u32 TranslateProgramAddress(u32 address) const override;
+
[[nodiscard]] u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) override;
[[nodiscard]] Shader::TextureType ReadTextureType(u32 handle) override;
diff --git a/src/video_core/texture_cache/accelerated_swizzle.cpp b/src/video_core/texture_cache/accelerated_swizzle.cpp
index 4c3f724d79..8ae0ed218d 100644
--- a/src/video_core/texture_cache/accelerated_swizzle.cpp
+++ b/src/video_core/texture_cache/accelerated_swizzle.cpp
@@ -37,6 +37,7 @@ BlockLinearSwizzle2DParams MakeBlockLinearSwizzle2DParams(const SwizzleParameter
.x_shift = GOB_SIZE_SHIFT + block.height + block.depth,
.block_height = block.height,
.block_height_mask = (1U << block.height) - 1,
+ .native_tiling = info.native_tiling,
};
}
@@ -63,6 +64,7 @@ BlockLinearSwizzle3DParams MakeBlockLinearSwizzle3DParams(const SwizzleParameter
.block_height_mask = (1U << block.height) - 1,
.block_depth = block.depth,
.block_depth_mask = (1U << block.depth) - 1,
+ .native_tiling = info.native_tiling,
};
}
diff --git a/src/video_core/texture_cache/accelerated_swizzle.h b/src/video_core/texture_cache/accelerated_swizzle.h
index d4250da689..8296950c38 100644
--- a/src/video_core/texture_cache/accelerated_swizzle.h
+++ b/src/video_core/texture_cache/accelerated_swizzle.h
@@ -20,6 +20,7 @@ struct BlockLinearSwizzle2DParams {
u32 x_shift;
u32 block_height;
u32 block_height_mask;
+ u32 native_tiling;
};
struct BlockLinearSwizzle3DParams {
@@ -33,6 +34,7 @@ struct BlockLinearSwizzle3DParams {
u32 block_height_mask;
u32 block_depth;
u32 block_depth_mask;
+ u32 native_tiling;
};
[[nodiscard]] BlockLinearSwizzle2DParams MakeBlockLinearSwizzle2DParams(
diff --git a/src/video_core/texture_cache/format_lookup_table.cpp b/src/video_core/texture_cache/format_lookup_table.cpp
index d6e019cddf..20bb96aae3 100644
--- a/src/video_core/texture_cache/format_lookup_table.cpp
+++ b/src/video_core/texture_cache/format_lookup_table.cpp
@@ -157,6 +157,7 @@ PixelFormat PixelFormatFromTextureInfo(TextureFormat format, ComponentType red,
return PixelFormat::S8_UINT_D24_UNORM;
case Hash(TextureFormat::G24R8, UINT, UNORM, UNORM, UNORM, LINEAR):
return PixelFormat::S8_UINT_D24_UNORM;
+ case Hash(TextureFormat::S8Z24, UNORM, UINT, UNORM, UNORM, LINEAR):
case Hash(TextureFormat::S8Z24, UNORM, UINT, UINT, UINT, LINEAR):
return PixelFormat::D24_UNORM_S8_UINT;
case Hash(TextureFormat::Z32_X24S8, FLOAT, UINT, UNORM, UNORM, LINEAR):
diff --git a/src/video_core/texture_cache/image_info.h b/src/video_core/texture_cache/image_info.h
index eb490a6424..7407062d19 100644
--- a/src/video_core/texture_cache/image_info.h
+++ b/src/video_core/texture_cache/image_info.h
@@ -14,6 +14,17 @@ namespace VideoCommon {
using Tegra::Texture::TICEntry;
using VideoCore::Surface::PixelFormat;
+// Native descriptors address the selected view directly. The legacy NVN
+// layer-base bookkeeping is not an additional array prefix for this ABI.
+inline TICEntry TextureViewHeader(TICEntry entry, bool native) {
+ if (native) {
+ entry.layer_base_0_2.Assign(0);
+ entry.layer_base_3_7.Assign(0);
+ entry.layer_base_8_10.Assign(0);
+ }
+ return entry;
+}
+
struct ImageInfo {
ImageInfo() = default;
explicit ImageInfo(const TICEntry& config) noexcept;
@@ -37,6 +48,7 @@ struct ImageInfo {
u32 maybe_unaligned_layer_stride = 0;
u32 num_samples = 1;
u32 tile_width_spacing = 0;
+ bool native_tiling = false;
bool rescaleable = false;
bool downscaleable = false;
bool forced_flushed = false;
diff --git a/src/video_core/texture_cache/pending_download.h b/src/video_core/texture_cache/pending_download.h
new file mode 100644
index 0000000000..52a48e82d3
--- /dev/null
+++ b/src/video_core/texture_cache/pending_download.h
@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+#pragma once
+
+#include <algorithm>
+#include <memory>
+#include <unordered_set>
+#include <cstddef>
+#include <utility>
+#include "common/slot_vector.h"
+
+namespace VideoCommon {
+
+template <class Image>
+struct PendingImageDownload {
+ bool is_swizzle;
+ size_t async_buffer_id;
+ Common::SlotId object_id;
+ std::shared_ptr<Image> retired_image{};
+
+ Image& GetImage(Common::SlotVector<Image>& images) const {
+ return retired_image ? *retired_image : images[object_id];
+ }
+};
+
+// Whole-image copies are recorded together at commit time. Repeated requests
+// for the same live image therefore observe the same contents. Keep the first
+// occurrence so reverse CPU writeback order remains unchanged for aliases.
+template <class Image, class Downloads>
+size_t DeduplicatePendingImageDownloads(Downloads& downloads, Common::SlotVector<Image>& images) {
+ std::unordered_set<const Image*> seen;
+ const size_t previous_size = downloads.size();
+ const auto end = std::remove_if(downloads.begin(), downloads.end(), [&](const auto& download) {
+ return download.is_swizzle && !seen.insert(&download.GetImage(images)).second;
+ });
+ downloads.erase(end, downloads.end());
+ return previous_size - downloads.size();
+}
+
+// Cache slots may be recycled before queued GPU downloads are submitted or
+// consumed. Keep the actual image, including its metadata, until those finish.
+template <class Image, class Downloads, class Batches>
+bool RetainPendingImageDownloads(Common::SlotId id, Image& image, Downloads& pending,
+ Batches& submitted) {
+ std::shared_ptr<Image> retained;
+ const auto retain = [&](auto& downloads) {
+ for (auto& download : downloads) {
+ if (!download.is_swizzle || download.object_id != id || download.retired_image) continue;
+ if (!retained) retained = std::make_shared<Image>(std::move(image));
+ download.retired_image = retained;
+ }
+ };
+ retain(pending);
+ for (auto& downloads : submitted) retain(downloads);
+ return retained != nullptr;
+}
+
+} // namespace VideoCommon
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index e92e54f8ce..905f73999b 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -803,6 +803,17 @@ std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImage
if (it == page_table.end()) {
return {};
}
+ const auto view_format = [&]() {
+ switch (config.pixel_format) {
+ case Service::android::PixelFormat::Rgb565:
+ return PixelFormat::R5G6B5_UNORM;
+ case Service::android::PixelFormat::Bgra8888:
+ return PixelFormat::B8G8R8A8_UNORM;
+ default:
+ return PixelFormat::A8B8G8R8_UNORM;
+ }
+ }();
+
const auto& image_map_ids = it->second;
boost::container::small_vector<ImageId, 4> valid_image_ids;
for (const ImageMapId map_id : image_map_ids) {
@@ -814,20 +825,17 @@ std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImage
if (image.image_view_ids.empty()) {
continue;
}
+ if (!VideoCore::Surface::IsViewCompatible(image.info.format, view_format, false, true)) {
+ static u32 rejected_alias_count{};
+ if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && rejected_alias_count++ < 8) {
+ LOG_WARNING(HW_GPU, "Ignoring incompatible framebuffer alias: address={:#x}, image={}, view={}",
+ cpu_addr, image.info.format, view_format);
+ }
+ continue;
+ }
valid_image_ids.push_back(map.image_id);
}
- const auto view_format = [&]() {
- switch (config.pixel_format) {
- case Service::android::PixelFormat::Rgb565:
- return PixelFormat::R5G6B5_UNORM;
- case Service::android::PixelFormat::Bgra8888:
- return PixelFormat::B8G8R8A8_UNORM;
- default:
- return PixelFormat::A8B8G8R8_UNORM;
- }
- }();
-
const auto GetImageViewForFramebuffer = [&](ImageId image_id) {
ImageViewInfo info{ImageViewType::e2D, view_format};
if (config.blending == Tegra::BlendMode::Opaque) {
@@ -869,6 +877,7 @@ void TextureCache<P>::CommitAsyncFlushes() {
// This is intentionally passing the value by copy
if constexpr (IMPLEMENTS_ASYNC_DOWNLOADS) {
auto& download_ids = uncommitted_downloads;
+ const size_t redundant_images = DeduplicatePendingImageDownloads(download_ids, slot_images);
if (download_ids.empty()) {
committed_downloads.emplace_back(std::move(uncommitted_downloads));
uncommitted_downloads.clear();
@@ -882,17 +891,32 @@ void TextureCache<P>::CommitAsyncFlushes() {
for (PendingDownload& download_info : download_ids) {
if (download_info.is_swizzle) {
total_size_bytes +=
- Common::AlignUp(slot_images[download_info.object_id].unswizzled_size_bytes, 64);
+ Common::AlignUp(download_info.GetImage(slot_images).unswizzled_size_bytes, 64);
any_none_dma = true;
download_info.async_buffer_id = last_async_buffer_id;
}
}
if (any_none_dma) {
+ if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS") && total_size_bytes >= 64_MiB) {
+ LOG_WARNING(HW_GPU, "Large image readback batch: bytes={}, entries={}, duplicates_removed={}, submitted_batches={}, retained_buffers={}",
+ total_size_bytes, download_ids.size(), redundant_images,
+ committed_downloads.size(), async_buffers_death_ring.size());
+ size_t traced_images = 0;
+ for (const auto& download : download_ids) {
+ if (!download.is_swizzle || traced_images++ >= 8) continue;
+ const auto& image = download.GetImage(slot_images);
+ LOG_WARNING(HW_GPU, "Readback image: gpu={:#x}, cpu={:#x}, format={}, size={}x{}x{}, levels={}, layers={}, layer_stride={}, guest_bytes={}, download_bytes={}",
+ image.gpu_addr, image.cpu_addr, image.info.format,
+ image.info.size.width, image.info.size.height, image.info.size.depth,
+ image.info.resources.levels, image.info.resources.layers,
+ image.info.layer_stride, image.guest_size_bytes, image.unswizzled_size_bytes);
+ }
+ }
auto download_map = runtime.DownloadStagingBuffer(total_size_bytes, true);
for (const PendingDownload& download_info : download_ids) {
if (download_info.is_swizzle) {
- Image& image = slot_images[download_info.object_id];
+ Image& image = download_info.GetImage(slot_images);
const auto copies = FullDownloadCopies(image.info);
image.DownloadMemory(download_map, copies);
download_map.offset += Common::AlignUp(image.unswizzled_size_bytes, 64);
@@ -925,7 +949,7 @@ void TextureCache<P>::PopAsyncFlushes() {
auto& download_info = download_ids[i - 1];
auto& download_buffer = download_map[download_info.async_buffer_id];
if (download_info.is_swizzle) {
- const ImageBase& image = slot_images[download_info.object_id];
+ const ImageBase& image = download_info.GetImage(slot_images);
const auto copies = FullDownloadCopies(image.info);
download_buffer.offset -= Common::AlignUp(image.unswizzled_size_bytes, 64);
std::span<u8> download_span =
@@ -954,7 +978,7 @@ void TextureCache<P>::PopAsyncFlushes() {
size_t total_size_bytes = 0;
for (const PendingDownload& download_info : download_ids) {
if (download_info.is_swizzle) {
- total_size_bytes += slot_images[download_info.object_id].unswizzled_size_bytes;
+ total_size_bytes += download_info.GetImage(slot_images).unswizzled_size_bytes;
}
}
auto download_map = runtime.DownloadStagingBuffer(total_size_bytes);
@@ -963,7 +987,7 @@ void TextureCache<P>::PopAsyncFlushes() {
if (!download_info.is_swizzle) {
continue;
}
- Image& image = slot_images[download_info.object_id];
+ Image& image = download_info.GetImage(slot_images);
const auto copies = FullDownloadCopies(image.info);
image.DownloadMemory(download_map, copies);
download_map.offset += image.unswizzled_size_bytes;
@@ -976,7 +1000,7 @@ void TextureCache<P>::PopAsyncFlushes() {
if (!download_info.is_swizzle) {
continue;
}
- const ImageBase& image = slot_images[download_info.object_id];
+ const ImageBase& image = download_info.GetImage(slot_images);
const auto copies = FullDownloadCopies(image.info);
SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, download_span, swizzle_data_buffer);
download_map.offset += image.unswizzled_size_bytes;
@@ -1178,7 +1202,18 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging)
}
template <class P>
-ImageViewId TextureCache<P>::CreateImageView(const TICEntry& config) {
+ImageViewId TextureCache<P>::CreateImageView(const TICEntry& raw_config) {
+ const bool native = gpu_memory->UsesNativeTiling();
+ if (raw_config.BaseLayer() && std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) {
+ static u32 texture_view_traces{};
+ if (texture_view_traces++ < 8) {
+ LOG_WARNING(HW_GPU, "Native texture view header: native={}, graphics_native={}, address={:#x}, dimensions={}x{}x{}, type={}, legacy_layer={}, raw={:016x}/{:016x}/{:016x}/{:016x}",
+ native, maxwell3d->GetNativeMacroEngine().native_turing, raw_config.Address(), raw_config.Width(), raw_config.Height(), raw_config.Depth(),
+ static_cast<u32>(raw_config.texture_type.Value()), raw_config.BaseLayer(),
+ raw_config.raw[0], raw_config.raw[1], raw_config.raw[2], raw_config.raw[3]);
+ }
+ }
+ const TICEntry config = TextureViewHeader(raw_config, native);
const ImageInfo info(config);
if (info.type == ImageType::Buffer) {
const ImageViewInfo view_info(config, 0);
@@ -1526,7 +1561,9 @@ ImageId TextureCache<P>::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr,
}
}
ASSERT_MSG(cpu_addr, "Tried to insert an image to an invalid gpu_addr={:#x}", gpu_addr);
- const ImageId image_id = JoinImages(info, gpu_addr, *cpu_addr);
+ ImageInfo storage_info = info;
+ storage_info.native_tiling = gpu_memory->UsesNativeTiling();
+ const ImageId image_id = JoinImages(storage_info, gpu_addr, *cpu_addr);
const Image& image = slot_images[image_id];
// Using "image.gpu_addr" instead of "gpu_addr" is important because it might be different
const auto [it, is_new] = image_allocs_table.try_emplace(image.gpu_addr);
@@ -2385,7 +2422,11 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) {
}
slot_image_views.erase(image_view_id);
}
- if (!immediate_delete) {
+ const bool retained = RetainPendingImageDownloads(image_id, slot_images[image_id],
+ uncommitted_downloads, committed_downloads);
+ if (retained) {
+ LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value());
+ } else if (!immediate_delete) {
sentenced_images.Push(std::move(slot_images[image_id]));
}
slot_images.erase(image_id);
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index 8a2318cc72..d103d66189 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -28,6 +28,7 @@
#include <ranges>
#include "common/scratch_buffer.h"
#include "common/slot_vector.h"
+#include "video_core/texture_cache/pending_download.h"
#include "common/thread_worker.h"
#include "video_core/compatible_formats.h"
#include "video_core/control/channel_state_cache.h"
@@ -463,11 +464,7 @@ private:
size_t size;
};
- struct PendingDownload {
- bool is_swizzle;
- size_t async_buffer_id;
- Common::SlotId object_id;
- };
+ using PendingDownload = PendingImageDownload<Image>;
Common::SlotVector<Image> slot_images;
Common::SlotVector<ImageMapView> slot_map_views;
diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp
index e55d0752ec..5bcea68977 100644
--- a/src/video_core/texture_cache/util.cpp
+++ b/src/video_core/texture_cache/util.cpp
@@ -439,7 +439,7 @@ template <u32 GOB_EXTENT>
}
layers = 1;
} else {
- layers = (std::max)(resources.layers, info.resources.layers + base->layer);
+ layers = (std::max)(info.resources.layers, resources.layers + base->layer);
}
return OverlapResult{
.gpu_addr = overlap.gpu_addr,
@@ -566,7 +566,7 @@ void SwizzleBlockLinearImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr
dst(gpu_memory, gpu_addr + guest_offset, subresource_size, &tmp_buffer);
SwizzleTexture(dst, src, bytes_per_block, num_tiles.width, num_tiles.height,
- num_tiles.depth, block.height, block.depth);
+ num_tiles.depth, block.height, block.depth, 1, gpu_memory.UsesNativeTiling());
}
host_offset += host_bytes_per_layer;
@@ -838,7 +838,7 @@ bool IsValidEntry(const Tegra::MemoryManager& gpu_memory, const TICEntry& config
if (gpu_memory.GpuToCpuAddress(address).has_value()) {
return true;
}
- const ImageInfo info{config};
+ const ImageInfo info{TextureViewHeader(config, gpu_memory.UsesNativeTiling())};
const size_t guest_size_bytes = CalculateGuestSizeInBytes(info);
return gpu_memory.GpuToCpuAddress(address, guest_size_bytes).has_value();
}
@@ -913,7 +913,7 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory
const std::span<u8> dst = output.subspan(host_offset);
const std::span<const u8> src = input.subspan(guest_offset + guest_layer_offset);
UnswizzleTexture(dst, src, 1U << bpp_log2, num_tiles.width, num_tiles.height,
- num_tiles.depth, block.height, block.depth, stride_alignment);
+ num_tiles.depth, block.height, block.depth, stride_alignment, gpu_memory.UsesNativeTiling());
guest_layer_offset += layer_stride;
host_offset += host_bytes_per_layer;
}
diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp
index 25b0860b0f..ef50be156b 100644
--- a/src/video_core/textures/decoders.cpp
+++ b/src/video_core/textures/decoders.cpp
@@ -18,6 +18,11 @@
namespace Tegra::Texture {
namespace {
+constexpr u32 NativeGobOffset(u32 offset, bool native_tiling) {
+ if (!native_tiling) return offset;
+ const u32 difference = ((offset >> 5) ^ (offset >> 6)) & 1;
+ return offset ^ (difference << 5) ^ (difference << 6);
+}
template <u32 mask>
constexpr u32 pdep(u32 value) {
u32 result = 0;
@@ -38,7 +43,7 @@ void incrpdep(u32& value) {
template <bool TO_LINEAR, u32 BYTES_PER_PIXEL>
void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 height, u32 depth,
- u32 block_height, u32 block_depth, u32 stride) {
+ u32 block_height, u32 block_depth, u32 stride, bool native_tiling) {
// The origin of the transformation can be configured here, leave it as zero as the current API
// doesn't expose it.
static constexpr u32 origin_x = 0;
@@ -77,7 +82,7 @@ void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32
const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift;
const u32 base_swizzled_offset = offset_z + offset_y + offset_x;
- const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y);
+ const u32 swizzled_offset = NativeGobOffset(base_swizzled_offset + (swizzled_x | swizzled_y), native_tiling);
const u32 unswizzled_offset =
slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL;
@@ -94,7 +99,7 @@ void SwizzleImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32
template <bool TO_LINEAR, u32 BYTES_PER_PIXEL>
void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 width, u32 height,
u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 num_lines,
- u32 block_height, u32 block_depth, u32 pitch_linear) {
+ u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) {
// The origin of the transformation can be configured here, leave it as zero as the current API
// doesn't expose it.
static constexpr u32 origin_z = 0;
@@ -136,7 +141,7 @@ void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 wid
const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift;
const u32 base_swizzled_offset = offset_z + offset_y + offset_x;
- const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y);
+ const u32 swizzled_offset = NativeGobOffset(base_swizzled_offset + (swizzled_x | swizzled_y), native_tiling);
const u32 unswizzled_offset =
slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL;
@@ -156,12 +161,12 @@ void SwizzleSubrectImpl(std::span<u8> output, std::span<const u8> input, u32 wid
template <bool TO_LINEAR>
void Swizzle(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
- u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment) {
+ u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment, bool native_tiling) {
switch (bytes_per_pixel) {
#define BPP_CASE(x) \
case x: \
return SwizzleImpl<TO_LINEAR, x>(output, input, width, height, depth, block_height, \
- block_depth, stride_alignment);
+ block_depth, stride_alignment, native_tiling);
BPP_CASE(1)
BPP_CASE(2)
BPP_CASE(3)
@@ -181,35 +186,35 @@ void Swizzle(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixe
void UnswizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel,
u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth,
- u32 stride_alignment) {
+ u32 stride_alignment, bool native_tiling) {
const u32 stride = Common::AlignUpLog2(width, stride_alignment) * bytes_per_pixel;
const u32 new_bpp = (std::min)(4U, static_cast<u32>(std::countr_zero(width * bytes_per_pixel)));
width = (width * bytes_per_pixel) >> new_bpp;
bytes_per_pixel = 1U << new_bpp;
Swizzle<false>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth,
- stride);
+ stride, native_tiling);
}
void SwizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
u32 height, u32 depth, u32 block_height, u32 block_depth,
- u32 stride_alignment) {
+ u32 stride_alignment, bool native_tiling) {
const u32 stride = Common::AlignUpLog2(width, stride_alignment) * bytes_per_pixel;
const u32 new_bpp = (std::min)(4U, static_cast<u32>(std::countr_zero(width * bytes_per_pixel)));
width = (width * bytes_per_pixel) >> new_bpp;
bytes_per_pixel = 1U << new_bpp;
Swizzle<true>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth,
- stride);
+ stride, native_tiling);
}
void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 extent_y,
- u32 block_height, u32 block_depth, u32 pitch_linear) {
+ u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) {
switch (bytes_per_pixel) {
#define BPP_CASE(x) \
case x: \
return SwizzleSubrectImpl<true, x>(output, input, width, height, depth, origin_x, \
origin_y, extent_x, extent_y, block_height, \
- block_depth, pitch_linear);
+ block_depth, pitch_linear, native_tiling);
BPP_CASE(1)
BPP_CASE(2)
BPP_CASE(3)
@@ -227,13 +232,13 @@ void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_p
void UnswizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel,
u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x,
- u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear) {
+ u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling) {
switch (bytes_per_pixel) {
#define BPP_CASE(x) \
case x: \
return SwizzleSubrectImpl<false, x>(output, input, width, height, depth, origin_x, \
origin_y, extent_x, extent_y, block_height, \
- block_depth, pitch_linear);
+ block_depth, pitch_linear, native_tiling);
BPP_CASE(1)
BPP_CASE(2)
BPP_CASE(3)
diff --git a/src/video_core/textures/decoders.h b/src/video_core/textures/decoders.h
index 1e14a38753..6151967869 100644
--- a/src/video_core/textures/decoders.h
+++ b/src/video_core/textures/decoders.h
@@ -29,12 +29,12 @@ constexpr u32 SWIZZLE_Y_BITS = 0b011010000;
/// Unswizzles a block linear texture into linear memory.
void UnswizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel,
u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth,
- u32 stride_alignment = 1);
+ u32 stride_alignment = 1, bool native_tiling = false);
/// Swizzles linear memory into a block linear texture.
void SwizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
u32 height, u32 depth, u32 block_height, u32 block_depth,
- u32 stride_alignment = 1);
+ u32 stride_alignment = 1, bool native_tiling = false);
/// This function calculates the correct size of a texture depending if it's tiled or not.
std::size_t CalculateSize(bool tiled, u32 bytes_per_pixel, u32 width, u32 height, u32 depth,
@@ -43,12 +43,12 @@ std::size_t CalculateSize(bool tiled, u32 bytes_per_pixel, u32 width, u32 height
/// Copies an untiled subrectangle into a tiled surface.
void SwizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x, u32 extent_y,
- u32 block_height, u32 block_depth, u32 pitch_linear);
+ u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling = false);
/// Copies a tiled subrectangle into a linear surface.
void UnswizzleSubrect(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel,
u32 width, u32 height, u32 depth, u32 origin_x, u32 origin_y, u32 extent_x,
- u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear);
+ u32 extent_y, u32 block_height, u32 block_depth, u32 pitch_linear, bool native_tiling = false);
/// Obtains the offset of the gob for positions 'dst_x' & 'dst_y'
u64 GetGOBOffset(u32 width, u32 height, u32 dst_x, u32 dst_y, u32 block_height,
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
index f4721543ae..57b07c3707 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
@@ -221,7 +221,31 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const
VkImage handle{};
VmaAllocation allocation{};
VmaAllocationInfo alloc_info{};
- vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info));
+ const VkResult result = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info);
+ if (result != VK_SUCCESS) {
+ LOG_ERROR(Render_Vulkan,
+ "Vulkan image allocation failed (result={}): {}x{}x{}, format={:#x}, type={}, "
+ "layers={}, mips={}, samples={}, usage={:#x}, flags={:#x}",
+ static_cast<s32>(result), ci.extent.width, ci.extent.height, ci.extent.depth,
+ static_cast<u32>(ci.format), static_cast<u32>(ci.imageType), ci.arrayLayers,
+ ci.mipLevels, static_cast<u32>(ci.samples), ci.usage, ci.flags);
+ VmaBudget budgets[VK_MAX_MEMORY_HEAPS]{};
+ vmaGetHeapBudgets(allocator, budgets);
+ const auto format_properties = device.GetPhysical().GetFormatProperties(ci.format);
+ const auto physical_properties = device.GetPhysical().GetProperties();
+ LOG_ERROR(Render_Vulkan,
+ "Image device limits: max2D={}, max allocations={}, format optimal features={:#x}",
+ physical_properties.limits.maxImageDimension2D,
+ physical_properties.limits.maxMemoryAllocationCount,
+ format_properties.optimalTilingFeatures);
+ for (u32 heap = 0; heap < properties.memoryHeapCount; ++heap) {
+ LOG_ERROR(Render_Vulkan,
+ "Vulkan heap {} budget at image failure: usage={} MiB, budget={} MiB, blocks={}, allocations={}",
+ heap, budgets[heap].usage / (1024 * 1024), budgets[heap].budget / (1024 * 1024),
+ budgets[heap].statistics.blockCount, budgets[heap].statistics.allocationCount);
+ }
+ }
+ vk::Check(result);
// Log GPU memory allocation for images
if (GPU::Logging::IsActive() &&
@@ -258,7 +282,12 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa
VmaAllocation allocation{};
VkMemoryPropertyFlags property_flags{};
- vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info));
+ const VkResult result = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info);
+ if (result != VK_SUCCESS) {
+ LOG_CRITICAL(Render_Vulkan, "Buffer allocation failed: bytes={}, usage={}, buffer_flags={:#x}, result={}",
+ ci.size, static_cast<u32>(usage), ci.usage, static_cast<s32>(result));
+ }
+ vk::Check(result);
vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags);
// Log GPU memory allocation for buffers
diff --git a/src/yuzu_cmd/yuzu.cpp b/src/yuzu_cmd/yuzu.cpp
index 473a5b1014..488f1ef47d 100644
--- a/src/yuzu_cmd/yuzu.cpp
+++ b/src/yuzu_cmd/yuzu.cpp
@@ -159,11 +159,13 @@ static void OnStatusMessageReceived(const Network::StatusMessageEntry& msg) {
struct SdlState {
Core::System system{};
+ InputCommon::InputSubsystem input_subsystem{};
std::unique_ptr<EmuWindow_SDL3> emu_window;
};
extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) {
SdlState* state = new SdlState();
+ *appstate = state;
#ifdef _WIN32
if (AttachConsole(ATTACH_PARENT_PROCESS)) {
@@ -202,8 +204,6 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) {
state->system.Initialize();
- InputCommon::InputSubsystem input_subsystem{};
-
// Apply the command line arguments
state->system.ApplySettings();
@@ -212,14 +212,14 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) {
case Settings::RendererBackend::OpenGL_GLSL:
case Settings::RendererBackend::OpenGL_GLASM:
case Settings::RendererBackend::OpenGL_SPIRV:
- state->emu_window = std::make_unique<EmuWindow_SDL3_GL>(&input_subsystem, state->system, lp.fullscreen);
+ state->emu_window = std::make_unique<EmuWindow_SDL3_GL>(&state->input_subsystem, state->system, lp.fullscreen);
break;
#endif
case Settings::RendererBackend::Vulkan:
- state->emu_window = std::make_unique<EmuWindow_SDL3_VK>(&input_subsystem, state->system, lp.fullscreen);
+ state->emu_window = std::make_unique<EmuWindow_SDL3_VK>(&state->input_subsystem, state->system, lp.fullscreen);
break;
case Settings::RendererBackend::Null:
- state->emu_window = std::make_unique<EmuWindow_SDL3_Null>(&input_subsystem, state->system, lp.fullscreen);
+ state->emu_window = std::make_unique<EmuWindow_SDL3_Null>(&state->input_subsystem, state->system, lp.fullscreen);
break;
default:
LOG_CRITICAL(Frontend, "Invalid renderer backend");
@@ -295,7 +295,7 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) {
// QLaunch launched applications (should) now reload shader cache.
static std::mutex shader_cache_reload_mutex;
- state->system.RegisterApplicationChangedCallback([&state](u64 changed_program_id) {
+ state->system.RegisterApplicationChangedCallback([state](u64 changed_program_id) {
if (!Settings::values.use_disk_shader_cache.GetValue()) {
return;
}
@@ -319,7 +319,7 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) {
void(state->system.Run());
if (state->system.DebuggerEnabled())
state->system.InitializeDebugger();
- return SDL_APP_SUCCESS;
+ return SDL_APP_CONTINUE;
}
extern "C" SDL_AppResult SDL_AppIterate(void *appstate) {
SdlState *state = (SdlState *)appstate;
@@ -328,10 +328,13 @@ extern "C" SDL_AppResult SDL_AppIterate(void *appstate) {
extern "C" SDL_AppResult SDL_AppEvent(void *appstate, SDL_Event *event) {
SdlState *state = (SdlState *)appstate;
state->emu_window->OnEvent(*event);
- return SDL_APP_SUCCESS;
+ return SDL_APP_CONTINUE;
}
extern "C" void SDL_AppQuit(void *appstate, SDL_AppResult result) {
SdlState *state = (SdlState *)appstate;
+ if (!state) {
+ return;
+ }
state->system.DetachDebugger();
void(state->system.Pause());
state->system.ShutdownMainProcess();