diff options
| author | nowhere02 <140357165+nowhere02@users.noreply.github.com> | 2026-11-02 00:17:14 -0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-11-02 00:17:14 -0300 |
| commit | 9c55e70d790a4cc9507d80dbb9a9d6224175574e (patch) | |
| tree | 4c53ccbf16ab56bb00e512cc7926b1d0d438792d | |
| parent | b8d2c08d6efd312620b066cdab2f0238ebc94a26 (diff) | |
| parent | 06e0541e18e5bb8ab8d3d03660d3d82cb112f62b (diff) | |
Merge pull request #270 from Duck912/fe-render
Fe render
35 files changed, 1724 insertions, 171 deletions
diff --git a/src/audio_core/common/feature_support.h b/src/audio_core/common/feature_support.h index 4585be45ef..d6571c0661 100644 --- a/src/audio_core/common/feature_support.h +++ b/src/audio_core/common/feature_support.h @@ -17,6 +17,16 @@ namespace AudioCore { constexpr u32 CurrentRevision = 15; +constexpr u32 NativeCurrentRevision = 16; + +constexpr bool IsNativeRevision(u32 revision) { + return (revision & 0xffffff) == Common::MakeMagic('R', 'E', 'v', '\0'); +} + +constexpr u32 EncodeRevision(u32 revision, bool native) { + return Common::MakeMagic('R', 'E', native ? 'v' : 'V', + static_cast<char>('0' + revision)); +} enum class SupportTags { CommandProcessingTimeEstimatorVersion4, @@ -57,12 +67,15 @@ enum class SupportTags { }; constexpr u32 GetRevisionNum(u32 user_revision) { - if (user_revision >= 0x100) { - user_revision -= Common::MakeMagic('R', 'E', 'V', '0'); - user_revision >>= 24; + if (user_revision < 0x100) { + return user_revision; } - - return user_revision; + if (!IsNativeRevision(user_revision) && + (user_revision & 0xffffff) != Common::MakeMagic('R', 'E', 'V', '\0')) { + return ~u32{0}; + } + const u32 encoded = user_revision >> 24; + return encoded >= '0' ? encoded - '0' : ~u32{0}; }; constexpr bool CheckFeatureSupported(SupportTags tag, u32 user_revision) { @@ -108,11 +121,12 @@ constexpr bool CheckFeatureSupported(SupportTags tag, u32 user_revision) { return false; } user_revision = GetRevisionNum(user_revision); - return (*feature).second <= user_revision; + return user_revision != ~u32{0} && (*feature).second <= user_revision; } constexpr bool CheckValidRevision(u32 user_revision) { - return GetRevisionNum(user_revision) <= CurrentRevision; + return GetRevisionNum(user_revision) <= + (IsNativeRevision(user_revision) ? NativeCurrentRevision : CurrentRevision); }; } // namespace AudioCore diff --git a/src/audio_core/renderer/behavior/behavior_info.cpp b/src/audio_core/renderer/behavior/behavior_info.cpp index 63b80c503a..983cfd5b85 100644 --- a/src/audio_core/renderer/behavior/behavior_info.cpp +++ b/src/audio_core/renderer/behavior/behavior_info.cpp @@ -16,8 +16,7 @@ u32 BehaviorInfo::GetProcessRevisionNum() const { } u32 BehaviorInfo::GetProcessRevision() const { - return Common::MakeMagic('R', 'E', 'V', - static_cast<char>(static_cast<u8>('0') + process_revision)); + return EncodeRevision(process_revision, native_platform); } u32 BehaviorInfo::GetUserRevisionNum() const { @@ -25,11 +24,12 @@ u32 BehaviorInfo::GetUserRevisionNum() const { } u32 BehaviorInfo::GetUserRevision() const { - return Common::MakeMagic('R', 'E', 'V', - static_cast<char>(static_cast<u8>('0') + user_revision)); + return EncodeRevision(user_revision, native_platform); } void BehaviorInfo::SetUserLibRevision(const u32 user_revision_) { + native_platform = IsNativeRevision(user_revision_); + process_revision = native_platform ? NativeCurrentRevision : CurrentRevision; user_revision = GetRevisionNum(user_revision_); } diff --git a/src/audio_core/renderer/behavior/behavior_info.h b/src/audio_core/renderer/behavior/behavior_info.h index 757f163c5a..318d4e1ff4 100644 --- a/src/audio_core/renderer/behavior/behavior_info.h +++ b/src/audio_core/renderer/behavior/behavior_info.h @@ -400,6 +400,7 @@ public: u32 process_revision; /// User version u32 user_revision{}; + bool native_platform{}; /// Behaviour flags Flags flags{}; /// Errors generated and reported during Update diff --git a/src/audio_core/renderer/behavior/info_updater.cpp b/src/audio_core/renderer/behavior/info_updater.cpp index ca3eaece94..a8fbcf9730 100644 --- a/src/audio_core/renderer/behavior/info_updater.cpp +++ b/src/audio_core/renderer/behavior/info_updater.cpp @@ -122,12 +122,21 @@ Result InfoUpdater::UpdateVoices(VoiceContext& voice_context, u8 flush_wave_buffer_count; u16 reserved2; VoiceInfo::Flags flags; + u8 reserved_after_flags; SrcQuality src_quality; + u8 crossfade_curve; u32 external_ctx; u32 external_ctx_size; u32 reserved3[2]; }; + static_assert(sizeof(VoiceInParameterV2) == 0x188); + static_assert(offsetof(VoiceInParameterV2, crossfade_curve) == 0x177); const auto* vin = reinterpret_cast<const VoiceInParameterV2*>(input + i * in_stride); + if (behaviour.GetUserRevisionNum() >= 16 && vin->in_use && vin->crossfade_curve != 0) { + LOG_ERROR(Service_Audio, "Unsupported voice biquad crossfade curve {}", + vin->crossfade_curve); + return Service::Audio::ResultInvalidUpdateInfo; + } local_in.id = vin->id; local_in.node_id = vin->node_id; local_in.is_new = vin->is_new; diff --git a/src/core/hle/service/am/frontend/applet_software_keyboard.cpp b/src/core/hle/service/am/frontend/applet_software_keyboard.cpp index db42781e18..061394bff8 100644 --- a/src/core/hle/service/am/frontend/applet_software_keyboard.cpp +++ b/src/core/hle/service/am/frontend/applet_software_keyboard.cpp @@ -222,6 +222,14 @@ void SoftwareKeyboard::InitializeForeground() { sizeof(SwkbdConfigNew)); swkbd_config_common.use_utf8 = true; break; + case SwkbdAppletVersion::Version1441810: { + // Fire Emblem: Fortune's Weave asks for UTF-8 and keeps the reply up to the first NUL, so the + // Switch 1 layout's use_utf8 (a byte of the native initial_string_offset) left one letter. + const bool decoded = + DecodeNativeSwkbdConfig(swkbd_config_data, swkbd_config_common, swkbd_config_new); + ASSERT(decoded); + break; + } default: UNIMPLEMENTED_MSG("Unknown SwkbdConfig revision={} with size={}", swkbd_applet_version, swkbd_config_data.size()); diff --git a/src/core/hle/service/am/frontend/applet_software_keyboard_types.h b/src/core/hle/service/am/frontend/applet_software_keyboard_types.h index f263a704cd..8ff8391aa1 100644 --- a/src/core/hle/service/am/frontend/applet_software_keyboard_types.h +++ b/src/core/hle/service/am/frontend/applet_software_keyboard_types.h @@ -7,6 +7,9 @@ #pragma once #include <array> +#include <cstddef> +#include <cstring> +#include <span> #include "common/bit_field.h" #include "common/common_funcs.h" @@ -31,6 +34,7 @@ enum class SwkbdAppletVersion : u32_le { Version393227 = 0x6000B, // 6.0.0 - 7.0.1 Version524301 = 0x8000D, // 8.0.0+ Version1310736 = 0x140010, // Switch 2 (Pokemon Pokopia): 0x530-byte config, UTF-8 text + Version1441810 = 0x160012, // Switch 2 (Fire Emblem: Fortune's Weave): native 0x530-byte config }; enum class SwkbdType : u32 { @@ -184,8 +188,64 @@ struct SwkbdConfigNew { }; static_assert(sizeof(SwkbdConfigNew) == 0x4C8 - sizeof(SwkbdConfigCommon), "SwkbdConfigNew has incorrect size."); + +// SwkbdAppletVersion 0x160012: the text prefix of the common part keeps its layout, the flags, +// lengths, string offsets and dictionary fields move (checked against +// nn::swkbd::keyboardConfig2kbdConfig in Fire Emblem: Fortune's Weave's sdk). +struct SwkbdConfigNative { + std::array<u8, 0x3AA> common_prefix{}; + bool use_utf8{}; + bool use_blur_background{}; + u32 max_text_length{}; + u32 min_text_length{}; + SwkbdPasswordMode password_mode{}; + SwkbdTextDrawType text_draw_type{}; + u32 initial_string_offset{}; + u32 initial_string_length{}; + u32 user_dictionary_offset{}; + u32 user_dictionary_entries{}; + std::array<u64, 24> customized_dictionary_set_entries{}; + u8 total_customized_dictionary_set_entries{}; + bool use_text_check{}; + bool disable_cancel_button{}; + u8 trigger{}; + std::array<u32, 8> text_grouping{}; + INSERT_PADDING_BYTES(0x80); +}; +static_assert(sizeof(SwkbdConfigNative) == 0x530); +static_assert(offsetof(SwkbdConfigNative, use_text_check) == 0x48D); +static_assert(offsetof(SwkbdConfigNative, text_grouping) == 0x490); #pragma pack(pop) +inline bool DecodeNativeSwkbdConfig(std::span<const u8> data, SwkbdConfigCommon& common, + SwkbdConfigNew& config) { + if (data.size() != sizeof(SwkbdConfigNative)) { + return false; + } + SwkbdConfigNative native{}; + std::memcpy(&native, data.data(), sizeof(native)); + common = {}; + config = {}; + std::memcpy(static_cast<void*>(&common), native.common_prefix.data(), native.common_prefix.size()); + common.enable_return_button = native.common_prefix[0x1B] != 0; + common.use_utf8 = native.use_utf8; + common.use_blur_background = native.use_blur_background; + common.max_text_length = native.max_text_length; + common.min_text_length = native.min_text_length; + common.password_mode = native.password_mode; + common.text_draw_type = native.text_draw_type; + common.initial_string_offset = native.initial_string_offset; + common.initial_string_length = native.initial_string_length; + common.user_dictionary_offset = native.user_dictionary_offset; + common.user_dictionary_entries = native.user_dictionary_entries; + common.use_text_check = native.use_text_check; + config.text_grouping = native.text_grouping; + config.customized_dictionary_set_entries = native.customized_dictionary_set_entries; + config.total_customized_dictionary_set_entries = native.total_customized_dictionary_set_entries; + config.disable_cancel_button = native.disable_cancel_button; + return true; +} + struct SwkbdTextCheck { SwkbdTextCheckResult text_check_result{}; std::array<char16_t, STRING_BUFFER_SIZE / 2> text_check_message{}; diff --git a/src/core/hle/service/nvdrv/core/container.cpp b/src/core/hle/service/nvdrv/core/container.cpp index 9edce03f64..22db243e3f 100644 --- a/src/core/hle/service/nvdrv/core/container.cpp +++ b/src/core/hle/service/nvdrv/core/container.cpp @@ -11,6 +11,7 @@ #include "core/hle/service/nvdrv/core/heap_mapper.h" #include "core/hle/service/nvdrv/core/nvmap.h" #include "core/hle/service/nvdrv/core/syncpoint_manager.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/memory.h" #include "video_core/host1x/host1x.h" @@ -130,6 +131,7 @@ void Container::CloseSession(SessionId session_id) { session.has_preallocated_area = false; } session.is_active = false; + Devices::Nx2GpuMemoryRegistry::Get().ForgetDeviceRanges(session.asid.id); smmu.UnregisterProcess(impl->sessions[session_id.id].asid); impl->id_pool.emplace_front(session_id.id); } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index e66c72cd26..9d696c027e 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -5,6 +5,7 @@ // SPDX-FileCopyrightText: 2021 Skyline Team and Contributors // SPDX-License-Identifier: GPL-3.0-or-later +#include <cstdlib> #include <cstring> #include <utility> @@ -35,6 +36,13 @@ nvhost_as_gpu::nvhost_as_gpu(Core::System& system_, Module& module_, NvCore::Con nvhost_as_gpu::~nvhost_as_gpu() { Nx2GpuMemoryRegistry::Get().RemoveListener(this); + // Leave shared ranges to the process's other address spaces; like this address space's own + // ranges, one nobody else maps stays allocated. + for (const auto& [key, range] : nx2_device_ranges) { + if (range.shared) { + Nx2GpuMemoryRegistry::Get().ReleaseDeviceRange(range.asid, key.first, key.second); + } + } } NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input, @@ -134,25 +142,37 @@ void nvhost_as_gpu::Nx2Map(u64 gpu_address, u64 cpu_address, u64 size) { } // The same CPU range is often mapped twice (at its own address and in a sparse region), so // reuse its SMMU mapping. - DAddr device_address{}; - bool heap = false; + Nx2DeviceRange range{}; if (const auto it = nx2_device_ranges.find({cpu_address, size}); it != nx2_device_ranges.end()) { - device_address = it->second.address; - heap = it->second.heap; + range = it->second; } else if (session->has_preallocated_area && session->mapper->IsInBounds(cpu_address, size)) { - device_address = session->mapper->Map(cpu_address, size); - heap = true; + range = {session->mapper->Map(cpu_address, size), true}; } else { - auto& smmu = system.Host1x().MemoryManager(); - device_address = smmu.Allocate(Common::AlignUp(size, 0x20000)); - if (device_address == 0) { + const auto allocate = [&]() -> u64 { + auto& smmu = system.Host1x().MemoryManager(); + const DAddr address = smmu.Allocate(Common::AlignUp(size, 0x20000)); + if (address != 0) { + smmu.Map(address, cpu_address, size, session->asid, true); + } + return address; + }; + // EDEN_NX2_PRIVATE_DEVICE_RANGES=1 (A/B aid): a range per address space. + static const bool private_ranges = std::getenv("EDEN_NX2_PRIVATE_DEVICE_RANGES") != nullptr; + if (private_ranges) { + range = {allocate(), false}; + } else { + range = {Nx2GpuMemoryRegistry::Get().AcquireDeviceRange(session->asid.id, cpu_address, + size, allocate), + false, true, session->asid.id}; + } + if (range.address == 0) { LOG_CRITICAL(Service_NVDRV, "Switch 2 GPU memory {:#x}+{:#x}: out of SMMU space", cpu_address, size); return; } - smmu.Map(device_address, cpu_address, size, session->asid, true); } - nx2_device_ranges[{cpu_address, size}] = {device_address, heap}; + const DAddr device_address = range.address; + nx2_device_ranges[{cpu_address, size}] = range; Nx2GpuMemoryRegistry::Get().RecordDeviceRange(cpu_address, size, device_address); gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false); LOG_DEBUG(Service_NVDRV, "Switch 2 GPU mapping {:#x}+{:#x} -> CPU {:#x}", gpu_address, size, @@ -177,6 +197,10 @@ void nvhost_as_gpu::Nx2Release(u64 cpu_address, u64 size) { } const Nx2DeviceRange range = it->second; nx2_device_ranges.erase(it); + if (range.shared && + !Nx2GpuMemoryRegistry::Get().ReleaseDeviceRange(range.asid, cpu_address, size)) { + return; // another address space of the process still maps it + } auto* session = container.GetSession(nx2_session); auto* const mapper = range.heap && session != nullptr ? session->mapper.get() : nullptr; auto& smmu = system.Host1x().MemoryManager(); diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h index 39b63ea369..37b6d0d4c4 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h @@ -238,6 +238,8 @@ private: struct Nx2DeviceRange { DAddr address; bool heap; ///< Mapped through the session's heap mapper (else an SMMU allocation) + bool shared{}; ///< A reference to the process's range in Nx2GpuMemoryRegistry + size_t asid{}; }; std::map<std::pair<u64, u64>, Nx2DeviceRange> nx2_device_ranges; void Nx2Release(u64 cpu_address, u64 size); diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp index 5998940afa..cf4a473496 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp @@ -113,12 +113,22 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> case 0x6: return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetTPCMasks3, input, output, inline_output); - case 0x13: - LOG_DEBUG(Service_NVDRV, "(STUBBED) called."); - - // TODO (jarrodnorwell) - - return NvResult::NotImplemented; + case 0x13: { + // NVGPU_GPU_IOCTL_VSMS_MAPPING: an 8-byte argument struct plus a buffer of + // {gpc_index, tpc_index} byte pairs, one per virtual SM. The SDK treats a pair left at + // its 0xc0c0 sentinel as failure, so fill every pair: one GPC, two SMs per TPC. + if (command.raw != 0xc0084713 || input.size() != 8 || output.size() != 8 || + inline_output.size() < 2) { + return NvResult::BadParameter; + } + std::memcpy(output.data(), input.data(), output.size()); + for (size_t i = 0; i + 1 < inline_output.size(); i += 2) { + inline_output[i] = 0; // gpc_index + inline_output[i + 1] = static_cast<u8>(i / 4); // tpc_index + } + LOG_DEBUG(Service_NVDRV, "VSMS mapping for {} SMs", inline_output.size() / 2); + return NvResult::Success; + } default: break; } @@ -218,6 +228,9 @@ void ApplyNx2Characteristics(GC& gc) { if (remap) { gc.flags |= 1ULL << 35; } + // Bits 40..43: the 2D, 3D, compute and inline-to-memory class families. The SDK filters its + // class enumeration by them; without them Fire Emblem: Fortune's Weave aborts at boot. + gc.flags |= 0xfULL << 40; // The Switch 2 layout keeps nine s16 interface versions at 0x70..0x80 (like nvgpu's // *_ioctl_nr_last); its NvRm requires e.g. [0x78] >= 6 and enables features above 0xa, 0x1f diff --git a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h index 869e17caf6..8dee9ccfb7 100644 --- a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h +++ b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h @@ -5,6 +5,7 @@ #include <algorithm> #include <atomic> +#include <compare> #include <functional> #include <map> #include <mutex> @@ -166,12 +167,50 @@ public: { std::scoped_lock lk{device_mutex}; device_ranges.clear(); + shared_ranges.clear(); } std::scoped_lock lk{mutex}; mappings.clear(); sparse_regions.clear(); } + // Device (SMMU) range of a mapping outside the heap, shared by every address space of one + // process: the caches key memory by device address, so a range per address space made separate + // copies of the same memory, and presentation could read one the game never drew into. The + // first address space to map the range allocates it with `allocate` (0: out of space); every + // call takes a reference. + u64 AcquireDeviceRange(size_t asid, u64 cpu_address, u64 size, + const std::function<u64()>& allocate) { + std::scoped_lock lk{device_mutex}; + const SharedRangeKey key{asid, cpu_address, size}; + if (const auto it = shared_ranges.find(key); it != shared_ranges.end()) { + ++it->second.references; + return it->second.device_address; + } + const u64 device_address = allocate(); + if (device_address != 0) { + shared_ranges.emplace(key, SharedRange{device_address, 1}); + } + return device_address; + } + + // Drops a reference. True for the last one: the caller frees the range. + bool ReleaseDeviceRange(size_t asid, u64 cpu_address, u64 size) { + std::scoped_lock lk{device_mutex}; + const auto it = shared_ranges.find(SharedRangeKey{asid, cpu_address, size}); + if (it == shared_ranges.end() || --it->second.references != 0) { + return false; + } + shared_ranges.erase(it); + return true; + } + + // The process's session closed and its ASID can be reused: stop sharing its ranges. + void ForgetDeviceRanges(size_t asid) { + std::scoped_lock lk{device_mutex}; + std::erase_if(shared_ranges, [asid](const auto& entry) { return entry.first.asid == asid; }); + } + // Device (SMMU) address of a CPU range an address space mapped; used to present images. void RecordDeviceRange(u64 cpu_address, u64 size, u64 device_address) { std::scoped_lock lk{device_mutex}; @@ -197,6 +236,18 @@ private: std::mutex device_mutex; std::map<u64, std::pair<u64, u64>> device_ranges; // cpu start -> (size, device address) + struct SharedRangeKey { + size_t asid; + u64 cpu_address; + u64 size; + auto operator<=>(const SharedRangeKey&) const = default; + }; + struct SharedRange { + u64 device_address; + u32 references; + }; + std::map<SharedRangeKey, SharedRange> shared_ranges; + struct Mapping { u64 gpu_address; u64 cpu_address; diff --git a/src/core/hle/service/nvnflinger/pixel_format.h b/src/core/hle/service/nvnflinger/pixel_format.h index b6e8b5f8a0..a38c8be777 100644 --- a/src/core/hle/service/nvnflinger/pixel_format.h +++ b/src/core/hle/service/nvnflinger/pixel_format.h @@ -19,6 +19,7 @@ enum class PixelFormat : u32 { Bgra8888 = 5, Rgba5551 = 6, Rgba4444 = 7, + Rgba1010102 = 43, }; } // namespace Service::android diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h index 628bdeda5b..5a2963f179 100644 --- a/src/core/hle/service/nx2_gsv.h +++ b/src/core/hle/service/nx2_gsv.h @@ -21,6 +21,7 @@ #include <optional> #include <memory> #include <mutex> +#include <utility> #include <vector> #include "common/alignment.h" @@ -45,13 +46,17 @@ #include "core/hle/kernel/k_resource_limit.h" #include "core/hle/kernel/k_server_session.h" #include "core/hle/kernel/k_event.h" +#include "core/hle/kernel/k_hardware_timer.h" #include "core/hle/kernel/k_readable_event.h" #include "core/hle/kernel/k_session.h" +#include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/k_transfer_memory.h" +#include "core/hle/kernel/svc_results.h" #include "core/memory.h" #include "video_core/host1x/host1x.h" #include "video_core/host1x/nx2_video_decoder.h" #include "video_core/nx2.h" +#include "video_core/textures/decoders.h" #include "core/hle/service/hle_ipc.h" #include "core/hle/service/ipc_helpers.h" #include "core/hle/service/kernel_helpers.h" @@ -304,6 +309,8 @@ public: {21, &INx2GpuSession::Start, "Start (guessed)"}, {22, &INx2GpuSession::OpenSubSession, "OpenSubSession (guessed)"}, {33, &INx2GpuSession::GetCount33, "Unknown33 (u32 count)"}, + {41, &INx2GpuSession::CreateGraphicsDevice, "CreateGraphicsDevice"}, + {42, &INx2GpuSession::DestroyGraphicsDevice, "DestroyGraphicsDevice"}, {201, &INx2GpuSession::GetUserPartition, "GetUserPartition (newer SDKs)"}, {202, &INx2GpuSession::GetGsvPartition, "GetGsvPartition (newer SDKs)"}, {203, &INx2GpuSession::GetVaRange, "GetVaRange (guessed)"}, @@ -395,9 +402,39 @@ private: static_cast<u32>(second), static_cast<u32>(second >> 32)}); } + // An empty partition is min = ~0, max = 0 (min > max). Fire Emblem: Fortune's Weave's SDK + // reads (0, 0) as a one-byte partition at address 0 and aborts in nn::gpu::sdk. void GetGsvPartition(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 202 [2]={:#x}, empty", ctx.CommandBuffer()[2]); - TipcReplyWords(ctx, {0, 0, 0, 0}); + TipcReplyWords(ctx, {~0U, ~0U, 0, 0}); + } + + // Fire Emblem: Fortune's Weave: 41(name u64, flags u64, debug label u64) returns a device id + // that later NvRm calls name; 0 is not a valid id. Only flags 0 has been seen. + void CreateGraphicsDevice(HLERequestContext& ctx) { + const auto* b = ctx.CommandBuffer(); + const u64 name = u64{b[2]} | (u64{b[3]} << 32); + const u64 flags = u64{b[4]} | (u64{b[5]} << 32); + u32 id = 0; + for (u32 candidate = 1; candidate < m_devices.size(); ++candidate) { + if (!m_devices[candidate]) { + m_devices[candidate] = true; + id = candidate; + break; + } + } + LOG_INFO(Service, "nx2:gpu 41 create graphics device name={:#x} flags={:#x} -> id {}", name, + flags, id); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(id != 0 ? ResultSuccess : Kernel::ResultOutOfResource); + rb.Push(id); + } + + void DestroyGraphicsDevice(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2] & 0xff; + const bool existed = id < m_devices.size() && std::exchange(m_devices[id], false); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(existed ? ResultSuccess : Kernel::ResultNotFound); } void OpenSubSession(HLERequestContext& ctx) { @@ -1246,6 +1283,7 @@ private: std::array<Kernel::KEvent*, NumIds> m_events{}; std::map<u32, GpuMemory> m_memory; u32 m_next_memory_handle{1}; + std::array<bool, 64> m_devices{}; // gpu 41 graphics device ids (0 unused) }; // Native memory and images handed out by the Switch 2 graphics server (gsv 111/114/423/429/315), @@ -1264,6 +1302,10 @@ struct Nx2NativeObjects { u64 offset{}; // into its memory (images of a decoder pool share one) bool window_pool{}; bool decoder{}; // a video decoder output image (gsv 911) + // A decoder image the client reads on the GPU (gsv 428/426): written block-linear. + bool gpu_read{}; + u32 gpu_maps{}; // outstanding 426 maps + bool mapped_here{}; // 426 added its GPU mapping (no other mapping covered it) }; std::mutex mutex; std::map<u32, Memory> memories; @@ -1407,11 +1449,12 @@ public: {11, &INx2Vi2Session::ReturnEvent, "Unknown11"}, {31, &INx2Vi2Session::GetVsyncEvent, "GetVsyncEvent (guessed)"}, {32, &INx2Vi2Session::GetVsyncEvent, "GetDisplayVsyncEvent (guessed)"}, - // Title 1 calls these ~75 times a second with the display name "default"; the + {104, &INx2Vi2Session::GetHdrStatus, "GetHdrStatus"}, + // Title 1 calls this ~75 times a second with the display name "default"; the // auto stub's empty success reply works, but logged two lines per call (a 50 MB log // in an 18 minute session). - {104, &INx2Vi2Session::SucceedQuietly, "Unknown104 (display \"default\")"}, {105, &INx2Vi2Session::SucceedQuietly, "Unknown105 (display \"default\")"}, + {106, &INx2Vi2Session::GetBufferMaxResolution, "GetBufferMaxResolution"}, {228, &INx2Vi2Session::CancelClientBuffer, "CancelClientBuffer"}, {229, &INx2Vi2Session::PresentClientBuffer, "PresentClientBuffer"}, }); @@ -1464,9 +1507,9 @@ private: WaitForFrame(~w[22], w[24]); } const auto waited = std::chrono::steady_clock::now(); - Composite(w[5], ~w[22], w[24]); + Composite(w[5], ~w[22], w[24], w[7]); const auto composited = std::chrono::steady_clock::now(); - PaceToVblank(); + PaceToVblank(w[12]); const auto paced = std::chrono::steady_clock::now(); // Where each frame's time goes on the present thread, averaged per 600 presents: the // game is blocked here while Eden's GPU thread catches up (fence) and composites. @@ -1522,18 +1565,39 @@ private: // present faster than that. Every slot is reported free here: hold each present until the // next vblank instead (title 4 ran at up to 160 fps, its timing and audio with it). std::chrono::steady_clock::time_point m_next_vblank{}; + u32 m_swap_word{}; - void PaceToVblank() { + // timing: present word 12, (swap interval << 16) | refresh rate. Fire Emblem: Fortune's Weave + // sends 0x1003c during its boot logos and 0x2003c (two vblanks per frame, 30 fps) after; paced + // at every vblank it ran at double speed. EDEN_NX2_SWAP_INTERVAL=<n> forces the interval. + void PaceToVblank(u32 timing) { if (!Settings::values.use_speed_limit.GetValue() || Settings::SpeedLimit() == 0) { return; } - const auto interval = std::chrono::nanoseconds{100'000'000'000LL / 60 / Settings::SpeedLimit()}; + static const u32 forced_interval = [] { + const char* value = std::getenv("EDEN_NX2_SWAP_INTERVAL"); + return value ? static_cast<u32>(std::strtoul(value, nullptr, 10)) : 0U; + }(); + const u32 swap_interval = + forced_interval != 0 ? forced_interval : std::clamp<u32>(timing >> 16, 1, 4); + const u32 refresh_rate = (timing & 0xffff) >= 30 && (timing & 0xffff) <= 240 ? timing & 0xffff : 60; + if (timing != std::exchange(m_swap_word, timing)) { + LOG_INFO(Service, "nx2:vi2 present #{} swap interval {} at {} Hz (word 12 = {:#x})", + m_presents, swap_interval, refresh_rate, timing); + } + const auto interval = std::chrono::nanoseconds{ + 100'000'000'000LL * swap_interval / refresh_rate / Settings::SpeedLimit()}; const auto now = std::chrono::steady_clock::now(); if (now > m_next_vblank + interval) { - // Running slower than 60 fps: show at once and don't bank the lost time. + // Running slower than the swap interval: show at once and don't bank the lost time. m_next_vblank = now; } else if (now < m_next_vblank) { - std::this_thread::sleep_until(m_next_vblank); + // Sleep this service thread in the guest kernel. It runs on guest core 3, and a host + // sleep held that core for the whole wait: game threads scheduled there ran only + // between presents (Fortune's Weave's loader read one RomFS block per frame). + const auto wait = std::chrono::duration_cast<std::chrono::nanoseconds>(m_next_vblank - now); + static_cast<void>(Kernel::GetCurrentThread(kernel).Sleep( + kernel, kernel.HardwareTimer().GetTick() + wait.count() + 2)); } m_next_vblank += interval; } @@ -1586,9 +1650,12 @@ private: // Shows a presented native image: word 5 is the image id, ~word 22 / word 24 the GPU fence // (syncpoint, value) to wait for (layout from changes.patch). The image is a block-linear // RGBA8 surface in the console's native GOB layout at GPU address == CPU address. - void Composite(u32 image_id, u32 fence_id, u32 fence_value) { + // present_size: word 7 of the present, (height - 1) << 16 | (width - 1) of the frame drawn + // into the image's top-left corner (Fire Emblem: Fortune's Weave: 1280x720 in 3840x2160). + void Composite(u32 image_id, u32 fence_id, u32 fence_value, u32 present_size = 0) { u64 address{}; Nx2ImageLayout layout{}; + u32 image_format{}; { std::scoped_lock lk{m_objects->mutex}; const auto image = m_objects->images.find(image_id); @@ -1601,6 +1668,7 @@ private: } address = memory->second.address; layout = Nx2CalculateImageLayout(image->second.preference.data()); + image_format = image->second.preference[6]; } const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address); if (!device) { @@ -1610,6 +1678,11 @@ private: } u32 crop_width = layout.width; u32 crop_height = layout.height; + // Present word 7 holds the size of the frame drawn into the image's top-left corner. + if (present_size != 0) { + crop_width = (std::min)(crop_width, (present_size & 0xffffU) + 1); + crop_height = (std::min)(crop_height, (present_size >> 16) + 1); + } // Handheld titles keep their docked present images (up to 3840x2160) and draw the frame // into the top-left 1920x1080, the handheld panel's resolution; the console's display // scans out only that region. Showing whole images put the frame in a corner inside black @@ -1648,7 +1721,9 @@ private: .width = layout.width, .height = layout.height, .stride = static_cast<u32>(layout.pitch / 4), - .pixel_format = android::PixelFormat::Rgba8888, + // Preference word 6: 1 = RGBA8, 4 = RGB10A2. + .pixel_format = image_format == 4 ? android::PixelFormat::Rgba1010102 + : android::PixelFormat::Rgba8888, .transform_flags = android::BufferTransformFlags::Unset, .crop_rect = {0, 0, static_cast<int>(crop_width), static_cast<int>(crop_height)}, .blending = Tegra::BlendMode::Opaque, @@ -1752,6 +1827,50 @@ private: TipcReplyWords(ctx, {1}); } + // 104(u64 display name): result, u64 output format, u32 0. Format 1 is SDR. A bare success + // left the format to stale reply words and a game could set up HDR output from them. + void GetHdrStatus(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 name = u64{packet[2]} | (u64{packet[3]} << 32); + constexpr u64 DefaultDisplay = 0x00746c7561666564ULL; // "default" + Result result = Kernel::ResultInvalidCombination; + u64 format{}; + if (packet[0] == 0x78 && packet[1] == 2) { + result = Result{ErrorModule::VI, 7}; + if (name == DefaultDisplay) { + format = 1; + result = ResultSuccess; + } + } + IPC::ResponseBuilder rb{ctx, 5}; + rb.Push(result.raw); + rb.Push(format); + rb.Push<u32>(0); + } + + // The highest buffer resolution mode of the "default" display. Fire Emblem: Fortune's Weave picks + // its output from it: 1 -> 1280x720, 2 -> 1920x1080, 4 and 8 -> 2560x1440; 0 (the auto stub's + // reply), 3 and 5 fall back to 720p. Docked reports 4, handheld 2 (the screen is 1080p); + // EDEN_VI2_MAX_RESOLUTION_MODE=<n> overrides. + void GetBufferMaxResolution(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 name = u64{packet[2]} | (u64{packet[3]} << 32); + constexpr u64 DefaultDisplay = 0x00746c7561666564ULL; // "default" + Result result = Kernel::ResultInvalidCombination; + u64 mode{}; + if (packet[0] == 0x7a && packet[1] == 2) { + result = Result{ErrorModule::VI, 7}; + if (name == DefaultDisplay) { + static const char* forced = std::getenv("EDEN_VI2_MAX_RESOLUTION_MODE"); + mode = forced ? std::strtoull(forced, nullptr, 0) : Settings::IsDockedMode() ? 4 : 2; + result = ResultSuccess; + } + } + IPC::ResponseBuilder rb{ctx, 4}; + rb.Push(result.raw); + rb.Push(mode); + } + // The same reply as the auto stub (success, no data), without its log lines. void SucceedQuietly(HLERequestContext& ctx) { IPC::ResponseBuilder rb{ctx, 2}; @@ -2221,6 +2340,7 @@ private: const Decoded& frame, std::vector<u8>& bytes) { u64 address{}; Nx2DecoderImageLayout layout{}; + bool block_linear{}; { std::scoped_lock lk{m_objects->mutex}; const auto image = m_objects->images.find(image_id); @@ -2228,6 +2348,8 @@ private: return {m_width, m_height}; } layout = Nx2CalculateDecoderImageLayout(image->second.preference.data()); + block_linear = image->second.gpu_read && + layout.format != Tegra::Host1x::Nx2PictureFormat::Nv12; const auto memory = m_objects->memories.find(image->second.memory); if (memory != m_objects->memories.end() && memory->second.address != 0) { address = memory->second.address + image->second.offset; @@ -2241,10 +2363,25 @@ private: const u32 width = std::min(frame.picture->width, layout.width); const u32 height = std::min(frame.picture->height, layout.height); bytes.resize(layout.size); + auto& memory = system.ApplicationMemory(); + if (block_linear) { + // Read on the GPU: RGBA (the blit's format 0xd5) in the native block-linear layout, + // 16-GOB blocks, as our Eden tree writes these images. + decoder->Render(*frame.picture, Tegra::Host1x::Nx2PictureFormat::Rgba8, width, height, + layout.pitch, 0, bytes); + m_tiled.resize(layout.size); + Tegra::Texture::SwizzleTexture(m_tiled, bytes, 4, static_cast<u32>(layout.pitch / 4), + layout.height, 1, 4, 0, 1, true); + memory.WriteBlock(address, m_tiled.data(), m_tiled.size()); + if (const auto device = + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address)) { + system.GPU().InvalidateRegion(*device, m_tiled.size()); + } + return {width, height}; + } decoder->Render(*frame.picture, layout.format, width, height, layout.pitch, layout.chroma_offset, bytes); DumpFrame(bytes, layout, width, height); - auto& memory = system.ApplicationMemory(); memory.WriteBlock(address, bytes.data(), layout.pitch * height); if (layout.format == Tegra::Host1x::Nx2PictureFormat::Nv12) { memory.WriteBlock(address + layout.chroma_offset, bytes.data() + layout.chroma_offset, @@ -2290,6 +2427,7 @@ private: u64 m_submitted{}; u32 m_syncpoint{NoSyncpoint}; ///< Counts finished frames (allocated at the first register) u32 m_sync_base{}; ///< Its value before the first frame + std::vector<u8> m_tiled; ///< Decode thread: block-linear copy of a frame (WriteImage) // Shared with the decode thread. std::mutex m_mutex; @@ -2333,6 +2471,7 @@ public: {429, &INx2GraphicsSystemSession::CreateNativeImage, "CreateNativeImage"}, {430, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyNativeImage"}, {426, &INx2GraphicsSystemSession::QueryNativeImage, "MapNativeImage"}, + {427, &INx2GraphicsSystemSession::UnmapNativeImage, "UnmapNativeImage"}, {428, &INx2GraphicsSystemSession::QueryNativeImage, "QueryNativeImage"}, {435, &INx2GraphicsSystemSession::CalculateNativeImageSize, "CalculateNativeImageSize"}, {1402, &INx2GraphicsSystemSession::OpenNvsrvSession, "OpenNvsrvSession (guessed)"}, @@ -2597,8 +2736,19 @@ private: void DestroyNativeMemory(HLERequestContext& ctx) { const u32 id = ctx.CommandBuffer()[2]; - std::scoped_lock lk{m_objects->mutex}; - m_objects->memories.erase(id); + std::vector<std::optional<std::pair<u64, u64>>> unmaps; + { + std::scoped_lock lk{m_objects->mutex}; + for (auto& [image_id, image] : m_objects->images) { + if (image.memory == id && image.mapped_here) { + unmaps.push_back(TakeDecoderMapping(image)); + } + } + m_objects->memories.erase(id); + } + for (const auto& range : unmaps) { + DropDecoderMapping(range); + } LOG_DEBUG(Service, "nx2:gsv 112 DestroyNativeMemory id {}", id); TipcReplyWords(ctx, {}); } @@ -2673,6 +2823,9 @@ private: // the 429 preference words (0x10, 0x14), a guess. void QueryNativeImage(HLERequestContext& ctx) { const u32 handle = ctx.CommandBuffer()[2]; + if (QueryDecoderImage(ctx, handle)) { + return; + } std::array<u32, 16> preference{}; u64 image_offset = 0; u64 memory_address = 0; @@ -2717,6 +2870,108 @@ private: } } + // 428/426 for a decoder output image (Fire Emblem: Fortune's Weave's movie player blits each + // one, block-linear RGBA, format 0xd5, block height 16 GOBs, into a texture of its own). The + // reply follows our Eden tree, where the movies play: words 3-4 the GPU address, 5-6 the last + // byte's address, then the descriptor {1, 0, dimensions, .., 0x01000001, 1, offset, size} + // (offset 0 once mapped). 426 maps the image at its own address for the GPU: decoder pool + // memory is process memory (114) that no GPU mapping covers, and the blit read zeros. + bool QueryDecoderImage(HLERequestContext& ctx, u32 handle) { + const bool map = ctx.GetCommand() == 426; + u64 address{}; + u64 size{}; + u64 offset{}; + u32 dimensions{}; + bool register_mapping = false; + { + std::scoped_lock lk{m_objects->mutex}; + const auto image = m_objects->images.find(handle); + if (image == m_objects->images.end() || !image->second.decoder) { + return false; + } + const auto memory = m_objects->memories.find(image->second.memory); + if (memory == m_objects->memories.end() || memory->second.address == 0) { + return false; + } + image->second.gpu_read = true; + offset = image->second.offset; + size = image->second.size; + dimensions = image->second.preference[2]; + if (map) { + address = memory->second.address + offset; + register_mapping = image->second.gpu_maps++ == 0 && + !Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address); + image->second.mapped_here |= register_mapping; + } + } + if (register_mapping) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Register(address, size); + } + LOG_INFO(Service, "nx2:gsv {} decoder image {} at {:#x}+{:#x}{}", ctx.GetCommand(), handle, + address, size, register_mapping ? ", mapped for the GPU" : ""); + std::array<u32, 37> words{}; + if (map) { + const u64 last = address + size - 1; + words[0] = static_cast<u32>(address); + words[1] = static_cast<u32>(address >> 32); + words[2] = static_cast<u32>(last); + words[3] = static_cast<u32>(last >> 32); + } + u32* const desc = words.data() + 4; + desc[0] = 1; + desc[2] = dimensions; + desc[8] = 0x01000001; + desc[9] = 1; + desc[10] = map ? 0 : static_cast<u32>(offset); + desc[11] = static_cast<u32>(size); + IPC::ResponseBuilder rb{ctx, static_cast<u32>(2 + words.size())}; + rb.PushRaw<u32>(ResultSuccess.raw); + for (const u32 w : words) { + rb.Push<u32>(w); + } + return true; + } + + // 427(u32 handle, u32 address space, u64 address, u64 last address): undoes a 426. The last + // unmap of a decoder image drops the GPU mapping 426 made. + void UnmapNativeImage(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u32 handle = w[2]; + std::optional<std::pair<u64, u64>> unmap; + { + std::scoped_lock lk{m_objects->mutex}; + if (const auto image = m_objects->images.find(handle); + image != m_objects->images.end() && image->second.gpu_maps != 0 && + --image->second.gpu_maps == 0) { + unmap = TakeDecoderMapping(image->second); + } + } + LOG_DEBUG(Service, "nx2:gsv 427 image {} [{:#x}, {:#x}]", handle, Pair(w, 4), Pair(w, 6)); + DropDecoderMapping(unmap); + TipcReplyWords(ctx, {}); + } + + // The range 426 mapped for an image, if it did; called with m_objects->mutex held. + std::optional<std::pair<u64, u64>> TakeDecoderMapping(Nx2NativeObjects::Image& image) { + image.gpu_maps = 0; + if (!std::exchange(image.mapped_here, false)) { + return std::nullopt; + } + const auto memory = m_objects->memories.find(image.memory); + if (memory == m_objects->memories.end() || memory->second.address == 0) { + return std::nullopt; + } + return std::pair{memory->second.address + image.offset, image.size}; + } + + static void DropDecoderMapping(const std::optional<std::pair<u64, u64>>& range) { + if (range) { + auto& registry = Nvidia::Devices::Nx2GpuMemoryRegistry::Get(); + registry.Unmap(range->first, range->second); + registry.ReleaseBacking(range->first, range->second); + } + } + // 911(preference[16], u32 count, u32 memory id) (nn::codec::InitializeDecoderImagePool): // result, the id of the first of `count` decoder output images; the client names image i of // the pool id + i. They share the pool's memory, one image size apart. @@ -2775,8 +3030,15 @@ private: void DestroyNativeImage(HLERequestContext& ctx) { const u32 id = ctx.CommandBuffer()[2]; - std::scoped_lock lk{m_objects->mutex}; - m_objects->images.erase(id); + std::optional<std::pair<u64, u64>> unmap; + { + std::scoped_lock lk{m_objects->mutex}; + if (const auto image = m_objects->images.find(id); image != m_objects->images.end()) { + unmap = TakeDecoderMapping(image->second); + m_objects->images.erase(image); + } + } + DropDecoderMapping(unmap); TipcReplyWords(ctx, {}); } diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h index 031f7fac75..489d003e60 100644 --- a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h +++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h @@ -56,7 +56,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { for (const auto& instruction : code) { const u32 op = instruction.Bits(0, 12); uses_uniform_bank |= op == 0x3c4 || op == 0x2bd || op == 0xc0c || op == 0x986 || op == 0x9a8 || op == 0xc12 || op == 0xc17 || op == 0xe24 || op == 0xc24 || op == 0xd00 || op == 0xd09 || op == 0x9c3 || op == 0xc45 || op == 0x2a4 || op == 0x891 || op == 0x890 || op == 0x290 || op == 0x28c || op == 0x88c || op == 0x287 || op == 0x887 || op == 0x958 || op == 0xab9 || op == 0xabb || op == 0x899 || op == 0xc20 || op == 0xe21 || op == 0xe23 || op == 0xc02 || - op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || op == 0x292 || op == 0x892 || op == 0x291 || op == 0x981 || op == 0xc13 || op == 0xc10 || op == 0xc23 || op == 0x526 || op == 0x2be || op == 0x8a4 || op == 0xd05 || (op == 0x984 && instruction.Bits(91, 1)) || op == 0x89c || op == 0x883 || op == 0x2a5 || op == 0xd06 || op == 0x299 || op == 0xc09 || (op == 0x81c && instruction.Bits(67, 1)); + op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || op == 0x292 || op == 0x892 || op == 0x291 || op == 0x981 || op == 0xc13 || op == 0xc10 || op == 0xc23 || op == 0x526 || op == 0x2be || op == 0x8a4 || op == 0xd05 || (op == 0x984 && instruction.Bits(91, 1)) || op == 0x89c || op == 0x883 || op == 0x2a5 || op == 0xd06 || op == 0x299 || op == 0xc09 || (op == 0x81c && instruction.Bits(67, 1)) || op == 0xf60 || op == 0xc11 || op == 0xc0a; } const auto address_of = [](u32 index) { return 0x50U + index * 8 + ((index + 1) / 3) * 8; }; const auto fail = [&](u32 index, std::string_view reason) { @@ -92,14 +92,20 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { active_link = call_links[call]; } } - // RET has no destination GPR; bits 16..23 are reserved, not a write to R0. - if (active_link && opcode != 0x950 && insn.Bits(16, 8) == *active_link) + // RET, BRA, NOP and ISETP/FSETP (predicate outputs) have no destination GPR; bits 16..23 + // are not a write to R0. IMAD.WIDE writes a register pair. + const bool no_destination = opcode == 0x950 || opcode == 0x947 || opcode == 0x918 || + (opcode & 0x1ff) == 0x00c || (opcode & 0x1ff) == 0x00b; + const bool pair = opcode == 0x225 || opcode == 0x625 || opcode == 0x825; + if (active_link && !no_destination && + (insn.Bits(16, 8) == *active_link || (pair && insn.Bits(16, 8) + 1 == *active_link))) return fail(index, "native callee overwrites link register"); } // Scratch values and emulated uniform state occupy reserved high GPRs. // Reject native accesses that could alias that state. for (u32 bit : {16U, 24U, 32U, 64U}) { - if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66) continue; + if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66 && opcode != 0x361 && + opcode != 0xf60) continue; const u32 reg = insn.Bits(bit, 8); if (reg >= (uses_uniform_bank ? 216U : 246U) && reg < 255) return fail(index, "native program uses lowering scratch registers"); } @@ -163,6 +169,13 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { }; std::string_view operand_error; const auto emit = [&](u64 word) { result.instructions.push_back(word); }; + // Bound Maxwell texture instructions share one descriptor constant buffer per program. + const auto bind_texture_buffer = [&](u32 binding) { + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return false; + result.required_texture_buffer = binding; + return true; + }; // P(guard slot) while a uniform guard is swapped in; P6 for lowerings that use it as // scratch (R2UR, MATCH, HFMA2.RELU), which save and restore it with the defaults below. const u32 guard_mask = guard_slot < 7 ? 1U << guard_slot : 0U; @@ -466,6 +479,20 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.Bits(64, 8)} << 20)); break; } + if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 && (insn.words[2] & ~0x07800affU) == 0x000e0400 && + (!rewritten_uniform || insn.Bits(87, 3) == 7)) { + // IMAD.X RZ, RZ, c, Pn: the high word of a 64-bit add (bit 75 inverts c). + u32 c = insn.Bits(64, 8); + mov_imm(250, 1); + emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + if (insn.Bits(75, 1)) { + emit(0x0400000000000000ULL | predicate | 251 | (u64{c} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53)); + c = 251; + } + emit(base(0x5c10000000000000ULL) | (u64{c} << 8) | (u64{250} << 20)); + break; + } if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (insn.words[1] & ~0xffU)) return fail(index, "native integer register multiply mode"); multiply_low(insn.Bits(24, 8), insn.Bits(32, 8), insn.Bits(64, 8), false); @@ -496,16 +523,87 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0x5c10000000070000ULL | d_low | (u64{250} << 8) | (u64{255} << 20)); break; } - case 0x827: { // IMAD.HI.U32, captured zero addend - if (insn.words[2] != 0x078e00ff) return fail(index, "native high multiply mode"); - const u32 a = insn.Bits(24, 8), b = insn.words[1]; + case 0x226: { // IDP.2A.LO.U16.U8: a.lo16 * b.byte0 + a.hi16 * b.byte1 + c + if ((insn.words[2] & ~0xffU) != 0x1000 || (insn.words[1] & ~0xffU)) + return fail(index, "native packed dot-product mode"); + const u32 a = insn.Bits(24, 8), b = insn.Bits(32, 8), c = insn.Bits(64, 8); + for (u32 component = 0; component < 2; ++component) { + emit(0x3828000000000000ULL | predicate | (250 + component) | (u64{b} << 8) | (u64{component * 8} << 20)); + emit(0x0400000000000000ULL | predicate | (250 + component) | (u64{250 + component} << 8) | (u64{0xff} << 20)); + } + xmad(252, a, 250, c, false, false, 0, false); + xmad(static_cast<u32>(dest), a, 251, 252, true, false, 0, false); + break; + } + case 0x225: case 0x227: case 0x625: case 0x825: case 0x827: case 0xa27: { + // IMAD.HI Rd / IMAD.WIDE Rd:Rd+1 = a * b + c:c+1 (.U32 or signed, bit 73), with an optional + // carry-out (bits 81-83). b is a register, an immediate (0x8xx) or a cbuf (0xa27); 0x625 takes + // b from bits 64-71 and adds a cbuf pair. + const bool wide = opcode == 0x225 || opcode == 0x625 || opcode == 0x825; + const bool immediate = opcode == 0x825 || opcode == 0x827, signed_product = insn.Bits(73, 1); + const u32 c = opcode == 0x625 ? 255U : insn.Bits(64, 8), carry_output = insn.Bits(81, 3); + if ((insn.words[2] & ~0x000e02ffU) != 0x07800000 || + ((opcode == 0x225 || opcode == 0x227) && (insn.words[1] & ~0xffU)) || + ((opcode == 0xa27 || opcode == 0x625) && (!valid_cbuf() || insn.Bits(62, 2))) || + (opcode == 0x625 && insn.Bits(38, 16) > 0xfff8) || (c != 255 && (c & 1)) || + (wide && (dest == 255 || (dest & 1))) || (signed_product && carry_output != 7)) + return fail(index, "native high multiply mode"); + const u32 a = insn.Bits(24, 8); + const u32 b = immediate ? insn.words[1] : opcode == 0xa27 ? 254U : insn.Bits(opcode == 0x625 ? 64 : 32, 8); + if (opcode == 0xa27) emit(0x4c98000000000000ULL | predicate | 254 | (u64{15} << 39) | cbuf()); + if (signed_product) { + // The signed high word is the unsigned one minus (a < 0 ? b : 0) + (b < 0 ? a : 0), + // which R253 holds (R252 is free until the multiplication). + const u32 b_register = immediate ? 254U : b; + if (immediate) mov_imm(254, b); + emit(0x3828000000000000ULL | predicate | 253 | (u64{a} << 8) | (u64{31} << 20) | (u64{1} << 48)); + emit(0x5c40000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{b_register} << 20) | (u64{7} << 48)); + emit(0x3828000000000000ULL | predicate | 252 | (u64{b_register} << 8) | (u64{31} << 20) | (u64{1} << 48)); + emit(0x5c40000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{a} << 20) | (u64{7} << 48)); + emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{252} << 20)); + } // Base-2^16 long multiplication. Each intermediate sum fits U32. - xmad(250, a, b, 255, false, false, 0, true); - xmad(251, a, b, 250, true, false, 2, true); - xmad(250, a, b, 251, false, true, 1, true); - xmad(252, a, b, 251, true, true, 2, true); + xmad(250, a, b, 255, false, false, 0, immediate); + xmad(251, a, b, 250, true, false, 2, immediate); + xmad(250, a, b, 251, false, true, 1, immediate); + xmad(252, a, b, 251, true, true, 2, immediate); emit(0x3828000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{16} << 20)); - emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20)); + if (signed_product) + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{253} << 20) | (u64{1} << 48)); + if (!wide && c == 255 && carry_output == 7) { + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20)); + break; + } + // High word in R253 and low word in R250 (as UIMAD.WIDE.U32), then c:c+1 is added. + emit(0x5c10000000000000ULL | predicate | 253 | (u64{250} << 8) | (u64{252} << 20)); + xmad(250, a, b, 255, false, false, 0, immediate); + xmad(251, a, b, 255, true, false, 0, immediate); + xmad(251, a, b, 251, false, true, 0, immediate); + emit(0x5c18000000000000ULL | predicate | 250 | (u64{251} << 8) | (u64{250} << 20) | (u64{16} << 39)); + u32 c_low = c, c_high = c == 255 ? 255U : c + 1; + if (opcode == 0x625) { + emit(0x4c98000000000000ULL | predicate | 252 | (u64{15} << 39) | cbuf()); + emit(0x4c98000000000000ULL | predicate | 254 | (u64{15} << 39) | (cbuf() + (u64{1} << 20))); + c_low = 252; + c_high = 254; + } + const u32 high = wide ? static_cast<u32>(dest) + 1 : static_cast<u32>(dest); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{c_low} << 20) | (u64{1} << 47)); + if (carry_output == 7) { + emit(0x5c10000000000000ULL | predicate | high | (u64{253} << 8) | (u64{c_high} << 20) | (u64{1} << 43)); + if (wide) emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{255} << 20)); + break; + } + // Maxwell IADD has no .X with .CC: R251 takes the low carry, R254 counts the high word's. + emit(0x5c10000000000000ULL | predicate | 251 | (u64{255} << 8) | (u64{255} << 20) | (u64{1} << 43)); + emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{c_high} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 254 | (u64{255} << 8) | (u64{255} << 20) | (u64{1} << 43)); + emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{251} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 254 | (u64{254} << 8) | (u64{255} << 20) | (u64{1} << 43)); + emit(0x5c98000000000000ULL | predicate | high | (u64{253} << 20) | (u64{15} << 39)); + if (wide) emit(base(0x5c98000000000000ULL) | (u64{250} << 20) | (u64{15} << 39)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{carry_output} << 3) | (u64{254} << 8) | + (u64{255} << 20) | (u64{7} << 39) | (u64{5} << 49)); break; } case 0xa24: { // IMAD, constant-buffer multiplier @@ -547,6 +645,21 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } case 0x824: { // Captured low-word IMAD forms; bit 75 negates the addend. const bool negate_c = insn.Bits(75, 1) != 0; + if (insn.words[1] == 1 && (insn.words[2] & ~(0x07800800U | 0xffU)) == 0x000e0600U) { + // IMAD.X Rd, Ra, 1, Rc, Pn: Ra + Rc plus the carry predicate in bits 87-90 (~Rc with bit 75). + u32 c = insn.Bits(64, 8); + mov_imm(252, 1); + emit(0x5ca0000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + if (negate_c) { + emit(0x0400000000000000ULL | predicate | 251 | (u64{c} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53)); + c = 251; + } + emit(0x5c10000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | + (u64{c} << 20)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20)); + break; + } if (insn.words[1] == 1 && (insn.words[2] & 0xfffff700U) == 0x078e0200) { emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(64, 8)} << 20) | (u64{negate_c} << 48)); @@ -708,27 +821,27 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } case 0x899: { // USHF.L.U32; accept a zero word outside the selected half. const u32 dst = insn.Bits(16, 8), low = insn.Bits(24, 8), high = insn.Bits(64, 8); - if (insn.words[2] == (0x08011400U | high) && low == 63 && high < 16 && - dst < 16 && native_predicate == 7 && insn.words[1] <= 32) { - emit(0x3828000000000000ULL | predicate | (dst + 216) | - (u64{high + 216} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48)); + if (insn.words[2] == (0x08011400U | high) && low == 63 && high < 28 && + dst < 28 && native_predicate == 7 && insn.words[1] <= 32) { + emit(0x3828000000000000ULL | predicate | uniform_register(dst) | + (u64{uniform_register(high)} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48)); break; } // Logical right shifts of one register (bit 76 right, bit 80 high word, bits 73-74 // U32): USHF.R.U32 URd, URa, n, URZ and USHF.R.U32.HI URd, URZ, n, URb. - const bool right_low = insn.words[2] == 0x0800163f && low < 16; - const bool right_high = insn.words[2] == (0x08011600U | high) && low == 63 && high < 16; - if ((right_low || right_high) && dst < 16 && native_predicate == 7 && insn.words[1] <= 32) { - emit(0x3828000000000000ULL | predicate | (dst + 216) | - (u64{(right_high ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20)); + const bool right_low = insn.words[2] == 0x0800163f && low < 28; + const bool right_high = insn.words[2] == (0x08011600U | high) && low == 63 && high < 28; + if ((right_low || right_high) && dst < 28 && native_predicate == 7 && insn.words[1] <= 32) { + emit(0x3828000000000000ULL | predicate | uniform_register(dst) | + (u64{uniform_register(right_high ? high : low)} << 8) | (u64{insn.words[1]} << 20)); break; } - const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high < 16; - const bool low_result = insn.words[2] == 0x0800063f && low < 16; - if (dst >= 16 || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result)) + const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high < 28; + const bool low_result = insn.words[2] == 0x0800063f && low < 28; + if (dst >= 28 || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result)) return fail(index, "native uniform shift mode"); - emit(0x3848000000000000ULL | predicate | (dst + 216) | - (u64{(high_result ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20)); + emit(0x3848000000000000ULL | predicate | uniform_register(dst) | + (u64{uniform_register(high_result ? high : low)} << 8) | (u64{insn.words[1]} << 20)); break; } case 0xb82: { // LDC.64 indexed byte offset @@ -759,16 +872,18 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (u64{insn.Bits(64, 8)} << 20) | (u64{maxwell} << 28)); break; } - case 0xb6c: { // TXD 2D or ARRAY_2D, four channels; the two destination pairs anywhere. + case 0xb6c: { // TXD 2D or ARRAY_2D, any channels; the two destination pairs anywhere. const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); const u32 coordinates = insn.Bits(24, 8), derivatives = insn.Bits(32, 8); // Ampere dimensions: 1 = 2D, 5 = ARRAY_2D; Maxwell TXD's are 2 and 3. The coordinates // (x, y[, layer]) and the four gradients sit in the same registers for both. const u32 dimension = insn.Bits(61, 3); const u32 high = insn.Bits(64, 8); // destination of the third and fourth channels - if ((insn.words[2] & ~0xffU) != 0x001e0f00 || insn.Bits(59, 2) || + const u32 mask = insn.Bits(72, 4), channels = static_cast<u32>(std::popcount(mask)); + if ((insn.words[2] & ~(0xfffU | (1U << 26))) != 0x001e0000 || mask == 0 || insn.Bits(59, 2) || (dimension != 1 && dimension != 5) || binding >= 18 || offset > 0x1fff || - (dest & 1) || dest >= 248 || (high & 1) || (high >= 248 && high != 255) || + (channels > 1 && (dest & 1)) || dest >= 248 || + (channels > 2 ? (high & 1) || high >= 248 : high != 255) || coordinates >= 248 || derivatives >= 247) return fail(index, "native gradient texture mode"); if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) @@ -776,16 +891,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { result.required_texture_buffer = binding; // Maxwell writes four consecutive registers: when the second pair is elsewhere (Splatoon // Raiders: R12/R18, R18/R44), write them to scratch and move each pair into place. - const bool consecutive = high == dest + 2; + const bool consecutive = channels <= 2 || high == dest + 2; const u64 target = consecutive ? dest : 248; emit(0xde00000000000000ULL | predicate | target | (u64{coordinates} << 8) | - (u64{derivatives} << 20) | (u64{dimension == 5 ? 3U : 2U} << 28) | (u64{15} << 31) | + (u64{derivatives} << 20) | (u64{dimension == 5 ? 3U : 2U} << 28) | (u64{mask} << 31) | (u64{offset} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51)); if (!consecutive) { - for (u32 i = 0; i < 2; ++i) { - emit(0x5c98000000000000ULL | predicate | (dest + i) | (u64{15} << 39) | (u64{248 + i} << 20)); - if (high != 255) - emit(0x5c98000000000000ULL | predicate | (high + i) | (u64{15} << 39) | (u64{250 + i} << 20)); + for (u32 i = 0; i < channels; ++i) { + emit(0x5c98000000000000ULL | predicate | (i < 2 ? dest + i : high + i - 2) | + (u64{15} << 39) | (u64{248 + i} << 20)); } } break; @@ -836,44 +950,53 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0x3828000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 8) | (u64{1} << 39) | (u64{1} << 40)); break; - case 0x3a1: { // MATCH.ALL.U64 with only the all-equal predicate retained. + case 0x3a1: { // MATCH.ALL.U32/U64 (bit 73) with only the all-equal predicate retained. const u32 source = insn.Bits(24, 8), target = insn.Bits(81, 3); - if (dest != 255 || source >= 254 || (source & 1) || target >= 6 || - native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0000U) != 0x200) + const bool wide = insn.Bits(73, 1); + if (dest != 255 || source >= 254 || (wide && (source & 1)) || target >= 6 || + native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0200U)) return fail(index, "native warp match mode"); save_predicate(); emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49)); emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48)); emit(0x5c30000000070000ULL | 250 | (u64{250} << 20)); - for (u32 word = 0; word < 2; ++word) + for (u32 word = 0; word < (wide ? 2U : 1U); ++word) emit(0xef10000000070000ULL | (251 + word) | (u64{source + word} << 8) | (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source} << 8) | (u64{251} << 20) | (u64{7} << 39) | (u64{2} << 49)); - emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source + 1} << 8) | - (u64{252} << 20) | (u64{6} << 39) | (u64{2} << 49)); + if (wide) + emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source + 1} << 8) | + (u64{252} << 20) | (u64{6} << 39) | (u64{2} << 49)); emit(0x50d8000000070000ULL | 255 | (u64{6} << 39) | (u64{target} << 45)); restore_predicate(); break; } - case 0x3c4: { // REDUX.SUM (modulo 2^32) or .MAX (unsigned) over the actual active lanes. + case 0x3c4: { // REDUX.SUM (modulo 2^32), .OR, .MIN or .MAX (unsigned) over the actual active lanes. const u32 source = insn.Bits(24, 8), target = uniform_register(static_cast<u32>(dest)); - const bool maximum = insn.words[2] == 0x14000; + const bool maximum = insn.words[2] == 0x14000, minimum = insn.words[2] == 0x10000; + const bool bitwise_or = insn.words[2] == 0x4000; if (target == ~0U || target == 255 || insn.words[1] || - (insn.words[2] != 0xc000 && !maximum) || native_predicate != 7) + (insn.words[2] != 0xc000 && !maximum && !minimum && !bitwise_or) || native_predicate != 7) return fail(index, "native warp sum mode"); emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); - mov_imm(target, 0); + mov_imm(target, minimum ? ~0U : 0U); for (u32 lane = 0; lane < 32; ++lane) { emit(0xef10000000070000ULL | 251 | (u64{source} << 8) | (u64{lane} << 20) | (u64{1} << 28) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); emit(0x3828000000070000ULL | 252 | (u64{250} << 8) | (u64{lane} << 20)); emit(0x0400000000070000ULL | 252 | (u64{252} << 8) | (u64{1} << 20)); emit(0x5c10000000070000ULL | 252 | (u64{255} << 8) | (u64{252} << 20) | (u64{1} << 48)); - emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48)); - if (maximum) { - emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | (u64{15} << 39)); + // Inactive lanes give the identity: zero, or all ones for MIN (OR with ~mask). + emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | + (minimum ? (u64{1} << 40) | (u64{1} << 41) : 0) | (u64{7} << 48)); + if (maximum || minimum) { + emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | + (u64{maximum ? 15U : 7U} << 39)); + } else if (bitwise_or) { + emit(0x5c40000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | + (u64{1} << 41) | (u64{7} << 48)); } else { emit(0x5c10000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20)); } @@ -894,12 +1017,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0xec00000000000000ULL | predicate | 255 | (u64{address} << 8) | (u64{250} << 20)); break; } - case 0x38c: { // ATOMS.ADD.U32, naturally aligned shared byte offset. - const u32 offset = insn.Bits(40, 24); - if (insn.words[2] || (offset & 3)) return fail(index, "native shared atomic mode"); + case 0x38c: { // ATOMS.ADD/MIN/MAX/OR.U32, naturally aligned shared byte offset. + // The operation (bits 87-90) numbers ADD 0, MIN 1, MAX 2, OR 6 as Maxwell's (bits 52-55). + const u32 offset = insn.Bits(40, 24), operation = insn.Bits(87, 4); + if ((operation != 0 && operation != 1 && operation != 2 && operation != 6) || + insn.words[2] != (operation << 23) || (offset & 3)) + return fail(index, "native shared atomic mode"); emit(0xec00000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) | - (u64{offset >> 2} << 30)); + (u64{offset >> 2} << 30) | (u64{operation} << 52)); break; } case 0x3a8: { // ATOMG.E.ADD.STRONG.GPU U32 at a 64-bit register address plus an offset. @@ -944,14 +1070,20 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); const u32 data = insn.Bits(32, 8), coordinate = insn.Bits(24, 8); const u32 dimension = insn.Bits(61, 3); - if (insn.words[2] != 0x00100f00 || dimension < 3 || dimension > 5 || + const bool strong = insn.words[2] == 0x0010af00; // .STRONG.GPU: fenced on both sides + if ((insn.words[2] != 0x00100f00 && !strong) || dimension < 3 || dimension > 5 || binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || data > 252 || coordinate > (dimension >= 4 ? 252U : 253U)) return fail(index, "native formatted surface store mode"); - if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) - return fail(index, "native surface descriptor buffer conflict"); - result.required_texture_buffer = binding; + // A second descriptor buffer: SUST.B with the handle, loaded from that buffer, in R250. + const bool bindless = !bind_texture_buffer(binding); + if (bindless) { + emit(0x4c98000000070000ULL | 250 | (u64{15} << 39) | (u64{offset} << 20) | (u64{binding} << 34)); + } + if (strong) emit(0xef98000000000000ULL | predicate | (u64{1} << 8)); // MEMBAR.GL emit(0xeb20000000000000ULL | predicate | data | (u64{coordinate} << 8) | - (u64{15} << 20) | (u64{dimension} << 33) | (u64{offset} << 36) | (u64{1} << 51)); + (u64{15} << 20) | (u64{dimension} << 33) | + (bindless ? u64{250} << 39 : (u64{offset} << 36) | (u64{1} << 51))); + if (strong) emit(0xef98000000000000ULL | predicate | (u64{1} << 8)); break; } case 0x301: // BREV register. @@ -959,21 +1091,27 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(base(0x3828000000000000ULL) | (u64{insn.words[1]} << 8) | (u64{1} << 39) | (u64{1} << 40)); break; - case 0x219: case 0xc19: { // U32 register (or uniform, 0xc19) shifts with a zero word outside the selected result. + case 0x219: case 0xa19: case 0xc19: { // U32 register (or cbuf, 0xa19, or uniform, 0xc19) shifts with a zero word outside the selected result. const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8); - const u32 shift = opcode == 0xc19 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + const bool constant = opcode == 0xa19; + // A constant shift amount is loaded into R251 once the form is known. + const u32 shift = opcode == 0xc19 ? uniform_register(insn.Bits(32, 8)) : constant ? 251U : insn.Bits(32, 8); + const bool shift_operand = constant ? valid_cbuf() && !insn.Bits(62, 2) : insn.words[1] <= 255; + const auto load_shift = [&] { if (constant) materialize(2, 0, 251, false, false); }; const bool right = insn.Bits(76, 1) != 0, high_result = insn.Bits(80, 1) != 0; // SHF.R.U32 of the low word with a zero high word: a plain logical right shift. if (right && !high_result && high == 255 && insn.Bits(73, 2) == 3 && shift != ~0U && - insn.words[1] <= 255 && + shift_operand && (insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) == 0x00000600U) { + load_shift(); emit(base(0x5c28000000000000ULL) | (u64{low} << 8) | (u64{shift} << 20)); break; } - if (insn.words[1] > 255 || insn.Bits(73, 2) != 3 || shift == ~0U || + if (!shift_operand || insn.Bits(73, 2) != 3 || shift == ~0U || (insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) != 0x00000600U || (right ? (!high_result || low != 255) : (high_result || high != 255))) return fail(index, "native register shift mode"); + load_shift(); emit(base(right ? 0x5cf8000000000000ULL : 0x5bf8000000000000ULL) | (u64{right ? high : 255U} << 8) | (u64{shift} << 20) | (u64{right ? 255U : low} << 39) | (u64{insn.Bits(75, 1)} << 50)); @@ -1012,8 +1150,8 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { if (insn.words[1] || insn.words[2]) return fail(index, "native error barrier mode"); emit(0xef98000000000000ULL | predicate | (u64{1} << 8)); break; - case 0xb1d: // BAR.SYNC 0, full workgroup. - if (insn.words[1] || insn.words[2] || native_predicate != 7) + case 0xb1d: // BAR.SYNC 0 (bit 80: .DEFER_BLOCKING), full workgroup. + if (insn.words[1] || (insn.words[2] & ~0x10000U) || native_predicate != 7) return fail(index, "native workgroup barrier mode"); emit(0xf0a8000000000000ULL | predicate | (u64{1} << 43) | (u64{1} << 44) | (u64{1} << 39) | (u64{7} << 39)); @@ -1257,10 +1395,17 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { // The uniform base sits where a register source would (bits 32-37). const u32 uniform_address = insn.Bits(38, 2) ? ~0U : uniform_register(insn.Bits(32, 6)); - if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U | 0x2000U)) != 0x081e8100 || + // desc[URn][Ra.64]: the uniform pair is a memory (cache policy) descriptor and + // Ra the whole address. + const bool descriptor_address = (insn.words[2] & ~(0x3fU | 0xe00U)) == 0x0c1e1100; + if ((!descriptor_address && + (insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U | 0x2000U)) != 0x081e8100) || uniform_address == ~0U || (uniform_address & 1)) return fail(index, "native global load uniform addressing mode"); - if (address == 255) { + if (descriptor_address) { + if ((address & 1) || address >= 246) + return fail(index, "native global load register address pair"); + } else if (address == 255) { address = uniform_address; } else { if (insn.Bits(90, 1) && ((address & 1) || address >= 246)) @@ -1293,10 +1438,16 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { bool extended = insn.Bits(72, 1) != 0; if (opcode == 0x986) { const u32 uniform_address = uniform_register(insn.Bits(64, 6)); - if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U)) != 0x0810a100 || + // desc[URn][Ra.64]: the uniform pair is a memory (cache policy) descriptor and + // Ra the whole address. + const bool descriptor_address = (insn.words[2] & ~(0x3fU | 0xe00U)) == 0x0c101100; + if ((!descriptor_address && (insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U)) != 0x0810a100) || uniform_address == ~0U || (uniform_address & 1) || !extended) return fail(index, "native global store uniform addressing mode"); - if (address == 255) address = uniform_address; + if (descriptor_address) { + if ((address & 1) || address >= 246) + return fail(index, "native global store register address pair"); + } else if (address == 255) address = uniform_address; else { if (insn.Bits(90, 1) && ((address & 1) || address >= 254)) return fail(index, "native global store register address pair"); @@ -1319,13 +1470,29 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { return fail(index, "native fragment discard mode"); emit(0xe33000000000000fULL | predicate); break; + case 0x311: { // F2I.FTZ.U64.TRUNC, register source, destination pair + if (insn.words[2] != 0x0021d800 || (insn.words[1] & ~0xffU) || insn.Bits(24, 8) || + (dest & 1) || dest >= 246) + return fail(index, "native unsigned 64-bit float conversion mode"); + emit(base(0x5cb0000000000000ULL) | (u64{3} << 8) | (u64{2} << 10) | + (u64{insn.Bits(32, 8)} << 20) | (u64{3} << 39) | (u64{1} << 44)); + break; + } + case 0x312: { // I2F.F32.(U|S)64, register pair source; sign (74) and rounding (78-79) as Maxwell's + if ((insn.words[2] & ~0xc400U) != 0x00301000 || (insn.words[1] & ~0xffU) || insn.Bits(24, 8) || + (insn.Bits(32, 8) & 1) || insn.Bits(32, 8) >= 254) + return fail(index, "native 64-bit integer-to-float mode"); + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{3} << 10) | (u64{insn.Bits(74, 1)} << 13) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(78, 2)} << 39)); + break; + } case 0x305: case 0xb05: case 0xd05: { // F2I NTZ, register/constant-buffer/uniform source const u32 mode = insn.words[2] & ~(opcode == 0xd05 ? 0x08000000U : 0U); const bool half = opcode == 0x305 && mode == 0x0010f000; const bool small = mode == 0x00212800; const bool floor = (mode & ~0x10100U) == 0x00207000; const bool nearest = (mode & ~0x10100U) == 0x00203000; - if ((!half && !small && !floor && !nearest && (mode & ~0x100U) != 0x0021f000) || + if ((!half && !small && !floor && !nearest && (mode & ~0x10100U) != 0x0020f000) || (opcode != 0xb05 && (insn.words[1] & 0xffffff00U)) || (opcode == 0xd05 && uniform_register(insn.Bits(32, 8)) == ~0U)) { return fail(index, "native float-to-integer conversion mode"); @@ -1363,17 +1530,28 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0xb6f: { // TXQ texture dimensions, two packed scalar outputs. - if (insn.Bits(59, 5) || insn.Bits(64, 8) != 255 || insn.Bits(72, 4) == 0 || insn.Bits(72, 4) > 3 || - insn.Bits(76, 14) || insn.Bits(91, 1) || insn.Bits(54, 5) != 2 || - insn.Bits(40, 14) > 0x1fff || (insn.Bits(72, 4) == 3 && (dest & 1)) || dest >= 254) + case 0xb6f: { // TXQ texture dimensions (mask 1-3) or mip level count (mask 8). + const u32 mask = insn.Bits(72, 4); + if (insn.Bits(59, 5) || insn.Bits(64, 8) != 255 || (mask == 0 || (mask > 3 && mask != 8)) || + insn.Bits(76, 14) || insn.Bits(91, 1) || insn.Bits(54, 5) >= 18 || + insn.Bits(40, 14) > 0x1fff || (mask == 3 && (dest & 1)) || dest >= 254) return fail(index, "native texture dimension query mode"); - result.required_texture_buffer = 2; + if (!bind_texture_buffer(insn.Bits(54, 5))) + return fail(index, "native texture descriptor buffer conflict"); emit(0xdf48000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | - (u64{1} << 22) | (u64{insn.Bits(72, 4)} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{1} << 22) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49)); break; } + case 0x370: { // TXQ.B dimensions or mip count: handle in Ra, mip level in Ra+1. + const u32 mask = insn.Bits(72, 4), source = insn.Bits(24, 8); + if (insn.words[1] != 0x08000000 || insn.Bits(64, 8) != 255 || (mask == 0 || (mask > 3 && mask != 8)) || + (insn.words[2] & ~0x04000f00U) != 255 || source > 245 || (mask == 3 && (dest & 1)) || dest >= 254) + return fail(index, "native bindless texture dimension query mode"); + emit(0xdf50000000000000ULL | predicate | dest | (u64{source} << 8) | + (u64{1} << 22) | (u64{mask} << 31) | (u64{insn.Bits(90, 1)} << 49)); + break; + } case 0xb66: { // TLD.SCR.LZ, direct texel fetch const u32 mask = insn.Bits(72, 4); const u32 dimension = insn.Bits(61, 3); @@ -1402,9 +1580,25 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { return fail(index, "native texel fetch mode or descriptor binding"); } const u32 binding = insn.Bits(54, 5); - if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) - return fail(index, "native texture descriptor buffer conflict"); - result.required_texture_buffer = binding; + if (!bind_texture_buffer(binding)) { + // A second descriptor buffer: TLD.B (LZ) with the coordinates in R250.. (array + // index or Z after Ra) and the handle, loaded from that buffer, in R253. + const auto copy = [&](u32 target, u32 source) { + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20)); + }; + copy(250, source_a); + if (array || volume) copy(251, source_a + 1); + if (dimension != 0) copy(array || volume ? 252 : 251, source_b); + emit(0x4c98000000070000ULL | 253 | (u64{15} << 39) | (u64{insn.Bits(40, 14)} << 20) | + (u64{binding} << 34)); + emit(0xdd00000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{253} << 20) | + (u64{array ? 3U : volume ? 4U : dimension == 0 ? 0U : 2U} << 28) | (u64{mask} << 31) | + (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51)); + const u32 components = static_cast<u32>(std::popcount(mask)); + for (u32 i = 0; i < components; ++i) + copy(i < 2 || !split ? static_cast<u32>(dest) + i : second_dest + (i - 2), 250 + i); + break; + } if (array) { emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a + 1} << 20)); emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{source_b} << 20)); @@ -1475,7 +1669,10 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } + case 0x361: // TEX with the descriptor handle in a register (after the coordinates) + case 0xf60: // TEX with the descriptor handle in a uniform register case 0xb60: { // TEX, scalar 2D implicit-LOD sample + const bool register_handle = opcode == 0x361, uniform_handle = opcode == 0xf60; const u32 mask = insn.Bits(72, 4); const u32 second_dest = insn.Bits(64, 8); constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12}; @@ -1485,12 +1682,13 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { const bool depth_compare = insn.Bits(78, 1); const u32 dimension = insn.Bits(61, 3), lod = insn.Bits(87, 3); const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); - if (dimension == 5 && depth_compare && lod == 0 && !split && mask == 1) { + if (dimension == 5 && depth_compare && lod == 0 && !split && mask == 1 && !register_handle && + !uniform_handle) { if (!insn.Bits(60, 1) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || (source_a & 1) || (source_b & 1) || source_a >= 248 || source_b >= 248 || dest >= 248) return fail(index, "native array depth comparison sample mode"); - result.required_texture_buffer = 2; + if (!bind_texture_buffer(2)) return fail(index, "native texture descriptor buffer conflict"); for (u32 i = 0; i < 3; ++i) { const u32 source = i < 2 ? source_a + i : source_b; emit(0x5c98000000000000ULL | predicate | (250 + i) | @@ -1510,14 +1708,24 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { constexpr std::array<u32, 8> coordinate_count{1, 2, 3, 3, 2, 3, 0, 4}; constexpr std::array<u32, 8> maxwell_type{0, 2, 4, 6, 1, 3, 0, 7}; const bool offsets = insn.Bits(76, 1) != 0; - if (dimension == 6 || insn.Bits(77, 1) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || - insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 || (lod > 3 && lod != 6 && lod != 7) || - (depth_compare && mask != 1) || mask == 0) + // Handle forms: 0x361 has no descriptor fields (bit 59 set), 0xf60 names a UR + // in bits 40-45 (bit 91 set) and is only known with packed operands. + const u32 handle_register = uniform_handle ? uniform_register(insn.Bits(40, 6)) : ~0U; + if (register_handle ? !insn.Bits(59, 1) || insn.Bits(40, 19) + : uniform_handle ? !insn.Bits(60, 1) || insn.Bits(59, 1) || insn.Bits(46, 13) || + !insn.Bits(91, 1) || handle_register == ~0U || handle_register == 255 + : insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18) + return false; + if (dimension == 6 || insn.Bits(77, 1) || insn.Bits(81, 3) != 7 || + (lod > 3 && lod != 6 && lod != 7) || (depth_compare && mask != 1) || mask == 0) return false; const u32 coordinates = coordinate_count[dimension]; const bool has_lod = lod == 2 || lod == 3 || lod == 6 || lod == 7; - const u32 count = coordinates + (has_lod ? 1 : 0) + (offsets ? 1 : 0) + (depth_compare ? 1 : 0); - const u32 in_a = count <= 4 ? (count + 1) / 2 : 4; + const u32 count = coordinates + (register_handle ? 1 : 0) + (has_lod ? 1 : 0) + + (offsets ? 1 : 0) + (depth_compare ? 1 : 0); + // Unpacked 0x361 (bit 60 clear): Ra holds the coordinates, Rb the handle and the rest. + const u32 in_a = register_handle && !insn.Bits(60, 1) ? coordinates + : count <= 4 ? (count + 1) / 2 : 4; const auto value = [&](u32 i) -> u32 { if (i < in_a) return source_a == 255 ? 255U : source_a + i; return source_b == 255 ? 255U : source_b + (i - in_a); @@ -1525,25 +1733,34 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { if ((source_a != 255 && source_a + in_a > 246) || (count > in_a && source_b != 255 && source_b + (count - in_a) > 246)) return false; - const u32 binding = insn.Bits(54, 5); - if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) - return false; - result.required_texture_buffer = binding; + // Handle forms and a second descriptor buffer are read bindless: TEX.B takes the + // handle (the register's, or loaded from that buffer) as its first meta register. + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const bool bindless = register_handle || uniform_handle || !bind_texture_buffer(binding); const auto copy = [&](u32 target, u32 source) { emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20)); }; - // Coordinates to R250.., then LOD/bias, offsets and reference to R246... + // Coordinates to R250.., then the handle, LOD/bias, offsets and reference to R246... for (u32 i = 0; i < coordinates; ++i) copy(250 + i, value(i)); u32 meta = 246; u32 next = coordinates; + // The handle is copied unpredicated, so it is the same value on every path. + if (register_handle || uniform_handle) { + emit(0x5c98000000070000ULL | meta++ | (u64{15} << 39) | + (u64{register_handle ? value(next++) : handle_register} << 20)); + } else if (bindless) { + emit(0x4c98000000070000ULL | meta++ | (u64{15} << 39) | (u64{offset} << 20) | + (u64{binding} << 34)); + } if (has_lod) copy(meta++, value(next++)); if (offsets) copy(meta++, value(next++)); if (depth_compare) copy(meta++, value(next++)); - emit(0xc000000000000000ULL | predicate | 250 | (u64{250} << 8) | + emit((bindless ? 0xde80000000000000ULL : 0xc000000000000000ULL) | predicate | 250 | (u64{250} << 8) | (u64{meta == 246 ? 255U : 246U} << 20) | (u64{maxwell_type[dimension]} << 28) | - (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | - (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) | (u64{7} << 51) | - (u64{offsets} << 54) | (u64{lod} << 55)); + (u64{mask} << 31) | (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) | + (u64{7} << 51) | + (bindless ? (u64{offsets} << 36) | (u64{lod} << 37) + : (u64{offset} << 36) | (u64{offsets} << 54) | (u64{lod} << 55))); const u32 components = static_cast<u32>(std::popcount(mask)); if (insn.Bits(79, 1)) { // FP16 results: (c0, c1) packed into the first destination, (c2, c3) into @@ -1587,11 +1804,11 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) || (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) || (depth_compare && (dimension != 1 || split || mask != 1 || (source_a & 1) || source_a >= 249)) || - (dimension == 5 && ((source_a & 1) || source_a >= 249))) { + (dimension == 5 && ((source_a & 1) || source_a >= 249)) || register_handle || uniform_handle || + !bind_texture_buffer(2)) { if (!general_tex()) return fail(index, "native texture mode or descriptor binding"); break; } - result.required_texture_buffer = 2; emit((0xd800000000000000ULL & ~(u64{insn.Bits(79, 1)} << 59)) | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(64, 8)} << 28) | (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | @@ -1739,12 +1956,22 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { break; } case 0x209: case 0x809: case 0xa09: case 0xc09: { // FMNMX, including a uniform source - if (insn.Bits(74, 6) || insn.Bits(81, 6) || (opcode != 0xc09 && insn.Bits(91, 1))) + if (insn.Bits(74, 6) || insn.Bits(82, 5) || (opcode != 0xc09 && insn.Bits(91, 1))) return fail(index, "native floating min/max mode"); const u32 a = operand(0), b = operand(1); + // .NAN (bit 81): an unordered pair gives the canonical NaN. R252 = ~0 when both are numbers. + const bool propagate_nan = insn.Bits(81, 1); + if (propagate_nan) + emit(0x5800000000000000ULL | predicate | 252 | (u64{a} << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{7} << 48)); emit(base(0x5c60000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | (u64{insn.Bits(80, 1)} << 44)); + if (propagate_nan) { + emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{252} << 20) | (u64{7} << 48)); + emit(0x0400000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53)); + emit(0x0400000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{0x7fffffffU} << 20)); + emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{252} << 20) | (u64{1} << 41) | (u64{7} << 48)); + } break; } case 0x308: case 0xb08: { // MUFU @@ -1862,13 +2089,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } case 0x207: case 0x807: case 0xa07: case 0x208: case 0x808: case 0xa08: { // Bitwise SEL / FSEL - const u32 a = operand(0, false), b = operand(1, false); + // FSEL applies the floating operand modifiers (negate/absolute) before selecting. + const bool floating = (opcode & 0x1ff) == 0x008; + const u32 a = operand(0, floating), b = operand(1, floating); emit(base(0x5ca0000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); break; } - case 0x20a: case 0x80a: case 0xa0a: { // FSET.BF, combined with Pc (bits 87-90) by bits 74-75 - if (insn.Bits(64, 8) || insn.Bits(74, 2) == 3 || insn.Bits(81, 6) || insn.Bits(91, 1)) + case 0x20a: case 0x80a: case 0xa0a: case 0xc0a: { // FSET.BF, combined with Pc (bits 87-90) by bits 74-75 (0xc0a: uniform b) + if (insn.Bits(64, 8) || insn.Bits(74, 2) == 3 || insn.Bits(81, 6) || (opcode != 0xc0a && insn.Bits(91, 1))) return fail(index, "native floating comparison output mode"); const u32 a = operand(0), b = operand(1); emit(base(0x5800000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | @@ -1890,7 +2119,45 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { case 0x20b: case 0x80b: case 0xa0b: // FSETP case 0x20c: case 0x80c: case 0xa0c: case 0xc0c: { // ISETP const bool floating = (opcode & 0x1ff) == 0xb; - if (!floating && insn.Bits(72, 1)) return fail(index, "native extended integer comparison"); + if (!floating && insn.Bits(72, 1)) { + // ISETP.EX: the high words, chained to the low words' predicate in bits 68-71. + const u32 comparison = insn.Bits(76, 3), low_comparison = insn.Bits(68, 4); + if (insn.Bits(74, 2) || insn.Bits(84, 3) != 7 || insn.Bits(87, 4) != 7) + return fail(index, "native extended integer comparison"); + if (comparison == 1 || comparison == 3 || comparison == 4 || comparison == 6) { + // Unsigned LT/LE/GT/GE: (a < b, or > for GT/GE) || (a == b && low predicate), + // built from 0/~0 ISET masks, then one ISETP.NE into the output predicate. + const u32 a = insn.Bits(24, 8), b = insn.Bits(32, 8); + if (opcode != 0x20c || insn.Bits(73, 1) || insn.Bits(64, 4) || insn.Bits(80, 1) || + (insn.words[1] & ~0xffU)) + return fail(index, "native extended ordered comparison mode"); + const auto set_mask = [&](u32 target, u32 operation, u32 lhs, u32 rhs) { + emit(0x5b50000000000000ULL | predicate | target | (u64{lhs} << 8) | + (u64{rhs} << 20) | (u64{7} << 39) | (u64{operation} << 49)); + }; + if ((low_comparison & 7) == 7) { + mov_imm(252, (low_comparison & 8) ? 0U : ~0U); + } else { + emit(0x38e8000000000000ULL | predicate | 252 | (u64{255} << 8) | + (u64{1} << (20 + (low_comparison & 7)))); + set_mask(252, (low_comparison & 8) ? 2U : 5U, 252, 255); + } + set_mask(250, comparison == 1 || comparison == 3 ? 1U : 4U, a, b); + set_mask(251, 2, a, b); + emit(0x5c40000000000000ULL | predicate | 251 | (u64{251} << 8) | (u64{252} << 20) | + (u64{7} << 48)); + emit(0x5c40000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{251} << 20) | + (u64{1} << 41) | (u64{7} << 48)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) | + (u64{250} << 8) | (u64{255} << 20) | (u64{7} << 39) | (u64{5} << 49)); + break; + } + // EQ/NE: the high comparison ANDed (EQ) or ORed (NE) with that predicate. + if (comparison != 2 && comparison != 5) + return fail(index, "native extended integer comparison"); + insn.words[2] = (insn.words[2] & ~(0x100U | 0xc00U | 0x07800000U)) | + (insn.Bits(68, 4) << 23) | (comparison == 5 ? 0x400U : 0U); + } const u32 a = operand(0, floating), b = operand(1, floating); const u32 combine = insn.Bits(74, 2); if (combine > 2) return fail(index, "native predicate combination"); @@ -1938,15 +2205,19 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } case 0x217: case 0x817: case 0xa17: case 0xc17: { // IMNMX register, immediate, cbuf or uniform operand. if ((insn.words[2] & ~(0x04000200U | (opcode == 0xc17 ? 0x08000000U : 0U))) != 0x03800000 || - (opcode != 0xa17 && insn.words[1] > (opcode == 0x817 ? 0x7ffffU : 255U)) || + (opcode == 0x817 && insn.words[1] > 0x7ffffU && insn.words[1] < 0xfff80000U) || + (opcode != 0xa17 && opcode != 0x817 && insn.words[1] > 255U) || (opcode == 0xa17 && (!valid_cbuf() || insn.Bits(62, 2)))) return fail(index, "native integer min/max mode"); + // An immediate is a signed 20-bit value: 19 bits at 20, the sign in bit 56. const u32 source = opcode == 0xc17 ? uniform_register(insn.words[1]) : opcode == 0xa17 ? operand(1, false) + : opcode == 0x817 ? insn.words[1] & 0x7ffffU : insn.words[1]; if (source == ~0U) return fail(index, "native uniform min/max operand"); emit(base(opcode == 0x817 ? 0x3820000000000000ULL : 0x5c20000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{source} << 20) | + (u64{opcode == 0x817 ? insn.words[1] >> 31 : 0U} << 56) | (u64{insn.Bits(87, 4)} << 39) | (u64{insn.Bits(73, 1)} << 48)); break; } @@ -1976,6 +2247,20 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { break; } case 0xa11: { // LEA with constant base and carry predicate. + if ((insn.words[2] & ~(0x07800000U | 0x0000f800U)) == 0x000f06ffU && !insn.Bits(62, 2) && + insn.Bits(75, 5) != 0) { + // LEA.HI.X.SX32, no high source: (sign-extended Ra << shift) >> 32, plus the + // constant high word and the carry predicate (bits 87-90). + const u32 b = operand(1, false); + emit(0x3828000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | + (u64{32 - insn.Bits(75, 5)} << 20) | (u64{1} << 48)); + mov_imm(252, 1); + emit(0x5ca0000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{b} << 20)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20)); + break; + } if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || insn.Bits(81, 3) == 7 || insn.Bits(62, 2)) return fail(index, "native constant LEA carry mode"); @@ -1986,35 +2271,65 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); break; } - case 0x211: case 0x811: { // LEA low word, no carry result + case 0x211: case 0x811: case 0xc11: { // LEA low word (0xc11: uniform Rb) if (insn.Bits(80, 1)) { - // LEA.HI Rd, Ra, Rb (or an immediate, 0x811), RZ, s: Rb + high word of (Ra << s); - // bit 73 sign-extends Ra. - if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(81, 3) != 7 || - insn.Bits(64, 8) != 255 || (opcode == 0x211 && insn.Bits(62, 2))) + // LEA.HI Rd, Ra, Rb (or an immediate, 0x811), Rc, s: Rb + high word of (Rc:Ra << s); + // bit 73 sign-extends Ra (Rc is RZ) and .X (bit 74) adds the carry predicate in bits 87-90. + const bool extended = insn.Bits(74, 1) != 0; + const u32 high = insn.Bits(64, 8); + if (insn.Bits(72, 1) || insn.Bits(81, 3) != 7 || + (high != 255 && insn.Bits(73, 1)) || (opcode == 0x211 && insn.Bits(62, 2)) || + (opcode == 0xc11 && (insn.words[1] & ~0x3fU))) return fail(index, "native high effective address mode"); - u32 b = insn.Bits(32, 8); + u32 b = opcode == 0xc11 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + if (b == ~0U) return fail(index, "native uniform LEA register"); if (opcode == 0x811) { b = insn.words[1] ? 251U : 255U; if (b == 251) mov_imm(251, insn.words[1]); } const u64 shift = (u64{32 - insn.Bits(75, 5)} << 20) | (u64{insn.Bits(73, 1)} << 48); - if (b == 255) { - emit(base(0x3828000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | shift); + // The high word is a right shift of Ra, or with Rc a left funnel shift (SHF.L). + const auto high_word = [&](u64 target) { + if (high == 255) emit(target | 0x3828000000000000ULL | (u64{insn.Bits(24, 8)} << 8) | shift); + else emit(target | 0x36f8000000000000ULL | (u64{insn.Bits(24, 8)} << 8) | (u64{high} << 39) | + (u64{insn.Bits(75, 5)} << 20)); + }; + if (b == 255 && !extended) { + high_word(base(0)); } else { - emit(0x3828000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | shift); + high_word(predicate | 250); + if (extended) { + mov_imm(252, 1); + emit(0x5ca0000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{252} << 20)); + } emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{b} << 20)); } break; } - if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || insn.Bits(81, 3) != 7 || - (opcode == 0x211 && insn.Bits(62, 1))) return fail(index, "native LEA carry/high mode"); - // 0x211 bit 63 negates Rb (LEA Rd, Ra, -Rb, s), as Maxwell ISCADD's bit 48. - u32 b = insn.Bits(32, 8); + const u32 carry_output = insn.Bits(81, 3); + if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || + (carry_output != 7 && (rewritten_uniform || (opcode != 0x811 && insn.Bits(63, 1)))) || + (opcode == 0x211 && insn.Bits(62, 1)) || (opcode == 0xc11 && (insn.words[1] & ~0x8000003fU))) + return fail(index, "native LEA carry/high mode"); + // 0x211/0xc11 bit 63 negates Rb (LEA Rd, Ra, -Rb, s), as Maxwell ISCADD's bit 48. + u32 b = opcode == 0xc11 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + if (b == ~0U) return fail(index, "native uniform LEA register"); if (opcode == 0x811) { mov_imm(251, insn.words[1]); b = 251; } + if (carry_output != 7) { + // LEA Rd, Pu, Ra, Rb, s: Pu is the carry of (Ra << s) + Rb, set when the sum wraps + // below Ra << s. The guard may be Pu, so Rd is written first. + emit(0x3848000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(75, 5)} << 20)); + emit(0x5c10000000000000ULL | predicate | 252 | (u64{250} << 8) | (u64{b} << 20)); + emit(base(0x5c98000000000000ULL) | (u64{252} << 20) | (u64{15} << 39)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{carry_output} << 3) | (u64{252} << 8) | + (u64{250} << 20) | (u64{7} << 39) | (u64{1} << 49)); + break; + } emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39) | - (u64{opcode == 0x211 ? insn.Bits(63, 1) : 0U} << 48)); + (u64{opcode != 0x811 ? insn.Bits(63, 1) : 0U} << 48)); break; } case 0x210: case 0x810: case 0xa10: case 0xc10: { // IADD3 with zero third source and no carry outputs @@ -2022,7 +2337,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { // a carry-in predicate (bits 87-89, negate 90). Uniform forms keep UPn in R239 bit 6-n. if ((opcode == 0x210 || opcode == 0x810) && insn.Bits(64, 8) == 255 && !insn.Bits(73, 1) && !insn.Bits(75, 2) && insn.Bits(84, 3) == 7 && insn.Bits(77, 4) == 15 && !insn.Bits(72, 1) && - (insn.Bits(81, 3) != 7 || insn.Bits(74, 1))) { + (insn.Bits(81, 3) != 7 || insn.Bits(74, 1)) && (opcode == 0x810 || !insn.Bits(63, 1))) { const bool carry_in = insn.Bits(74, 1) != 0; const u32 carry_out = insn.Bits(81, 3); const u32 a = insn.Bits(24, 8); @@ -2065,9 +2380,65 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{251} << 20)); break; } - if (opcode == 0xa10 && insn.Bits(81, 3) != 7) { + // Three sources, negations or two carries: the sum builds up in R253 (.CC) with the carry + // count in R254 (.X). Pu/Pv (bits 81/84) are set when the count reaches 1/2, and IADD3.X + // adds both carry-ins (bits 87, 77). A negated source is ~x, plus 1 unless extended. + const bool extended = insn.Bits(74, 1) != 0; + const bool negate_a = insn.Bits(72, 1), negate_b = opcode != 0x810 && insn.Bits(63, 1), negate_c = insn.Bits(75, 1); + if ((extended || insn.Bits(81, 6) != 0x3f) && + (insn.Bits(64, 8) != 255 || insn.Bits(84, 3) != 7 || negate_a || negate_b || negate_c || + (extended && insn.Bits(77, 4) != 15))) { + const u32 first_output = insn.Bits(81, 3), second_output = insn.Bits(84, 3); + if (rewritten_uniform || guarded_uniform || insn.Bits(73, 1) || insn.Bits(76, 1) || + (first_output != 7 && first_output == second_output) || + (!extended && (insn.Bits(77, 4) != 15 || insn.Bits(87, 4) != 15)) || + (opcode == 0x210 && (insn.words[1] & ~0x800000ffU)) || (opcode == 0xa10 && insn.Bits(62, 1)) || + (opcode == 0xc10 && (insn.words[1] & ~0x8000003fU))) + return fail(index, "native three-input carry mode"); + u32 a = insn.Bits(24, 8), b = operand(1, false), c = insn.Bits(64, 8); + const auto invert = [&](u32& source, u32 target) { + emit(0x0400000000000000ULL | predicate | target | (u64{source} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53)); + source = target; + }; + if (negate_a) invert(a, 250); + if (negate_b) invert(b, 251); + if (negate_c) invert(c, 252); + const auto accumulate = [&](u32 source) { + emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{source} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 254 | (u64{254} << 8) | (u64{255} << 20) | (u64{1} << 43)); + }; + emit(0x5c10000000000000ULL | predicate | 253 | (u64{a} << 8) | (u64{b} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 254 | (u64{255} << 8) | (u64{255} << 20) | (u64{1} << 43)); + accumulate(c); + if (extended) { + for (const u32 carry : {insn.Bits(87, 4), insn.Bits(77, 4)}) { + if (carry == 15) continue; + mov_imm(250, 1); + if (carry != 7) + emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{carry & 7} << 39) | (u64{carry >> 3} << 42)); + accumulate(250); + } + } else if (const u32 ones = u32{negate_a} + u32{negate_b} + u32{negate_c}) { + mov_imm(250, ones); + accumulate(250); + } + if (dest != 255) emit(base(0x5c98000000000000ULL) | (u64{253} << 20) | (u64{15} << 39)); + // An output that is also the guard predicate is written last. + const bool guard_first = (native_predicate & 7) != 7 && first_output == (native_predicate & 7); + for (const u32 output : {guard_first ? 1U : 0U, guard_first ? 0U : 1U}) { + const u32 target = output ? second_output : first_output; + if (target == 7) continue; + mov_imm(250, output + 1); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{target} << 3) | (u64{254} << 8) | + (u64{250} << 20) | (u64{7} << 39) | (u64{6} << 49)); + } + break; + } + if ((opcode == 0xa10 || opcode == 0xc10) && insn.Bits(81, 3) != 7) { // cbuf or uniform b if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(84, 7) != 0x7f || - insn.Bits(77, 4) != 15 || insn.Bits(63, 1) || insn.Bits(72, 1)) + insn.Bits(77, 4) != 15 || insn.Bits(63, 1) || insn.Bits(72, 1) || + (opcode == 0xc10 && (insn.words[1] & ~0x3fU))) return fail(index, "native constant add carry mode"); const u32 b = operand(1, false); emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20)); @@ -2075,13 +2446,16 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); break; } - if (opcode == 0xa10 && insn.Bits(74, 1)) { - if (insn.Bits(24, 8) != 255 || (insn.words[2] & ~0x07800000U) != 0x007fe4ffU) + if ((opcode == 0xa10 || opcode == 0xc10) && insn.Bits(74, 1)) { // IADD3.X Ra + cbuf or UR + carry predicate + if ((insn.words[2] & ~(0x07800000U | (opcode == 0xc10 ? 0x08000000U : 0U))) != 0x007fe4ffU || + (opcode == 0xc10 && (insn.words[1] & ~0x3fU))) return fail(index, "native extended constant add mode"); const u32 b = operand(1, false); mov_imm(250, 1); emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + if (insn.Bits(24, 8) != 255) + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{insn.Bits(24, 8)} << 20)); emit(base(0x5c10000000000000ULL) | (u64{b} << 8) | (u64{250} << 20)); break; } diff --git a/src/shader_recompiler/ir_opt/position_pass.cpp b/src/shader_recompiler/ir_opt/position_pass.cpp index 4890346944..600a596bf3 100644 --- a/src/shader_recompiler/ir_opt/position_pass.cpp +++ b/src/shader_recompiler/ir_opt/position_pass.cpp @@ -83,6 +83,37 @@ void BlitConstantsPass(Environment& env, IR::Program& program) { if (env.ShaderStage() != Stage::VertexB && env.ShaderStage() != Stage::Fragment) { return; } + // Bank 24 is root table 0, which games may load themselves (3D methods 0x504/0x508; the buffer + // cache binds it as bank 17): Fire Emblem: Fortune's Weave keeps its material push constants + // there. A shader reading bank 17 outside the blit block reads such a table and keeps its reads, + // as does one reading only words the blit block leaves zero (an unbound bank reads zero anyway; + // the game's enemy health gauges read only 0x40 of their table). + const auto in_blit_block = [](u32 offset) { + return offset < 0x20 || (offset >= 0x30 && offset < 0x4c) || (offset >= 0x50 && offset < 0x60); + }; + const auto is_nonzero_word = [](u32 offset) { + return offset < 0x20 || offset == 0x38 || offset == 0x3c || offset == 0x58 || offset == 0x5c; + }; + bool reads_nonzero_word{}; + for (IR::Block* const block : program.post_order_blocks) { + for (IR::Inst& inst : block->Instructions()) { + const IR::Opcode opcode{inst.GetOpcode()}; + if ((opcode != IR::Opcode::GetCbufU32 && opcode != IR::Opcode::GetCbufF32 && + opcode != IR::Opcode::GetCbufU32x2) || + !inst.Arg(0).IsImmediate() || inst.Arg(0).U32() != 17 || !inst.Arg(1).IsImmediate()) { + continue; + } + const u32 offset{inst.Arg(1).U32()}; + const bool pair{opcode == IR::Opcode::GetCbufU32x2}; + if (!in_blit_block(offset) || (pair && !in_blit_block(offset + 4))) { + return; + } + reads_nonzero_word |= is_nonzero_word(offset) || (pair && is_nonzero_word(offset + 4)); + } + } + if (!reads_nonzero_word) { + return; + } bool used{}; for (IR::Block* const block : program.post_order_blocks) { for (IR::Inst& inst : block->Instructions()) { diff --git a/src/tests/video_core/ampere_returns.cpp b/src/tests/video_core/ampere_returns.cpp index 56b8b374da..163758e07e 100644 --- a/src/tests/video_core/ampere_returns.cpp +++ b/src/tests/video_core/ampere_returns.cpp @@ -48,4 +48,31 @@ TEST_CASE("Ampere callees still reject writes to the link", "[video_core][ampere REQUIRE(lowered.error == "native callee overwrites link register"); REQUIRE(lowered.error_offset == 0x30); } + +TEST_CASE("Ampere callees can compare into predicates with an R0 link", "[video_core][ampere]") { + // ISETP's zero destination field is not a write to R0. + const std::array<Instruction, 5> code{{ + {{0x00007802, 0x20, 0x00000f00, 0}}, + {{0x00007944, 0x10, 0x03c00000, 0}}, + {{0x0000794d, 0, 0x03800000, 0}}, + {{0x0100720c, 0x00000002, 0x03f06070, 0}}, + {{0x00007950, 0xffffffb0, 0x03c3ffff, 0}}, + }}; + REQUIRE(static_cast<bool>(LowerToMaxwell(code))); +} + +TEST_CASE("Ampere callees reject a wide multiply over the link", "[video_core][ampere]") { + // IMAD.WIDE.U32 R54:R55 writes the R55 link. + const std::array<Instruction, 5> code{{ + {{0x00377802, 0x20, 0x00000f00, 0}}, + {{0x00007944, 0x10, 0x03c00000, 0}}, + {{0x0000794d, 0, 0x03800000, 0}}, + {{0x01367225, 0x00000002, 0x078e00ff, 0}}, + {{0x37007950, 0xffffffb0, 0x03c3ffff, 0}}, + }}; + const auto lowered = LowerToMaxwell(code); + REQUIRE(!static_cast<bool>(lowered)); + REQUIRE(lowered.error == "native callee overwrites link register"); + REQUIRE(lowered.error_offset == 0x30); +} } // namespace diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e90af3214d..2fc74cb6c8 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -994,6 +994,26 @@ void BufferCache<P>::BindHostGraphicsUniformBuffers(size_t stage) { template <class P> void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind) { ++channel_state->uniform_cache_shots[0]; + // Bank 24 (root table 0, NVN2's push constants) is read from bank 17, which such games do not + // bind: stream the table's current words as that buffer. + if (index == 17 && maxwell3d->root_tables.loaded && channel_state->uniform_buffers[stage][index].size == 0) { + const auto& words = maxwell3d->root_tables.tables[0]; + const u32 size = (*channel_state->uniform_buffer_sizes)[stage][index]; + const u32 copied = (std::min)(size, static_cast<u32>(sizeof(words))); + if constexpr (IS_OPENGL) { + std::array<u8, sizeof(words)> data{}; + std::memcpy(data.data(), words.data(), copied); + runtime.BindFastUniformBuffer(stage, binding_index, copied); + runtime.PushFastUniformBuffer(stage, binding_index, std::span<const u8>(data.data(), copied)); + } else { + const std::span<u8> span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + std::memset(span.data(), 0, span.size()); + std::memcpy(span.data(), words.data(), copied); + } + channel_state->fast_bound_uniform_buffers[stage] |= 1u << binding_index; + channel_state->uniform_buffer_binding_sizes[stage][binding_index] = size; + return; + } const Binding& binding = channel_state->uniform_buffers[stage][index]; const DAddr device_addr = binding.device_addr; const u32 size = (std::min)(binding.size, (*channel_state->uniform_buffer_sizes)[stage][index]); diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index ab2ba10162..b8a6567907 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -6,6 +6,7 @@ #include <algorithm> #include <atomic> +#include <bit> #include <bitset> #include <cstdlib> #include <fstream> @@ -557,6 +558,7 @@ bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& // The console runs this on tensor cores; stand in with a plain upscale of its input. return UpscaleInsteadOfNativeProgram(); } + RememberDlssMotionVectors(); return false; } return true; @@ -588,9 +590,35 @@ void KeplerCompute::DescribeNativeResources(const NativeQmdScheduler::Descriptor std::atomic<u32> g_nx2_trace_draws_after_upscale{}; +// Fire Emblem: Fortune's Weave's first DLSS pass (rejected, before the network) has the input colour +// at cbuf0+0x278, its motion vectors (R16G16 float, the input's size) at +0x290 and this frame's +// camera jitter (x, y; input pixels, Halton bases 2 and 3 less 0.5) at +0x1b0. +void KeplerCompute::RememberDlssMotionVectors() { + const auto& cbuf = launch_description.const_buffer_config[0]; + if (cbuf.size.Value() < 0x294) return; + const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low; + const auto image = [&](u32 offset) -> std::optional<Texture::TICEntry> { + const u32 tic_index = memory_manager.Read<u32>(cbuf_address + offset) & 0xfffff; + if (tic_index > regs.tic.limit) return std::nullopt; + return GetTICEntry(tic_index); + }; + const auto color = image(0x278), motion = image(0x290); + if (!color || !motion || color->format != Texture::TextureFormat::B10G11R11 || + motion->format != Texture::TextureFormat::R16G16 || motion->Address() == 0 || + motion->Width() != color->Width() || motion->Height() != color->Height() || + !motion->IsBlockLinear()) + return; + dlss_motion_vectors = *motion; + dlss_motion_color = color->Address(); + dlss_jitter = {std::bit_cast<f32>(memory_manager.Read<u32>(cbuf_address + 0x1b0)), + std::bit_cast<f32>(memory_manager.Read<u32>(cbuf_address + 0x1b4))}; +} + // Title 4's upscaler reads the tonemapped frame through the texture handle at cbuf0+0x90 and // writes the output resolution image through the surface handle at cbuf0+0xa0 (from its -// TLD/SUST operands). Blit one to the other with bilinear filtering when both are plain 2D images +// TLD/SUST operands). Fire Emblem: Fortune's Weave's DLSS network ends with a pass that reads the +// input colour through cbuf0+0x328 and writes the DLSS output through cbuf0+0x348 (B10G11R11; 720p +// to 1440p docked). Blit one to the other with bilinear filtering when both are plain 2D images // of the same format and the output is larger. bool KeplerCompute::UpscaleInsteadOfNativeProgram() { const auto& cbuf = launch_description.const_buffer_config[0]; @@ -600,11 +628,20 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() { if (tic_index > regs.tic.limit) return std::nullopt; return GetTICEntry(tic_index); }; - const auto input = image(0x90), output = image(0xa0); - if (!input || !output || input->format != output->format || input->Address() == 0 || - output->Address() == 0 || output->Width() <= input->Width() || !input->IsBlockLinear() || - !output->IsBlockLinear()) - return false; + const auto usable = [](const std::optional<Texture::TICEntry>& input, + const std::optional<Texture::TICEntry>& output) { + return input && output && input->format == output->format && input->Address() != 0 && + output->Address() != 0 && output->Width() > input->Width() && input->IsBlockLinear() && + output->IsBlockLinear(); + }; + auto input = image(0x90), output = image(0xa0); + bool fortunes_weave = false; + if (!usable(input, output)) { + input = image(0x328); + output = image(0x348); + if (!usable(input, output)) return false; + fortunes_weave = true; + } using RTFormat = Tegra::RenderTargetFormat; RTFormat format{}; switch (input->format) { @@ -700,7 +737,24 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() { source.format = static_cast<RTFormat>(std::strtoul(value, nullptr, 16)); } } - const bool done = rasterizer->AccelerateSurfaceCopy(source, surface(*output), config); + // Fortune's Weave: a temporal upscale along its motion vectors that undoes this frame's camera + // jitter (both from the frame's first DLSS pass) and accumulates the frames as DLSS would. + // EDEN_NX2_TEMPORAL_UPSCALE=0 blits instead. + const auto upscale_temporally = [&] { + static const bool enabled = [] { + const char* value = std::getenv("EDEN_NX2_TEMPORAL_UPSCALE"); + return value == nullptr || std::string_view{value} != "0"; + }(); + if (!fortunes_weave || !enabled || !dlss_motion_vectors || + dlss_motion_color != input->Address()) + return false; + Fermi2D::Surface motion = surface(*dlss_motion_vectors); + motion.format = RTFormat::R16G16_FLOAT; + return rasterizer->AccelerateTemporalUpscale(source, surface(*output), motion, dlss_jitter, + {1.0f, 1.0f}); + }; + const bool done = upscale_temporally() || + rasterizer->AccelerateSurfaceCopy(source, surface(*output), config); if (static u32 checked{}; checked++ == sample_at) { const u64 size = u64{output->Width()} * output->Height() * 4; if (const auto device = memory_manager.GpuToCpuAddress(output->Address())) { diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index b525636dde..b5aee3706a 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -266,7 +266,15 @@ private: bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor); void ProcessNativeReportSemaphore(u32 argument); void DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd); + void RememberDlssMotionVectors(); bool UpscaleInsteadOfNativeProgram(); + + /// Motion vectors of Fire Emblem: Fortune's Weave's DLSS evaluation, from its first pass (run + /// earlier in the frame than the network pass the stand-in replaces), with the input colour + /// image and camera jitter they go with. + std::optional<Texture::TICEntry> dlss_motion_vectors; + GPUVAddr dlss_motion_color{}; + std::array<f32, 2> dlss_jitter{}; bool ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call); NativeQmdScheduler native_qmd; std::array<u32, 0xc00>* native_data_ram{}; diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 6adfcf89b8..4001284127 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -164,6 +164,8 @@ bool Maxwell3D::IsMethodExecutable(u32 method) { case MAXWELL3D_REG_INDEX(vertex_array_instance_first): case MAXWELL3D_REG_INDEX(vertex_array_instance_subsequent): case MAXWELL3D_REG_INDEX(draw_texture.src_y0): + case RootTables::SelectMethod: + case RootTables::LoadMethod: case MAXWELL3D_REG_INDEX(wait_for_idle): case MAXWELL3D_REG_INDEX(shadow_ram_control): case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): @@ -346,6 +348,17 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume LOG_DEBUG(HW_GPU, "Native GPU surface clear: value={:#x}", argument); } switch (method) { + case RootTables::SelectMethod: + root_tables.selected = argument & 7; + root_tables.offset = (argument >> 8) & 0xff; + return; + case RootTables::LoadMethod: + if (root_tables.offset < 256 && root_tables.offset % 4 == 0) { + root_tables.tables[root_tables.selected][root_tables.offset / 4] = argument; + root_tables.loaded = true; + } + root_tables.offset += 4; + return; case MAXWELL3D_REG_INDEX(wait_for_idle): return rasterizer->WaitForIdle(); case MAXWELL3D_REG_INDEX(shadow_ram_control): @@ -819,6 +832,8 @@ bool FrontHandledOutsideDrawManager(u32 method) { return true; } switch (method) { + case Maxwell3D::RootTables::SelectMethod: + case Maxwell3D::RootTables::LoadMethod: case MAXWELL3D_REG_INDEX(wait_for_idle): case MAXWELL3D_REG_INDEX(shadow_ram_control): case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 0121f1a01c..a32b31fd1e 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3160,6 +3160,18 @@ public: std::array<u32, 0xc00>& GetNativeDataRam() { return native_data_ram; } std::span<u32> GetNativeMacroPositions() { return native_macro_context ? std::span<u32>{native_macro_context->positions} : std::span<u32>{macro_positions}; } + /// Root tables: eight 256-byte tables written inline (method 0x504 selects a table, bits 0-2, + /// and a byte offset, bits 8-15; each 0x508 writes the next word) that shaders read as constant + /// banks 24-31. NVN2 keeps push constants in bank 24, which the Ampere lowering reads as bank 17. + struct RootTables { + static constexpr u32 SelectMethod = 0x504 / 4; + static constexpr u32 LoadMethod = 0x508 / 4; + std::array<std::array<u32, 64>, 8> tables{}; + u32 selected{}; + u32 offset{}; + bool loaded{}; + } root_tables; + /// Write the value to the register identified by method. void CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) override; diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index 35b350c2cd..ada5c779dc 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -71,6 +71,7 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/smaa_blending_weight_calculation.frag ${CMAKE_CURRENT_SOURCE_DIR}/smaa_neighborhood_blending.vert ${CMAKE_CURRENT_SOURCE_DIR}/smaa_neighborhood_blending.frag + ${CMAKE_CURRENT_SOURCE_DIR}/temporal_upscale.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_blit_depth_stencil.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_color_clear.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_color_clear.vert diff --git a/src/video_core/host_shaders/temporal_upscale.frag b/src/video_core/host_shaders/temporal_upscale.frag new file mode 100644 index 0000000000..0e76df2b43 --- /dev/null +++ b/src/video_core/host_shaders/temporal_upscale.frag @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// Stands in for a game's DLSS evaluation (the tensor-core network cannot run here): a temporal +// upscale. Each frame's input samples are placed where the camera jitter put them and weighted by +// their distance to the output pixel; they accumulate with the previous output, reprojected along +// the game's motion vectors (bicubic) and clipped to the current 3x3 neighbourhood. + +#version 450 + +layout(binding = 0) uniform sampler2D input_color; +layout(binding = 1) uniform sampler2D motion_vectors; +layout(binding = 2) uniform sampler2D history; + +layout(push_constant) uniform PushConstants { + // The vertex stage's texture scale and offset come first. + layout(offset = 16) vec2 jitter; // this frame's sample offset, in input pixels + vec2 motion_scale; // motion vector -> UV offset to the previous frame + float current_weight; // largest weight of this frame; 1 ignores the history +}; + +layout(location = 0) in vec2 texcoord; // 0 to 1 across the output +layout(location = 0) out vec4 color; + +// The blit helper's samplers take unnormalized (texel) coordinates. + +// Blending in a range-compressed space keeps bright pixels from dominating the average. +vec3 Compress(vec3 c) { + return c / (1.0 + max(max(c.r, c.g), c.b)); +} + +vec3 Expand(vec3 c) { + c = clamp(c, vec3(0.0), vec3(1.0 - 1.0 / 65504.0)); + return c / (1.0 - max(max(c.r, c.g), c.b)); +} + +vec3 RgbToYCoCg(vec3 c) { + return vec3(dot(c, vec3(0.25, 0.5, 0.25)), dot(c, vec3(0.5, 0.0, -0.5)), + dot(c, vec3(-0.25, 0.5, -0.25))); +} + +vec3 YCoCgToRgb(vec3 c) { + return vec3(c.x + c.y - c.z, c.x + c.z, c.x - c.y - c.z); +} + +// Moves value towards the box centre until it lies inside the box. +vec3 ClipToBox(vec3 value, vec3 low, vec3 high) { + const vec3 center = 0.5 * (low + high); + const vec3 extent = 0.5 * (high - low) + 1.0 / 65504.0; + const vec3 offset = value - center; + const vec3 units = abs(offset / extent); + const float largest = max(units.x, max(units.y, units.z)); + return largest > 1.0 ? center + offset / largest : value; +} + +// Catmull-Rom filtered history from five bilinear fetches: repeated bilinear reprojection would +// blur the accumulated image a little more every frame. +vec3 SampleHistory(vec2 position) { + const vec2 texel1 = floor(position - 0.5) + 0.5; + const vec2 f = position - texel1; + const vec2 w0 = f * (-0.5 + f * (1.0 - 0.5 * f)); + const vec2 w1 = 1.0 + f * f * (-2.5 + 1.5 * f); + const vec2 w2 = f * (0.5 + f * (2.0 - 1.5 * f)); + const vec2 w3 = f * f * (-0.5 + 0.5 * f); + const vec2 w12 = w1 + w2; + const vec2 texel0 = texel1 - 1.0; + const vec2 texel3 = texel1 + 2.0; + const vec2 texel12 = texel1 + w2 / w12; + vec3 sum = texture(history, vec2(texel12.x, texel0.y)).rgb * (w12.x * w0.y); + sum += texture(history, vec2(texel0.x, texel12.y)).rgb * (w0.x * w12.y); + sum += texture(history, texel12).rgb * (w12.x * w12.y); + sum += texture(history, vec2(texel3.x, texel12.y)).rgb * (w3.x * w12.y); + sum += texture(history, vec2(texel12.x, texel3.y)).rgb * (w12.x * w3.y); + const float weight = w12.x * w0.y + w0.x * w12.y + w12.x * w12.y + w3.x * w12.y + w12.x * w3.y; + return max(sum / weight, vec3(0.0)); +} + +void main() { + const ivec2 input_size = textureSize(input_color, 0); + const ivec2 motion_size = textureSize(motion_vectors, 0); + // The output pixel's centre in input pixels. Input texel k sampled the scene at k + 0.5 - jitter + // (the sign measured against consecutive frames). + const vec2 target = texcoord * vec2(input_size); + const ivec2 nearest = ivec2(floor(target + jitter)); + + vec3 filtered = vec3(0.0); + float total_weight = 0.0; + float nearest_weight = 0.0; + vec3 sum = vec3(0.0); + vec3 squares = vec3(0.0); + vec2 motion = vec2(0.0); + float longest = -1.0; + for (int y = -1; y <= 1; ++y) { + for (int x = -1; x <= 1; ++x) { + const ivec2 texel = clamp(nearest + ivec2(x, y), ivec2(0), input_size - 1); + const vec3 sample_color = RgbToYCoCg(Compress(texelFetch(input_color, texel, 0).rgb)); + // A Blackman-Harris-like window over the sample's distance to the output pixel. + const vec2 distance = vec2(texel) + 0.5 - jitter - target; + const float weight = exp(-2.29 * dot(distance, distance)); + filtered += sample_color * weight; + total_weight += weight; + nearest_weight = max(nearest_weight, weight); + sum += sample_color; + squares += sample_color * sample_color; + // The longest motion around the pixel, so that an edge follows the object in front. + const vec2 candidate = + texelFetch(motion_vectors, clamp(texel, ivec2(0), motion_size - 1), 0).rg; + const float length_squared = dot(candidate, candidate); + if (length_squared > longest) { + longest = length_squared; + motion = candidate; + } + } + } + const vec3 current = filtered / total_weight; + const vec3 mean = sum / 9.0; + const vec3 deviation = sqrt(max(squares / 9.0 - mean * mean, vec3(0.0))); + const vec3 low = mean - deviation; + const vec3 high = mean + deviation; + + const vec2 previous = texcoord + motion * motion_scale; + // A frame whose samples land close to this pixel counts more; one whose samples land between + // pixels mostly keeps the history (from other jitter positions) there. + float weight = current_weight * nearest_weight; + if (current_weight >= 1.0 || any(lessThan(previous, vec2(0.0))) || + any(greaterThan(previous, vec2(1.0)))) { + weight = 1.0; + } + const vec2 history_position = previous * vec2(textureSize(history, 0)); + const vec3 past = ClipToBox(RgbToYCoCg(Compress(SampleHistory(history_position))), low, high); + color = vec4(Expand(YCoCgToRgb(mix(past, current, weight))), 1.0); +} diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index b527c0e2a9..8486d44ac9 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -6,6 +6,7 @@ #pragma once +#include <array> #include <functional> #include <optional> #include <span> @@ -162,6 +163,16 @@ public: return false; } + /// Stand-in for a DLSS evaluation: temporal upscale of src into dst along motion_vectors + /// (jitter in src pixels; motion_scale turns a motion vector into a UV offset to the previous + /// frame). False when unsupported; the caller then falls back to AccelerateSurfaceCopy. + [[nodiscard]] virtual bool AccelerateTemporalUpscale( + const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale) { + return false; + } + [[nodiscard]] virtual Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() = 0; virtual void AccelerateInlineToMemory(GPUVAddr address, size_t copy_size, diff --git a/src/video_core/renderer_opengl/present/layer.cpp b/src/video_core/renderer_opengl/present/layer.cpp index 9e4a0f854a..2656892f4a 100644 --- a/src/video_core/renderer_opengl/present/layer.cpp +++ b/src/video_core/renderer_opengl/present/layer.cpp @@ -183,6 +183,11 @@ void Layer::ConfigureFramebufferTexture(const Tegra::FramebufferConfig& framebuf framebuffer_texture.gl_format = GL_RGBA; framebuffer_texture.gl_type = GL_UNSIGNED_INT_8_8_8_8_REV; break; + case Service::android::PixelFormat::Rgba1010102: + internal_format = GL_RGB10_A2; + framebuffer_texture.gl_format = GL_RGBA; + framebuffer_texture.gl_type = GL_UNSIGNED_INT_2_10_10_10_REV; + break; case Service::android::PixelFormat::Rgb565: internal_format = GL_RGB565; framebuffer_texture.gl_format = GL_RGB; diff --git a/src/video_core/renderer_vulkan/blit_image.cpp b/src/video_core/renderer_vulkan/blit_image.cpp index ddd52e9426..edd151b563 100644 --- a/src/video_core/renderer_vulkan/blit_image.cpp +++ b/src/video_core/renderer_vulkan/blit_image.cpp @@ -33,6 +33,7 @@ #include "video_core/host_shaders/convert_non_msaa_to_msaa_depth_stencil_frag_spv.h" #include "video_core/host_shaders/convert_s8d24_to_abgr8_frag_spv.h" #include "video_core/host_shaders/full_screen_triangle_vert_spv.h" +#include "video_core/host_shaders/temporal_upscale_frag_spv.h" #include "video_core/host_shaders/vulkan_blit_depth_stencil_frag_spv.h" #include "video_core/host_shaders/vulkan_color_clear_frag_spv.h" #include "video_core/host_shaders/vulkan_color_clear_vert_spv.h" @@ -91,6 +92,15 @@ struct PushConstants { std::array<float, 2> tex_offset; }; +// The vertex stage's PushConstants first, then those of temporal_upscale.frag. +struct TemporalUpscalePushConstants { + PushConstants vertex; + std::array<float, 2> jitter; + std::array<float, 2> motion_scale; + float current_weight; + float padding; +}; + struct MSAACopyPushConstants { std::array<s32, 2> dst_offset; std::array<s32, 2> src_offset; @@ -109,6 +119,11 @@ constexpr std::array TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{ TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>, TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>, }; +constexpr std::array THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{ + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>, + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>, + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<2>, +}; constexpr VkDescriptorSetLayoutCreateInfo ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{ .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .pNext = nullptr, @@ -133,6 +148,13 @@ constexpr VkDescriptorSetLayoutCreateInfo TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CRE .bindingCount = static_cast<u32>(TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()), .pBindings = TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(), }; +constexpr VkDescriptorSetLayoutCreateInfo THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{ + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .bindingCount = static_cast<u32>(THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()), + .pBindings = THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(), +}; template <VkShaderStageFlags stageFlags, size_t size> inline constexpr VkPushConstantRange PUSH_CONSTANT_RANGE{ .stageFlags = stageFlags, @@ -357,6 +379,32 @@ void UpdateOneTextureDescriptorSet(const Device& device, VkDescriptorSet descrip device.GetLogical().UpdateDescriptorSets(write_descriptor_set, nullptr); } +void UpdateThreeTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set, + VkSampler sampler, std::array<VkImageView, 3> image_views) { + std::array<VkDescriptorImageInfo, 3> image_infos{}; + std::array<VkWriteDescriptorSet, 3> write_descriptor_sets{}; + for (u32 binding = 0; binding < 3; ++binding) { + image_infos[binding] = VkDescriptorImageInfo{ + .sampler = sampler, + .imageView = image_views[binding], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + }; + write_descriptor_sets[binding] = VkWriteDescriptorSet{ + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = nullptr, + .dstSet = descriptor_set, + .dstBinding = binding, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImageInfo = &image_infos[binding], + .pBufferInfo = nullptr, + .pTexelBufferView = nullptr, + }; + } + device.GetLogical().UpdateDescriptorSets(write_descriptor_sets, nullptr); +} + void UpdateTwoTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set, VkSampler sampler, VkImageView image_view_0, VkImageView image_view_1) { @@ -590,10 +638,14 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), two_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout( TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), + three_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout( + THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), one_texture_descriptor_allocator{ descriptor_pool.Allocator(device_, scheduler_, *one_texture_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<1>)}, two_textures_descriptor_allocator{ descriptor_pool.Allocator(device_, scheduler_, *two_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<2>)}, + three_textures_descriptor_allocator{ + descriptor_pool.Allocator(device_, scheduler_, *three_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<3>)}, one_texture_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( one_texture_set_layout.address(), PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT, sizeof(PushConstants)>))), @@ -601,6 +653,11 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( two_textures_set_layout.address(), PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT, sizeof(PushConstants)>))), + temporal_upscale_pipeline_layout( + device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( + three_textures_set_layout.address(), + PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + sizeof(TemporalUpscalePushConstants)>))), clear_color_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( nullptr, PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(float) * 4>))), msaa_copy_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( @@ -612,6 +669,7 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(MSAACopyPushConstants)>))), full_screen_vert(BuildShader(device, FULL_SCREEN_TRIANGLE_VERT_SPV)), blit_color_to_color_frag(BuildShader(device, BLIT_COLOR_FLOAT_FRAG_SPV)), + temporal_upscale_frag(BuildShader(device, TEMPORAL_UPSCALE_FRAG_SPV)), blit_color_msaa_frag(BuildShader(device, BLIT_COLOR_MSAA_FRAG_SPV)), blit_depth_stencil_frag(device.IsExtShaderStencilExportSupported() ? BuildShader(device, VULKAN_BLIT_DEPTH_STENCIL_FRAG_SPV) @@ -702,6 +760,42 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView }); } +void BlitImageHelper::TemporalUpscale(const Framebuffer* dst_framebuffer, + const ImageView& input_color, const ImageView& motion_vectors, + VkImageView history, const Region2D& dst_region, + std::array<f32, 2> jitter, std::array<f32, 2> motion_scale, + f32 current_weight) { + const VkPipelineLayout layout = *temporal_upscale_pipeline_layout; + const VkPipeline pipeline = FindOrEmplaceTemporalUpscalePipeline(dst_framebuffer->RenderPass()); + const VkSampler sampler = *linear_sampler; + const std::array<VkImageView, 3> views{input_color.Handle(Shader::TextureType::Color2D), + motion_vectors.Handle(Shader::TextureType::Color2D), + history}; + const TemporalUpscalePushConstants push_constants{ + .vertex = {.tex_scale = {1.0f, 1.0f}, .tex_offset = {0.0f, 0.0f}}, + .jitter = jitter, + .motion_scale = motion_scale, + .current_weight = current_weight, + .padding = 0.0f, + }; + RecordShaderReadBarrier(scheduler, input_color); + RecordShaderReadBarrier(scheduler, motion_vectors); + scheduler.RequestRenderpass(dst_framebuffer); + scheduler.Record([this, dst_region, pipeline, layout, sampler, views, + push_constants](vk::CommandBuffer cmdbuf) { + const VkDescriptorSet descriptor_set = three_textures_descriptor_allocator.Commit(); + UpdateThreeTexturesDescriptorSet(device, descriptor_set, sampler, views); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, descriptor_set, + nullptr); + BindBlitState(cmdbuf, dst_region); + cmdbuf.PushConstants(layout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + push_constants); + cmdbuf.Draw(3, 1, 0, 0); + }); + scheduler.InvalidateState(); +} + void BlitImageHelper::BlitImpl(const Framebuffer* dst_framebuffer, const ImageView& src_image_view, const Region2D& dst_region, const Region2D& src_region, VkPipeline pipeline, VkSampler sampler, @@ -1285,6 +1379,59 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe return *blit_color_pipelines.back(); } +VkPipeline BlitImageHelper::FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass) { + const auto it = std::ranges::find(temporal_upscale_keys, renderpass); + if (it != temporal_upscale_keys.end()) { + return *temporal_upscale_pipelines[std::distance(temporal_upscale_keys.begin(), it)]; + } + temporal_upscale_keys.push_back(renderpass); + const std::array stages = MakeStages(*full_screen_vert, *temporal_upscale_frag); + const VkPipelineColorBlendAttachmentState blend_attachment{ + .blendEnable = VK_FALSE, + .srcColorBlendFactor = VK_BLEND_FACTOR_ZERO, + .dstColorBlendFactor = VK_BLEND_FACTOR_ZERO, + .colorBlendOp = VK_BLEND_OP_ADD, + .srcAlphaBlendFactor = VK_BLEND_FACTOR_ZERO, + .dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO, + .alphaBlendOp = VK_BLEND_OP_ADD, + .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | + VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, + }; + const VkPipelineColorBlendStateCreateInfo color_blend_create_info{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .logicOpEnable = VK_FALSE, + .logicOp = VK_LOGIC_OP_CLEAR, + .attachmentCount = 1, + .pAttachments = &blend_attachment, + .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, + }; + const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + temporal_upscale_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ + .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .stageCount = static_cast<u32>(stages.size()), + .pStages = stages.data(), + .pVertexInputState = &PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, + .pInputAssemblyState = &input_assembly_ci, + .pTessellationState = nullptr, + .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, + .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, + .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, + .pDepthStencilState = nullptr, + .pColorBlendState = &color_blend_create_info, + .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, + .layout = *temporal_upscale_pipeline_layout, + .renderPass = renderpass, + .subpass = 0, + .basePipelineHandle = VK_NULL_HANDLE, + .basePipelineIndex = 0, + }, device.StaticPipelineCache())); + return *temporal_upscale_pipelines.back(); +} + VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key) { const auto it = std::ranges::find(blit_depth_stencil_keys, key); if (it != blit_depth_stencil_keys.end()) { diff --git a/src/video_core/renderer_vulkan/blit_image.h b/src/video_core/renderer_vulkan/blit_image.h index 62e6743a23..303beeea5c 100644 --- a/src/video_core/renderer_vulkan/blit_image.h +++ b/src/video_core/renderer_vulkan/blit_image.h @@ -82,6 +82,13 @@ public: VkImage src_image, VkSampler src_sampler, const Region2D& dst_region, const Region2D& src_region, const Extent3D& src_size); + /// Temporal upscale (temporal_upscale.frag) into dst_framebuffer: the jittered input colour, + /// accumulated with the history image reprojected along the motion vectors. + void TemporalUpscale(const Framebuffer* dst_framebuffer, const ImageView& input_color, + const ImageView& motion_vectors, VkImageView history, + const Region2D& dst_region, std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale, f32 current_weight); + void BlitColorMSAA(const Framebuffer* dst_framebuffer, const ImageView& src_image_view, const Region2D& dst_region, const Region2D& src_region); @@ -170,6 +177,8 @@ private: [[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key); + [[nodiscard]] VkPipeline FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass); + [[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key); [[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key); @@ -207,15 +216,19 @@ private: vk::DescriptorSetLayout one_texture_set_layout; vk::DescriptorSetLayout two_textures_set_layout; + vk::DescriptorSetLayout three_textures_set_layout; DescriptorAllocator one_texture_descriptor_allocator; DescriptorAllocator two_textures_descriptor_allocator; + DescriptorAllocator three_textures_descriptor_allocator; vk::PipelineLayout one_texture_pipeline_layout; vk::PipelineLayout two_textures_pipeline_layout; + vk::PipelineLayout temporal_upscale_pipeline_layout; vk::PipelineLayout clear_color_pipeline_layout; vk::PipelineLayout msaa_copy_pipeline_layout; vk::PipelineLayout msaa_copy_depth_stencil_pipeline_layout; vk::ShaderModule full_screen_vert; vk::ShaderModule blit_color_to_color_frag; + vk::ShaderModule temporal_upscale_frag; vk::ShaderModule blit_color_msaa_frag; vk::ShaderModule blit_depth_stencil_frag; vk::ShaderModule blit_depth_frag; @@ -246,6 +259,8 @@ private: std::vector<BlitImagePipelineKey> blit_color_keys; std::vector<vk::Pipeline> blit_color_pipelines; + std::vector<VkRenderPass> temporal_upscale_keys; + std::vector<vk::Pipeline> temporal_upscale_pipelines; std::vector<BlitImagePipelineKey> blit_depth_stencil_keys; std::vector<vk::Pipeline> blit_depth_stencil_pipelines; std::vector<BlitImagePipelineKey> clear_color_keys; diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp index fa24c02f6d..5652090bd7 100644 --- a/src/video_core/renderer_vulkan/present/layer.cpp +++ b/src/video_core/renderer_vulkan/present/layer.cpp @@ -52,6 +52,8 @@ VkFormat GetFormat(const Tegra::FramebufferConfig& framebuffer) { return VK_FORMAT_R5G6B5_UNORM_PACK16; case Service::android::PixelFormat::Bgra8888: return VK_FORMAT_B8G8R8A8_UNORM; + case Service::android::PixelFormat::Rgba1010102: + return VK_FORMAT_A2B10G10R10_UNORM_PACK32; default: UNIMPLEMENTED_MSG("Unknown framebuffer pixel format: {}", static_cast<u32>(framebuffer.pixel_format)); diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index cb31a0e7e4..c0e8825bac 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1231,6 +1231,14 @@ bool RasterizerVulkan::AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surf return texture_cache.BlitImage(dst, src, copy_config); } +bool RasterizerVulkan::AccelerateTemporalUpscale( + const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale) { + std::scoped_lock lock{texture_cache.mutex}; + return texture_cache.TemporalUpscale(dst, src, motion_vectors, jitter, motion_scale); +} + Tegra::Engines::AccelerateDMAInterface& RasterizerVulkan::AccessAccelerateDMA() { return accelerate_dma; } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index ae9929ca0b..7aca376614 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -128,6 +128,11 @@ public: bool AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, const Tegra::Engines::Fermi2D::Config& copy_config) override; + bool AccelerateTemporalUpscale(const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale) override; Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() override; void AccelerateInlineToMemory(GPUVAddr address, size_t copy_size, std::span<const u8> memory) override; diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index ddd55de690..4e428229f6 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -1491,6 +1491,154 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src, }); } +bool TextureCacheRuntime::TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst, + ImageView& input_color, ImageView& motion_vectors, + std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale) { + if (dst.Samples() != VK_SAMPLE_COUNT_1_BIT || ImageAspectMask(dst.format) != VK_IMAGE_ASPECT_COLOR_BIT) { + return false; + } + const VkFormat format = + MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, false, dst.format).format; + const VkExtent2D extent{dst.size.width, dst.size.height}; + const VkImage output = dst.ImageHandle(); + bool reset = false; + if (!temporal_history.image || temporal_history.format != format || + temporal_history.extent.width != extent.width || + temporal_history.extent.height != extent.height) { + if (temporal_history.image) { + scheduler.Finish(); // the old history may still be sampled + } + temporal_history.image = memory_allocator.CreateImage(VkImageCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .imageType = VK_IMAGE_TYPE_2D, + .format = format, + .extent = {extent.width, extent.height, 1}, + .mipLevels = 1, + .arrayLayers = 1, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + }); + temporal_history.view = device.GetLogical().CreateImageView(VkImageViewCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .image = *temporal_history.image, + .viewType = VK_IMAGE_VIEW_TYPE_2D, + .format = format, + .components{}, + .subresourceRange{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + }, + }); + temporal_history.format = format; + temporal_history.extent = extent; + reset = true; + LOG_INFO(Render_Vulkan, "Temporal upscale into {}x{} (format {})", extent.width, extent.height, + static_cast<u32>(format)); + } + + // The history holds this pass's previous result, copied right after drawing it: the game may + // draw into its DLSS output afterwards (post-processing), which must not feed back. The history + // stays in the GENERAL layout, which the blit helper's descriptors use. + const VkImage history = *temporal_history.image; + const VkImageSubresourceLayers output_layers{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = static_cast<u32>(dst.range.base.level), + .baseArrayLayer = static_cast<u32>(dst.range.base.layer), + .layerCount = 1, + }; + static constexpr VkImageSubresourceLayers history_layers{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = 0, + .baseArrayLayer = 0, + .layerCount = 1, + }; + const auto barrier = [](VkImage image, VkAccessFlags src_access, VkAccessFlags dst_access, + VkImageLayout old_layout, const VkImageSubresourceLayers& layers) { + return VkImageMemoryBarrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = src_access, + .dstAccessMask = dst_access, + .oldLayout = old_layout, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = image, + .subresourceRange{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = layers.mipLevel, + .levelCount = 1, + .baseArrayLayer = layers.baseArrayLayer, + .layerCount = 1, + }, + }; + }; + if (reset) { + // A new history image has no contents yet; the first frame does not read it. + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([history, barrier](vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + 0, barrier(history, 0, VK_ACCESS_SHADER_READ_BIT, + VK_IMAGE_LAYOUT_UNDEFINED, history_layers)); + }); + } + const Region2D dst_region{ + .start = {0, 0}, + .end = {static_cast<s32>(extent.width), static_cast<s32>(extent.height)}, + }; + // Each frame weighs at most 0.1 against the history (scaled by how close its samples land to + // the pixel; the neighbourhood clip limits what older samples contribute). + constexpr f32 current_weight = 0.1f; + blit_image_helper.TemporalUpscale(dst_framebuffer, input_color, motion_vectors, + *temporal_history.view, dst_region, jitter, motion_scale, + reset ? 1.0f : current_weight); + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([output, history, extent, output_layers, barrier](vk::CommandBuffer cmdbuf) { + constexpr VkAccessFlags written = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + const std::array before{ + barrier(output, written, VK_ACCESS_TRANSFER_READ_BIT, VK_IMAGE_LAYOUT_GENERAL, + output_layers), + barrier(history, VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, + VK_IMAGE_LAYOUT_GENERAL, history_layers), + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, {}, before); + const VkImageCopy copy{ + .srcSubresource = output_layers, + .srcOffset = {0, 0, 0}, + .dstSubresource = history_layers, + .dstOffset = {0, 0, 0}, + .extent = {extent.width, extent.height, 1}, + }; + cmdbuf.CopyImage(output, VK_IMAGE_LAYOUT_GENERAL, history, VK_IMAGE_LAYOUT_GENERAL, copy); + const std::array after{ + barrier(output, VK_ACCESS_TRANSFER_READ_BIT, + VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_TRANSFER_READ_BIT, + VK_IMAGE_LAYOUT_GENERAL, output_layers), + barrier(history, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT, + VK_IMAGE_LAYOUT_GENERAL, history_layers), + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, + {}, {}, after); + }); + return true; +} + void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& src, const Region2D& dst_region, const Region2D& src_region, Tegra::Engines::Fermi2D::Filter filter, diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 43218d6ad3..25a6ce7c1e 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -87,6 +87,12 @@ public: Tegra::Engines::Fermi2D::Filter filter, Tegra::Engines::Fermi2D::Operation operation); + /// Stands in for a DLSS evaluation: temporal upscale of input_color into dst, accumulated with + /// a copy of dst's previous contents (the last output). False when dst cannot take it. + bool TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& input_color, + ImageView& motion_vectors, std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale); + void CopyImage(Image& dst, Image& src, std::span<const VideoCommon::ImageCopy> copies); void CopyImageMSAA(Image& dst, Image& src, std::span<const VideoCommon::ImageCopy> copies); @@ -169,6 +175,15 @@ public: StagingBufferPool& staging_buffer_pool; BlitImageHelper& blit_image_helper; RenderPassCache& render_pass_cache; + + /// TemporalUpscale's history: its previous result, copied right after drawing it. + struct TemporalHistory { + vk::Image image; + vk::ImageView view; + VkFormat format{}; + VkExtent2D extent{}; + } temporal_history; + std::optional<ASTCDecoderPass> astc_decoder_pass; std::optional<BcEncoderPass> bc_encoder_pass; diff --git a/src/video_core/surface.cpp b/src/video_core/surface.cpp index 66b9577cb5..ef4859ea3e 100644 --- a/src/video_core/surface.cpp +++ b/src/video_core/surface.cpp @@ -214,6 +214,8 @@ PixelFormat PixelFormatFromGPUPixelFormat(Service::android::PixelFormat format) return PixelFormat::R5G6B5_UNORM; case Service::android::PixelFormat::Bgra8888: return PixelFormat::B8G8R8A8_UNORM; + case Service::android::PixelFormat::Rgba1010102: + return PixelFormat::A2B10G10R10_UNORM; default: UNIMPLEMENTED_MSG("Unimplemented format={}", format); return PixelFormat::A8B8G8R8_UNORM; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 33ca051482..32775b8719 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -957,6 +957,73 @@ bool TextureCache<P>::BlitImage(const Tegra::Engines::Fermi2D::Surface& dst, } template <class P> +bool TextureCache<P>::TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array<f32, 2> jitter, std::array<f32, 2> motion_scale) { + if constexpr (FRAMEBUFFER_BLITS) { + return false; + } else { + const Tegra::Engines::Fermi2D::Config copy{ + .operation = Tegra::Engines::Fermi2D::Operation::SrcCopy, + .filter = Tegra::Engines::Fermi2D::Filter::Bilinear, + .must_accelerate = true, + .dst_x0 = 0, + .dst_y0 = 0, + .dst_x1 = static_cast<s32>(dst.width), + .dst_y1 = static_cast<s32>(dst.height), + .src_x0 = 0, + .src_y0 = 0, + .src_x1 = static_cast<s32>(src.width), + .src_y1 = static_cast<s32>(src.height), + }; + const auto result = GetBlitImages(dst, src, copy); + if (!result) { + return false; + } + const ImageInfo motion_info(motion_vectors); + has_deleted_images = false; + const ImageId motion_id = + FindOrInsertImage(motion_info, motion_vectors.Address(), RelaxedOptions{}); + if (has_deleted_images || !motion_id) { + return false; // the colour and output ids may be stale; the caller blits instead + } + const ImageId dst_id = result->dst_id; + const ImageId src_id = result->src_id; + PrepareImage(src_id, false, false); + PrepareImage(motion_id, false, false); + PrepareImage(dst_id, true, false); + for (const ImageId id : {src_id, motion_id, dst_id}) { + if (True(slot_images[id].flags & ImageFlagBits::Rescaled) || + slot_images[id].info.num_samples != 1) { + return false; + } + } + const auto view = [&](ImageId id, const Tegra::Engines::Fermi2D::Surface& surface, + PixelFormat format) -> std::optional<std::pair<FramebufferId, ImageViewId>> { + const std::optional base = slot_images[id].TryFindBase(surface.Address()); + if (!base) { + return std::nullopt; + } + const SubresourceRange range{.base = *base, .extent = {1, 1}}; + return RenderTargetFromImage(id, ImageViewInfo(ImageViewType::e2D, format, range)); + }; + const auto src_view = view(src_id, src, result->src_format); + const auto motion_view = view(motion_id, motion_vectors, motion_info.format); + const auto dst_view = view(dst_id, dst, result->dst_format); + if (!src_view || !motion_view || !dst_view) { + return false; + } + // After the views: creating them may reallocate the framebuffer slots. + Framebuffer* const dst_framebuffer = &slot_framebuffers[dst_view->first]; + return runtime.TemporalUpscale(dst_framebuffer, slot_image_views[dst_view->second], + slot_image_views[src_view->second], + slot_image_views[motion_view->second], jitter, + motion_scale); + } +} + +template <class P> std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImageView( const Tegra::FramebufferConfig& config, DAddr cpu_addr) { // TODO: Properly implement this diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 141e5b76d5..d4b0f597ae 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -276,6 +276,13 @@ public: const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Config& copy); + /// Stand-in for a DLSS evaluation: temporal upscale of src into dst (see the runtime's + /// TemporalUpscale). False when the images cannot take it; the caller then blits. + bool TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array<f32, 2> jitter, std::array<f32, 2> motion_scale); + /// Try to find a cached image view in the given CPU address [[nodiscard]] std::pair<ImageView*, bool> TryFindFramebufferImageView( const Tegra::FramebufferConfig& config, DAddr cpu_addr); |
