aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authornowhere02 <140357165+nowhere02@users.noreply.github.com>2026-11-02 00:17:14 -0300
committerGitHub <noreply@github.com>2026-11-02 00:17:14 -0300
commit9c55e70d790a4cc9507d80dbb9a9d6224175574e (patch)
tree4c53ccbf16ab56bb00e512cc7926b1d0d438792d
parentb8d2c08d6efd312620b066cdab2f0238ebc94a26 (diff)
parent06e0541e18e5bb8ab8d3d03660d3d82cb112f62b (diff)
Merge pull request #270 from Duck912/fe-render
Fe render
-rw-r--r--src/audio_core/common/feature_support.h28
-rw-r--r--src/audio_core/renderer/behavior/behavior_info.cpp8
-rw-r--r--src/audio_core/renderer/behavior/behavior_info.h1
-rw-r--r--src/audio_core/renderer/behavior/info_updater.cpp9
-rw-r--r--src/core/hle/service/am/frontend/applet_software_keyboard.cpp8
-rw-r--r--src/core/hle/service/am/frontend/applet_software_keyboard_types.h60
-rw-r--r--src/core/hle/service/nvdrv/core/container.cpp2
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp46
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h2
-rw-r--r--src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp25
-rw-r--r--src/core/hle/service/nvdrv/devices/nx2_stub_device.h51
-rw-r--r--src/core/hle/service/nvnflinger/pixel_format.h1
-rw-r--r--src/core/hle/service/nx2_gsv.h294
-rw-r--r--src/shader_recompiler/frontend/ampere/lower_to_maxwell.h614
-rw-r--r--src/shader_recompiler/ir_opt/position_pass.cpp31
-rw-r--r--src/tests/video_core/ampere_returns.cpp27
-rw-r--r--src/video_core/buffer_cache/buffer_cache.h20
-rw-r--r--src/video_core/engines/kepler_compute.cpp68
-rw-r--r--src/video_core/engines/kepler_compute.h8
-rw-r--r--src/video_core/engines/maxwell_3d.cpp15
-rw-r--r--src/video_core/engines/maxwell_3d.h12
-rw-r--r--src/video_core/host_shaders/CMakeLists.txt1
-rw-r--r--src/video_core/host_shaders/temporal_upscale.frag132
-rw-r--r--src/video_core/rasterizer_interface.h11
-rw-r--r--src/video_core/renderer_opengl/present/layer.cpp5
-rw-r--r--src/video_core/renderer_vulkan/blit_image.cpp147
-rw-r--r--src/video_core/renderer_vulkan/blit_image.h15
-rw-r--r--src/video_core/renderer_vulkan/present/layer.cpp2
-rw-r--r--src/video_core/renderer_vulkan/vk_rasterizer.cpp8
-rw-r--r--src/video_core/renderer_vulkan/vk_rasterizer.h5
-rw-r--r--src/video_core/renderer_vulkan/vk_texture_cache.cpp148
-rw-r--r--src/video_core/renderer_vulkan/vk_texture_cache.h15
-rw-r--r--src/video_core/surface.cpp2
-rw-r--r--src/video_core/texture_cache/texture_cache.h67
-rw-r--r--src/video_core/texture_cache/texture_cache_base.h7
35 files changed, 1724 insertions, 171 deletions
diff --git a/src/audio_core/common/feature_support.h b/src/audio_core/common/feature_support.h
index 4585be45ef..d6571c0661 100644
--- a/src/audio_core/common/feature_support.h
+++ b/src/audio_core/common/feature_support.h
@@ -17,6 +17,16 @@
namespace AudioCore {
constexpr u32 CurrentRevision = 15;
+constexpr u32 NativeCurrentRevision = 16;
+
+constexpr bool IsNativeRevision(u32 revision) {
+ return (revision & 0xffffff) == Common::MakeMagic('R', 'E', 'v', '\0');
+}
+
+constexpr u32 EncodeRevision(u32 revision, bool native) {
+ return Common::MakeMagic('R', 'E', native ? 'v' : 'V',
+ static_cast<char>('0' + revision));
+}
enum class SupportTags {
CommandProcessingTimeEstimatorVersion4,
@@ -57,12 +67,15 @@ enum class SupportTags {
};
constexpr u32 GetRevisionNum(u32 user_revision) {
- if (user_revision >= 0x100) {
- user_revision -= Common::MakeMagic('R', 'E', 'V', '0');
- user_revision >>= 24;
+ if (user_revision < 0x100) {
+ return user_revision;
}
-
- return user_revision;
+ if (!IsNativeRevision(user_revision) &&
+ (user_revision & 0xffffff) != Common::MakeMagic('R', 'E', 'V', '\0')) {
+ return ~u32{0};
+ }
+ const u32 encoded = user_revision >> 24;
+ return encoded >= '0' ? encoded - '0' : ~u32{0};
};
constexpr bool CheckFeatureSupported(SupportTags tag, u32 user_revision) {
@@ -108,11 +121,12 @@ constexpr bool CheckFeatureSupported(SupportTags tag, u32 user_revision) {
return false;
}
user_revision = GetRevisionNum(user_revision);
- return (*feature).second <= user_revision;
+ return user_revision != ~u32{0} && (*feature).second <= user_revision;
}
constexpr bool CheckValidRevision(u32 user_revision) {
- return GetRevisionNum(user_revision) <= CurrentRevision;
+ return GetRevisionNum(user_revision) <=
+ (IsNativeRevision(user_revision) ? NativeCurrentRevision : CurrentRevision);
};
} // namespace AudioCore
diff --git a/src/audio_core/renderer/behavior/behavior_info.cpp b/src/audio_core/renderer/behavior/behavior_info.cpp
index 63b80c503a..983cfd5b85 100644
--- a/src/audio_core/renderer/behavior/behavior_info.cpp
+++ b/src/audio_core/renderer/behavior/behavior_info.cpp
@@ -16,8 +16,7 @@ u32 BehaviorInfo::GetProcessRevisionNum() const {
}
u32 BehaviorInfo::GetProcessRevision() const {
- return Common::MakeMagic('R', 'E', 'V',
- static_cast<char>(static_cast<u8>('0') + process_revision));
+ return EncodeRevision(process_revision, native_platform);
}
u32 BehaviorInfo::GetUserRevisionNum() const {
@@ -25,11 +24,12 @@ u32 BehaviorInfo::GetUserRevisionNum() const {
}
u32 BehaviorInfo::GetUserRevision() const {
- return Common::MakeMagic('R', 'E', 'V',
- static_cast<char>(static_cast<u8>('0') + user_revision));
+ return EncodeRevision(user_revision, native_platform);
}
void BehaviorInfo::SetUserLibRevision(const u32 user_revision_) {
+ native_platform = IsNativeRevision(user_revision_);
+ process_revision = native_platform ? NativeCurrentRevision : CurrentRevision;
user_revision = GetRevisionNum(user_revision_);
}
diff --git a/src/audio_core/renderer/behavior/behavior_info.h b/src/audio_core/renderer/behavior/behavior_info.h
index 757f163c5a..318d4e1ff4 100644
--- a/src/audio_core/renderer/behavior/behavior_info.h
+++ b/src/audio_core/renderer/behavior/behavior_info.h
@@ -400,6 +400,7 @@ public:
u32 process_revision;
/// User version
u32 user_revision{};
+ bool native_platform{};
/// Behaviour flags
Flags flags{};
/// Errors generated and reported during Update
diff --git a/src/audio_core/renderer/behavior/info_updater.cpp b/src/audio_core/renderer/behavior/info_updater.cpp
index ca3eaece94..a8fbcf9730 100644
--- a/src/audio_core/renderer/behavior/info_updater.cpp
+++ b/src/audio_core/renderer/behavior/info_updater.cpp
@@ -122,12 +122,21 @@ Result InfoUpdater::UpdateVoices(VoiceContext& voice_context,
u8 flush_wave_buffer_count;
u16 reserved2;
VoiceInfo::Flags flags;
+ u8 reserved_after_flags;
SrcQuality src_quality;
+ u8 crossfade_curve;
u32 external_ctx;
u32 external_ctx_size;
u32 reserved3[2];
};
+ static_assert(sizeof(VoiceInParameterV2) == 0x188);
+ static_assert(offsetof(VoiceInParameterV2, crossfade_curve) == 0x177);
const auto* vin = reinterpret_cast<const VoiceInParameterV2*>(input + i * in_stride);
+ if (behaviour.GetUserRevisionNum() >= 16 && vin->in_use && vin->crossfade_curve != 0) {
+ LOG_ERROR(Service_Audio, "Unsupported voice biquad crossfade curve {}",
+ vin->crossfade_curve);
+ return Service::Audio::ResultInvalidUpdateInfo;
+ }
local_in.id = vin->id;
local_in.node_id = vin->node_id;
local_in.is_new = vin->is_new;
diff --git a/src/core/hle/service/am/frontend/applet_software_keyboard.cpp b/src/core/hle/service/am/frontend/applet_software_keyboard.cpp
index db42781e18..061394bff8 100644
--- a/src/core/hle/service/am/frontend/applet_software_keyboard.cpp
+++ b/src/core/hle/service/am/frontend/applet_software_keyboard.cpp
@@ -222,6 +222,14 @@ void SoftwareKeyboard::InitializeForeground() {
sizeof(SwkbdConfigNew));
swkbd_config_common.use_utf8 = true;
break;
+ case SwkbdAppletVersion::Version1441810: {
+ // Fire Emblem: Fortune's Weave asks for UTF-8 and keeps the reply up to the first NUL, so the
+ // Switch 1 layout's use_utf8 (a byte of the native initial_string_offset) left one letter.
+ const bool decoded =
+ DecodeNativeSwkbdConfig(swkbd_config_data, swkbd_config_common, swkbd_config_new);
+ ASSERT(decoded);
+ break;
+ }
default:
UNIMPLEMENTED_MSG("Unknown SwkbdConfig revision={} with size={}", swkbd_applet_version,
swkbd_config_data.size());
diff --git a/src/core/hle/service/am/frontend/applet_software_keyboard_types.h b/src/core/hle/service/am/frontend/applet_software_keyboard_types.h
index f263a704cd..8ff8391aa1 100644
--- a/src/core/hle/service/am/frontend/applet_software_keyboard_types.h
+++ b/src/core/hle/service/am/frontend/applet_software_keyboard_types.h
@@ -7,6 +7,9 @@
#pragma once
#include <array>
+#include <cstddef>
+#include <cstring>
+#include <span>
#include "common/bit_field.h"
#include "common/common_funcs.h"
@@ -31,6 +34,7 @@ enum class SwkbdAppletVersion : u32_le {
Version393227 = 0x6000B, // 6.0.0 - 7.0.1
Version524301 = 0x8000D, // 8.0.0+
Version1310736 = 0x140010, // Switch 2 (Pokemon Pokopia): 0x530-byte config, UTF-8 text
+ Version1441810 = 0x160012, // Switch 2 (Fire Emblem: Fortune's Weave): native 0x530-byte config
};
enum class SwkbdType : u32 {
@@ -184,8 +188,64 @@ struct SwkbdConfigNew {
};
static_assert(sizeof(SwkbdConfigNew) == 0x4C8 - sizeof(SwkbdConfigCommon),
"SwkbdConfigNew has incorrect size.");
+
+// SwkbdAppletVersion 0x160012: the text prefix of the common part keeps its layout, the flags,
+// lengths, string offsets and dictionary fields move (checked against
+// nn::swkbd::keyboardConfig2kbdConfig in Fire Emblem: Fortune's Weave's sdk).
+struct SwkbdConfigNative {
+ std::array<u8, 0x3AA> common_prefix{};
+ bool use_utf8{};
+ bool use_blur_background{};
+ u32 max_text_length{};
+ u32 min_text_length{};
+ SwkbdPasswordMode password_mode{};
+ SwkbdTextDrawType text_draw_type{};
+ u32 initial_string_offset{};
+ u32 initial_string_length{};
+ u32 user_dictionary_offset{};
+ u32 user_dictionary_entries{};
+ std::array<u64, 24> customized_dictionary_set_entries{};
+ u8 total_customized_dictionary_set_entries{};
+ bool use_text_check{};
+ bool disable_cancel_button{};
+ u8 trigger{};
+ std::array<u32, 8> text_grouping{};
+ INSERT_PADDING_BYTES(0x80);
+};
+static_assert(sizeof(SwkbdConfigNative) == 0x530);
+static_assert(offsetof(SwkbdConfigNative, use_text_check) == 0x48D);
+static_assert(offsetof(SwkbdConfigNative, text_grouping) == 0x490);
#pragma pack(pop)
+inline bool DecodeNativeSwkbdConfig(std::span<const u8> data, SwkbdConfigCommon& common,
+ SwkbdConfigNew& config) {
+ if (data.size() != sizeof(SwkbdConfigNative)) {
+ return false;
+ }
+ SwkbdConfigNative native{};
+ std::memcpy(&native, data.data(), sizeof(native));
+ common = {};
+ config = {};
+ std::memcpy(static_cast<void*>(&common), native.common_prefix.data(), native.common_prefix.size());
+ common.enable_return_button = native.common_prefix[0x1B] != 0;
+ common.use_utf8 = native.use_utf8;
+ common.use_blur_background = native.use_blur_background;
+ common.max_text_length = native.max_text_length;
+ common.min_text_length = native.min_text_length;
+ common.password_mode = native.password_mode;
+ common.text_draw_type = native.text_draw_type;
+ common.initial_string_offset = native.initial_string_offset;
+ common.initial_string_length = native.initial_string_length;
+ common.user_dictionary_offset = native.user_dictionary_offset;
+ common.user_dictionary_entries = native.user_dictionary_entries;
+ common.use_text_check = native.use_text_check;
+ config.text_grouping = native.text_grouping;
+ config.customized_dictionary_set_entries = native.customized_dictionary_set_entries;
+ config.total_customized_dictionary_set_entries = native.total_customized_dictionary_set_entries;
+ config.disable_cancel_button = native.disable_cancel_button;
+ return true;
+}
+
struct SwkbdTextCheck {
SwkbdTextCheckResult text_check_result{};
std::array<char16_t, STRING_BUFFER_SIZE / 2> text_check_message{};
diff --git a/src/core/hle/service/nvdrv/core/container.cpp b/src/core/hle/service/nvdrv/core/container.cpp
index 9edce03f64..22db243e3f 100644
--- a/src/core/hle/service/nvdrv/core/container.cpp
+++ b/src/core/hle/service/nvdrv/core/container.cpp
@@ -11,6 +11,7 @@
#include "core/hle/service/nvdrv/core/heap_mapper.h"
#include "core/hle/service/nvdrv/core/nvmap.h"
#include "core/hle/service/nvdrv/core/syncpoint_manager.h"
+#include "core/hle/service/nvdrv/devices/nx2_stub_device.h"
#include "core/memory.h"
#include "video_core/host1x/host1x.h"
@@ -130,6 +131,7 @@ void Container::CloseSession(SessionId session_id) {
session.has_preallocated_area = false;
}
session.is_active = false;
+ Devices::Nx2GpuMemoryRegistry::Get().ForgetDeviceRanges(session.asid.id);
smmu.UnregisterProcess(impl->sessions[session_id.id].asid);
impl->id_pool.emplace_front(session_id.id);
}
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
index e66c72cd26..9d696c027e 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
@@ -5,6 +5,7 @@
// SPDX-FileCopyrightText: 2021 Skyline Team and Contributors
// SPDX-License-Identifier: GPL-3.0-or-later
+#include <cstdlib>
#include <cstring>
#include <utility>
@@ -35,6 +36,13 @@ nvhost_as_gpu::nvhost_as_gpu(Core::System& system_, Module& module_, NvCore::Con
nvhost_as_gpu::~nvhost_as_gpu() {
Nx2GpuMemoryRegistry::Get().RemoveListener(this);
+ // Leave shared ranges to the process's other address spaces; like this address space's own
+ // ranges, one nobody else maps stays allocated.
+ for (const auto& [key, range] : nx2_device_ranges) {
+ if (range.shared) {
+ Nx2GpuMemoryRegistry::Get().ReleaseDeviceRange(range.asid, key.first, key.second);
+ }
+ }
}
NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input,
@@ -134,25 +142,37 @@ void nvhost_as_gpu::Nx2Map(u64 gpu_address, u64 cpu_address, u64 size) {
}
// The same CPU range is often mapped twice (at its own address and in a sparse region), so
// reuse its SMMU mapping.
- DAddr device_address{};
- bool heap = false;
+ Nx2DeviceRange range{};
if (const auto it = nx2_device_ranges.find({cpu_address, size}); it != nx2_device_ranges.end()) {
- device_address = it->second.address;
- heap = it->second.heap;
+ range = it->second;
} else if (session->has_preallocated_area && session->mapper->IsInBounds(cpu_address, size)) {
- device_address = session->mapper->Map(cpu_address, size);
- heap = true;
+ range = {session->mapper->Map(cpu_address, size), true};
} else {
- auto& smmu = system.Host1x().MemoryManager();
- device_address = smmu.Allocate(Common::AlignUp(size, 0x20000));
- if (device_address == 0) {
+ const auto allocate = [&]() -> u64 {
+ auto& smmu = system.Host1x().MemoryManager();
+ const DAddr address = smmu.Allocate(Common::AlignUp(size, 0x20000));
+ if (address != 0) {
+ smmu.Map(address, cpu_address, size, session->asid, true);
+ }
+ return address;
+ };
+ // EDEN_NX2_PRIVATE_DEVICE_RANGES=1 (A/B aid): a range per address space.
+ static const bool private_ranges = std::getenv("EDEN_NX2_PRIVATE_DEVICE_RANGES") != nullptr;
+ if (private_ranges) {
+ range = {allocate(), false};
+ } else {
+ range = {Nx2GpuMemoryRegistry::Get().AcquireDeviceRange(session->asid.id, cpu_address,
+ size, allocate),
+ false, true, session->asid.id};
+ }
+ if (range.address == 0) {
LOG_CRITICAL(Service_NVDRV, "Switch 2 GPU memory {:#x}+{:#x}: out of SMMU space",
cpu_address, size);
return;
}
- smmu.Map(device_address, cpu_address, size, session->asid, true);
}
- nx2_device_ranges[{cpu_address, size}] = {device_address, heap};
+ const DAddr device_address = range.address;
+ nx2_device_ranges[{cpu_address, size}] = range;
Nx2GpuMemoryRegistry::Get().RecordDeviceRange(cpu_address, size, device_address);
gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false);
LOG_DEBUG(Service_NVDRV, "Switch 2 GPU mapping {:#x}+{:#x} -> CPU {:#x}", gpu_address, size,
@@ -177,6 +197,10 @@ void nvhost_as_gpu::Nx2Release(u64 cpu_address, u64 size) {
}
const Nx2DeviceRange range = it->second;
nx2_device_ranges.erase(it);
+ if (range.shared &&
+ !Nx2GpuMemoryRegistry::Get().ReleaseDeviceRange(range.asid, cpu_address, size)) {
+ return; // another address space of the process still maps it
+ }
auto* session = container.GetSession(nx2_session);
auto* const mapper = range.heap && session != nullptr ? session->mapper.get() : nullptr;
auto& smmu = system.Host1x().MemoryManager();
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
index 39b63ea369..37b6d0d4c4 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
+++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
@@ -238,6 +238,8 @@ private:
struct Nx2DeviceRange {
DAddr address;
bool heap; ///< Mapped through the session's heap mapper (else an SMMU allocation)
+ bool shared{}; ///< A reference to the process's range in Nx2GpuMemoryRegistry
+ size_t asid{};
};
std::map<std::pair<u64, u64>, Nx2DeviceRange> nx2_device_ranges;
void Nx2Release(u64 cpu_address, u64 size);
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
index 5998940afa..cf4a473496 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
@@ -113,12 +113,22 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8>
case 0x6:
return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetTPCMasks3, input, output,
inline_output);
- case 0x13:
- LOG_DEBUG(Service_NVDRV, "(STUBBED) called.");
-
- // TODO (jarrodnorwell)
-
- return NvResult::NotImplemented;
+ case 0x13: {
+ // NVGPU_GPU_IOCTL_VSMS_MAPPING: an 8-byte argument struct plus a buffer of
+ // {gpc_index, tpc_index} byte pairs, one per virtual SM. The SDK treats a pair left at
+ // its 0xc0c0 sentinel as failure, so fill every pair: one GPC, two SMs per TPC.
+ if (command.raw != 0xc0084713 || input.size() != 8 || output.size() != 8 ||
+ inline_output.size() < 2) {
+ return NvResult::BadParameter;
+ }
+ std::memcpy(output.data(), input.data(), output.size());
+ for (size_t i = 0; i + 1 < inline_output.size(); i += 2) {
+ inline_output[i] = 0; // gpc_index
+ inline_output[i + 1] = static_cast<u8>(i / 4); // tpc_index
+ }
+ LOG_DEBUG(Service_NVDRV, "VSMS mapping for {} SMs", inline_output.size() / 2);
+ return NvResult::Success;
+ }
default:
break;
}
@@ -218,6 +228,9 @@ void ApplyNx2Characteristics(GC& gc) {
if (remap) {
gc.flags |= 1ULL << 35;
}
+ // Bits 40..43: the 2D, 3D, compute and inline-to-memory class families. The SDK filters its
+ // class enumeration by them; without them Fire Emblem: Fortune's Weave aborts at boot.
+ gc.flags |= 0xfULL << 40;
// The Switch 2 layout keeps nine s16 interface versions at 0x70..0x80 (like nvgpu's
// *_ioctl_nr_last); its NvRm requires e.g. [0x78] >= 6 and enables features above 0xa, 0x1f
diff --git a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h
index 869e17caf6..8dee9ccfb7 100644
--- a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h
+++ b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h
@@ -5,6 +5,7 @@
#include <algorithm>
#include <atomic>
+#include <compare>
#include <functional>
#include <map>
#include <mutex>
@@ -166,12 +167,50 @@ public:
{
std::scoped_lock lk{device_mutex};
device_ranges.clear();
+ shared_ranges.clear();
}
std::scoped_lock lk{mutex};
mappings.clear();
sparse_regions.clear();
}
+ // Device (SMMU) range of a mapping outside the heap, shared by every address space of one
+ // process: the caches key memory by device address, so a range per address space made separate
+ // copies of the same memory, and presentation could read one the game never drew into. The
+ // first address space to map the range allocates it with `allocate` (0: out of space); every
+ // call takes a reference.
+ u64 AcquireDeviceRange(size_t asid, u64 cpu_address, u64 size,
+ const std::function<u64()>& allocate) {
+ std::scoped_lock lk{device_mutex};
+ const SharedRangeKey key{asid, cpu_address, size};
+ if (const auto it = shared_ranges.find(key); it != shared_ranges.end()) {
+ ++it->second.references;
+ return it->second.device_address;
+ }
+ const u64 device_address = allocate();
+ if (device_address != 0) {
+ shared_ranges.emplace(key, SharedRange{device_address, 1});
+ }
+ return device_address;
+ }
+
+ // Drops a reference. True for the last one: the caller frees the range.
+ bool ReleaseDeviceRange(size_t asid, u64 cpu_address, u64 size) {
+ std::scoped_lock lk{device_mutex};
+ const auto it = shared_ranges.find(SharedRangeKey{asid, cpu_address, size});
+ if (it == shared_ranges.end() || --it->second.references != 0) {
+ return false;
+ }
+ shared_ranges.erase(it);
+ return true;
+ }
+
+ // The process's session closed and its ASID can be reused: stop sharing its ranges.
+ void ForgetDeviceRanges(size_t asid) {
+ std::scoped_lock lk{device_mutex};
+ std::erase_if(shared_ranges, [asid](const auto& entry) { return entry.first.asid == asid; });
+ }
+
// Device (SMMU) address of a CPU range an address space mapped; used to present images.
void RecordDeviceRange(u64 cpu_address, u64 size, u64 device_address) {
std::scoped_lock lk{device_mutex};
@@ -197,6 +236,18 @@ private:
std::mutex device_mutex;
std::map<u64, std::pair<u64, u64>> device_ranges; // cpu start -> (size, device address)
+ struct SharedRangeKey {
+ size_t asid;
+ u64 cpu_address;
+ u64 size;
+ auto operator<=>(const SharedRangeKey&) const = default;
+ };
+ struct SharedRange {
+ u64 device_address;
+ u32 references;
+ };
+ std::map<SharedRangeKey, SharedRange> shared_ranges;
+
struct Mapping {
u64 gpu_address;
u64 cpu_address;
diff --git a/src/core/hle/service/nvnflinger/pixel_format.h b/src/core/hle/service/nvnflinger/pixel_format.h
index b6e8b5f8a0..a38c8be777 100644
--- a/src/core/hle/service/nvnflinger/pixel_format.h
+++ b/src/core/hle/service/nvnflinger/pixel_format.h
@@ -19,6 +19,7 @@ enum class PixelFormat : u32 {
Bgra8888 = 5,
Rgba5551 = 6,
Rgba4444 = 7,
+ Rgba1010102 = 43,
};
} // namespace Service::android
diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h
index 628bdeda5b..5a2963f179 100644
--- a/src/core/hle/service/nx2_gsv.h
+++ b/src/core/hle/service/nx2_gsv.h
@@ -21,6 +21,7 @@
#include <optional>
#include <memory>
#include <mutex>
+#include <utility>
#include <vector>
#include "common/alignment.h"
@@ -45,13 +46,17 @@
#include "core/hle/kernel/k_resource_limit.h"
#include "core/hle/kernel/k_server_session.h"
#include "core/hle/kernel/k_event.h"
+#include "core/hle/kernel/k_hardware_timer.h"
#include "core/hle/kernel/k_readable_event.h"
#include "core/hle/kernel/k_session.h"
+#include "core/hle/kernel/k_thread.h"
#include "core/hle/kernel/k_transfer_memory.h"
+#include "core/hle/kernel/svc_results.h"
#include "core/memory.h"
#include "video_core/host1x/host1x.h"
#include "video_core/host1x/nx2_video_decoder.h"
#include "video_core/nx2.h"
+#include "video_core/textures/decoders.h"
#include "core/hle/service/hle_ipc.h"
#include "core/hle/service/ipc_helpers.h"
#include "core/hle/service/kernel_helpers.h"
@@ -304,6 +309,8 @@ public:
{21, &INx2GpuSession::Start, "Start (guessed)"},
{22, &INx2GpuSession::OpenSubSession, "OpenSubSession (guessed)"},
{33, &INx2GpuSession::GetCount33, "Unknown33 (u32 count)"},
+ {41, &INx2GpuSession::CreateGraphicsDevice, "CreateGraphicsDevice"},
+ {42, &INx2GpuSession::DestroyGraphicsDevice, "DestroyGraphicsDevice"},
{201, &INx2GpuSession::GetUserPartition, "GetUserPartition (newer SDKs)"},
{202, &INx2GpuSession::GetGsvPartition, "GetGsvPartition (newer SDKs)"},
{203, &INx2GpuSession::GetVaRange, "GetVaRange (guessed)"},
@@ -395,9 +402,39 @@ private:
static_cast<u32>(second), static_cast<u32>(second >> 32)});
}
+ // An empty partition is min = ~0, max = 0 (min > max). Fire Emblem: Fortune's Weave's SDK
+ // reads (0, 0) as a one-byte partition at address 0 and aborts in nn::gpu::sdk.
void GetGsvPartition(HLERequestContext& ctx) {
LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 202 [2]={:#x}, empty", ctx.CommandBuffer()[2]);
- TipcReplyWords(ctx, {0, 0, 0, 0});
+ TipcReplyWords(ctx, {~0U, ~0U, 0, 0});
+ }
+
+ // Fire Emblem: Fortune's Weave: 41(name u64, flags u64, debug label u64) returns a device id
+ // that later NvRm calls name; 0 is not a valid id. Only flags 0 has been seen.
+ void CreateGraphicsDevice(HLERequestContext& ctx) {
+ const auto* b = ctx.CommandBuffer();
+ const u64 name = u64{b[2]} | (u64{b[3]} << 32);
+ const u64 flags = u64{b[4]} | (u64{b[5]} << 32);
+ u32 id = 0;
+ for (u32 candidate = 1; candidate < m_devices.size(); ++candidate) {
+ if (!m_devices[candidate]) {
+ m_devices[candidate] = true;
+ id = candidate;
+ break;
+ }
+ }
+ LOG_INFO(Service, "nx2:gpu 41 create graphics device name={:#x} flags={:#x} -> id {}", name,
+ flags, id);
+ IPC::ResponseBuilder rb{ctx, 3};
+ rb.Push(id != 0 ? ResultSuccess : Kernel::ResultOutOfResource);
+ rb.Push(id);
+ }
+
+ void DestroyGraphicsDevice(HLERequestContext& ctx) {
+ const u32 id = ctx.CommandBuffer()[2] & 0xff;
+ const bool existed = id < m_devices.size() && std::exchange(m_devices[id], false);
+ IPC::ResponseBuilder rb{ctx, 2};
+ rb.Push(existed ? ResultSuccess : Kernel::ResultNotFound);
}
void OpenSubSession(HLERequestContext& ctx) {
@@ -1246,6 +1283,7 @@ private:
std::array<Kernel::KEvent*, NumIds> m_events{};
std::map<u32, GpuMemory> m_memory;
u32 m_next_memory_handle{1};
+ std::array<bool, 64> m_devices{}; // gpu 41 graphics device ids (0 unused)
};
// Native memory and images handed out by the Switch 2 graphics server (gsv 111/114/423/429/315),
@@ -1264,6 +1302,10 @@ struct Nx2NativeObjects {
u64 offset{}; // into its memory (images of a decoder pool share one)
bool window_pool{};
bool decoder{}; // a video decoder output image (gsv 911)
+ // A decoder image the client reads on the GPU (gsv 428/426): written block-linear.
+ bool gpu_read{};
+ u32 gpu_maps{}; // outstanding 426 maps
+ bool mapped_here{}; // 426 added its GPU mapping (no other mapping covered it)
};
std::mutex mutex;
std::map<u32, Memory> memories;
@@ -1407,11 +1449,12 @@ public:
{11, &INx2Vi2Session::ReturnEvent, "Unknown11"},
{31, &INx2Vi2Session::GetVsyncEvent, "GetVsyncEvent (guessed)"},
{32, &INx2Vi2Session::GetVsyncEvent, "GetDisplayVsyncEvent (guessed)"},
- // Title 1 calls these ~75 times a second with the display name "default"; the
+ {104, &INx2Vi2Session::GetHdrStatus, "GetHdrStatus"},
+ // Title 1 calls this ~75 times a second with the display name "default"; the
// auto stub's empty success reply works, but logged two lines per call (a 50 MB log
// in an 18 minute session).
- {104, &INx2Vi2Session::SucceedQuietly, "Unknown104 (display \"default\")"},
{105, &INx2Vi2Session::SucceedQuietly, "Unknown105 (display \"default\")"},
+ {106, &INx2Vi2Session::GetBufferMaxResolution, "GetBufferMaxResolution"},
{228, &INx2Vi2Session::CancelClientBuffer, "CancelClientBuffer"},
{229, &INx2Vi2Session::PresentClientBuffer, "PresentClientBuffer"},
});
@@ -1464,9 +1507,9 @@ private:
WaitForFrame(~w[22], w[24]);
}
const auto waited = std::chrono::steady_clock::now();
- Composite(w[5], ~w[22], w[24]);
+ Composite(w[5], ~w[22], w[24], w[7]);
const auto composited = std::chrono::steady_clock::now();
- PaceToVblank();
+ PaceToVblank(w[12]);
const auto paced = std::chrono::steady_clock::now();
// Where each frame's time goes on the present thread, averaged per 600 presents: the
// game is blocked here while Eden's GPU thread catches up (fence) and composites.
@@ -1522,18 +1565,39 @@ private:
// present faster than that. Every slot is reported free here: hold each present until the
// next vblank instead (title 4 ran at up to 160 fps, its timing and audio with it).
std::chrono::steady_clock::time_point m_next_vblank{};
+ u32 m_swap_word{};
- void PaceToVblank() {
+ // timing: present word 12, (swap interval << 16) | refresh rate. Fire Emblem: Fortune's Weave
+ // sends 0x1003c during its boot logos and 0x2003c (two vblanks per frame, 30 fps) after; paced
+ // at every vblank it ran at double speed. EDEN_NX2_SWAP_INTERVAL=<n> forces the interval.
+ void PaceToVblank(u32 timing) {
if (!Settings::values.use_speed_limit.GetValue() || Settings::SpeedLimit() == 0) {
return;
}
- const auto interval = std::chrono::nanoseconds{100'000'000'000LL / 60 / Settings::SpeedLimit()};
+ static const u32 forced_interval = [] {
+ const char* value = std::getenv("EDEN_NX2_SWAP_INTERVAL");
+ return value ? static_cast<u32>(std::strtoul(value, nullptr, 10)) : 0U;
+ }();
+ const u32 swap_interval =
+ forced_interval != 0 ? forced_interval : std::clamp<u32>(timing >> 16, 1, 4);
+ const u32 refresh_rate = (timing & 0xffff) >= 30 && (timing & 0xffff) <= 240 ? timing & 0xffff : 60;
+ if (timing != std::exchange(m_swap_word, timing)) {
+ LOG_INFO(Service, "nx2:vi2 present #{} swap interval {} at {} Hz (word 12 = {:#x})",
+ m_presents, swap_interval, refresh_rate, timing);
+ }
+ const auto interval = std::chrono::nanoseconds{
+ 100'000'000'000LL * swap_interval / refresh_rate / Settings::SpeedLimit()};
const auto now = std::chrono::steady_clock::now();
if (now > m_next_vblank + interval) {
- // Running slower than 60 fps: show at once and don't bank the lost time.
+ // Running slower than the swap interval: show at once and don't bank the lost time.
m_next_vblank = now;
} else if (now < m_next_vblank) {
- std::this_thread::sleep_until(m_next_vblank);
+ // Sleep this service thread in the guest kernel. It runs on guest core 3, and a host
+ // sleep held that core for the whole wait: game threads scheduled there ran only
+ // between presents (Fortune's Weave's loader read one RomFS block per frame).
+ const auto wait = std::chrono::duration_cast<std::chrono::nanoseconds>(m_next_vblank - now);
+ static_cast<void>(Kernel::GetCurrentThread(kernel).Sleep(
+ kernel, kernel.HardwareTimer().GetTick() + wait.count() + 2));
}
m_next_vblank += interval;
}
@@ -1586,9 +1650,12 @@ private:
// Shows a presented native image: word 5 is the image id, ~word 22 / word 24 the GPU fence
// (syncpoint, value) to wait for (layout from changes.patch). The image is a block-linear
// RGBA8 surface in the console's native GOB layout at GPU address == CPU address.
- void Composite(u32 image_id, u32 fence_id, u32 fence_value) {
+ // present_size: word 7 of the present, (height - 1) << 16 | (width - 1) of the frame drawn
+ // into the image's top-left corner (Fire Emblem: Fortune's Weave: 1280x720 in 3840x2160).
+ void Composite(u32 image_id, u32 fence_id, u32 fence_value, u32 present_size = 0) {
u64 address{};
Nx2ImageLayout layout{};
+ u32 image_format{};
{
std::scoped_lock lk{m_objects->mutex};
const auto image = m_objects->images.find(image_id);
@@ -1601,6 +1668,7 @@ private:
}
address = memory->second.address;
layout = Nx2CalculateImageLayout(image->second.preference.data());
+ image_format = image->second.preference[6];
}
const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address);
if (!device) {
@@ -1610,6 +1678,11 @@ private:
}
u32 crop_width = layout.width;
u32 crop_height = layout.height;
+ // Present word 7 holds the size of the frame drawn into the image's top-left corner.
+ if (present_size != 0) {
+ crop_width = (std::min)(crop_width, (present_size & 0xffffU) + 1);
+ crop_height = (std::min)(crop_height, (present_size >> 16) + 1);
+ }
// Handheld titles keep their docked present images (up to 3840x2160) and draw the frame
// into the top-left 1920x1080, the handheld panel's resolution; the console's display
// scans out only that region. Showing whole images put the frame in a corner inside black
@@ -1648,7 +1721,9 @@ private:
.width = layout.width,
.height = layout.height,
.stride = static_cast<u32>(layout.pitch / 4),
- .pixel_format = android::PixelFormat::Rgba8888,
+ // Preference word 6: 1 = RGBA8, 4 = RGB10A2.
+ .pixel_format = image_format == 4 ? android::PixelFormat::Rgba1010102
+ : android::PixelFormat::Rgba8888,
.transform_flags = android::BufferTransformFlags::Unset,
.crop_rect = {0, 0, static_cast<int>(crop_width), static_cast<int>(crop_height)},
.blending = Tegra::BlendMode::Opaque,
@@ -1752,6 +1827,50 @@ private:
TipcReplyWords(ctx, {1});
}
+ // 104(u64 display name): result, u64 output format, u32 0. Format 1 is SDR. A bare success
+ // left the format to stale reply words and a game could set up HDR output from them.
+ void GetHdrStatus(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 name = u64{packet[2]} | (u64{packet[3]} << 32);
+ constexpr u64 DefaultDisplay = 0x00746c7561666564ULL; // "default"
+ Result result = Kernel::ResultInvalidCombination;
+ u64 format{};
+ if (packet[0] == 0x78 && packet[1] == 2) {
+ result = Result{ErrorModule::VI, 7};
+ if (name == DefaultDisplay) {
+ format = 1;
+ result = ResultSuccess;
+ }
+ }
+ IPC::ResponseBuilder rb{ctx, 5};
+ rb.Push(result.raw);
+ rb.Push(format);
+ rb.Push<u32>(0);
+ }
+
+ // The highest buffer resolution mode of the "default" display. Fire Emblem: Fortune's Weave picks
+ // its output from it: 1 -> 1280x720, 2 -> 1920x1080, 4 and 8 -> 2560x1440; 0 (the auto stub's
+ // reply), 3 and 5 fall back to 720p. Docked reports 4, handheld 2 (the screen is 1080p);
+ // EDEN_VI2_MAX_RESOLUTION_MODE=<n> overrides.
+ void GetBufferMaxResolution(HLERequestContext& ctx) {
+ const auto* packet = ctx.CommandBuffer();
+ const u64 name = u64{packet[2]} | (u64{packet[3]} << 32);
+ constexpr u64 DefaultDisplay = 0x00746c7561666564ULL; // "default"
+ Result result = Kernel::ResultInvalidCombination;
+ u64 mode{};
+ if (packet[0] == 0x7a && packet[1] == 2) {
+ result = Result{ErrorModule::VI, 7};
+ if (name == DefaultDisplay) {
+ static const char* forced = std::getenv("EDEN_VI2_MAX_RESOLUTION_MODE");
+ mode = forced ? std::strtoull(forced, nullptr, 0) : Settings::IsDockedMode() ? 4 : 2;
+ result = ResultSuccess;
+ }
+ }
+ IPC::ResponseBuilder rb{ctx, 4};
+ rb.Push(result.raw);
+ rb.Push(mode);
+ }
+
// The same reply as the auto stub (success, no data), without its log lines.
void SucceedQuietly(HLERequestContext& ctx) {
IPC::ResponseBuilder rb{ctx, 2};
@@ -2221,6 +2340,7 @@ private:
const Decoded& frame, std::vector<u8>& bytes) {
u64 address{};
Nx2DecoderImageLayout layout{};
+ bool block_linear{};
{
std::scoped_lock lk{m_objects->mutex};
const auto image = m_objects->images.find(image_id);
@@ -2228,6 +2348,8 @@ private:
return {m_width, m_height};
}
layout = Nx2CalculateDecoderImageLayout(image->second.preference.data());
+ block_linear = image->second.gpu_read &&
+ layout.format != Tegra::Host1x::Nx2PictureFormat::Nv12;
const auto memory = m_objects->memories.find(image->second.memory);
if (memory != m_objects->memories.end() && memory->second.address != 0) {
address = memory->second.address + image->second.offset;
@@ -2241,10 +2363,25 @@ private:
const u32 width = std::min(frame.picture->width, layout.width);
const u32 height = std::min(frame.picture->height, layout.height);
bytes.resize(layout.size);
+ auto& memory = system.ApplicationMemory();
+ if (block_linear) {
+ // Read on the GPU: RGBA (the blit's format 0xd5) in the native block-linear layout,
+ // 16-GOB blocks, as our Eden tree writes these images.
+ decoder->Render(*frame.picture, Tegra::Host1x::Nx2PictureFormat::Rgba8, width, height,
+ layout.pitch, 0, bytes);
+ m_tiled.resize(layout.size);
+ Tegra::Texture::SwizzleTexture(m_tiled, bytes, 4, static_cast<u32>(layout.pitch / 4),
+ layout.height, 1, 4, 0, 1, true);
+ memory.WriteBlock(address, m_tiled.data(), m_tiled.size());
+ if (const auto device =
+ Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address)) {
+ system.GPU().InvalidateRegion(*device, m_tiled.size());
+ }
+ return {width, height};
+ }
decoder->Render(*frame.picture, layout.format, width, height, layout.pitch,
layout.chroma_offset, bytes);
DumpFrame(bytes, layout, width, height);
- auto& memory = system.ApplicationMemory();
memory.WriteBlock(address, bytes.data(), layout.pitch * height);
if (layout.format == Tegra::Host1x::Nx2PictureFormat::Nv12) {
memory.WriteBlock(address + layout.chroma_offset, bytes.data() + layout.chroma_offset,
@@ -2290,6 +2427,7 @@ private:
u64 m_submitted{};
u32 m_syncpoint{NoSyncpoint}; ///< Counts finished frames (allocated at the first register)
u32 m_sync_base{}; ///< Its value before the first frame
+ std::vector<u8> m_tiled; ///< Decode thread: block-linear copy of a frame (WriteImage)
// Shared with the decode thread.
std::mutex m_mutex;
@@ -2333,6 +2471,7 @@ public:
{429, &INx2GraphicsSystemSession::CreateNativeImage, "CreateNativeImage"},
{430, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyNativeImage"},
{426, &INx2GraphicsSystemSession::QueryNativeImage, "MapNativeImage"},
+ {427, &INx2GraphicsSystemSession::UnmapNativeImage, "UnmapNativeImage"},
{428, &INx2GraphicsSystemSession::QueryNativeImage, "QueryNativeImage"},
{435, &INx2GraphicsSystemSession::CalculateNativeImageSize, "CalculateNativeImageSize"},
{1402, &INx2GraphicsSystemSession::OpenNvsrvSession, "OpenNvsrvSession (guessed)"},
@@ -2597,8 +2736,19 @@ private:
void DestroyNativeMemory(HLERequestContext& ctx) {
const u32 id = ctx.CommandBuffer()[2];
- std::scoped_lock lk{m_objects->mutex};
- m_objects->memories.erase(id);
+ std::vector<std::optional<std::pair<u64, u64>>> unmaps;
+ {
+ std::scoped_lock lk{m_objects->mutex};
+ for (auto& [image_id, image] : m_objects->images) {
+ if (image.memory == id && image.mapped_here) {
+ unmaps.push_back(TakeDecoderMapping(image));
+ }
+ }
+ m_objects->memories.erase(id);
+ }
+ for (const auto& range : unmaps) {
+ DropDecoderMapping(range);
+ }
LOG_DEBUG(Service, "nx2:gsv 112 DestroyNativeMemory id {}", id);
TipcReplyWords(ctx, {});
}
@@ -2673,6 +2823,9 @@ private:
// the 429 preference words (0x10, 0x14), a guess.
void QueryNativeImage(HLERequestContext& ctx) {
const u32 handle = ctx.CommandBuffer()[2];
+ if (QueryDecoderImage(ctx, handle)) {
+ return;
+ }
std::array<u32, 16> preference{};
u64 image_offset = 0;
u64 memory_address = 0;
@@ -2717,6 +2870,108 @@ private:
}
}
+ // 428/426 for a decoder output image (Fire Emblem: Fortune's Weave's movie player blits each
+ // one, block-linear RGBA, format 0xd5, block height 16 GOBs, into a texture of its own). The
+ // reply follows our Eden tree, where the movies play: words 3-4 the GPU address, 5-6 the last
+ // byte's address, then the descriptor {1, 0, dimensions, .., 0x01000001, 1, offset, size}
+ // (offset 0 once mapped). 426 maps the image at its own address for the GPU: decoder pool
+ // memory is process memory (114) that no GPU mapping covers, and the blit read zeros.
+ bool QueryDecoderImage(HLERequestContext& ctx, u32 handle) {
+ const bool map = ctx.GetCommand() == 426;
+ u64 address{};
+ u64 size{};
+ u64 offset{};
+ u32 dimensions{};
+ bool register_mapping = false;
+ {
+ std::scoped_lock lk{m_objects->mutex};
+ const auto image = m_objects->images.find(handle);
+ if (image == m_objects->images.end() || !image->second.decoder) {
+ return false;
+ }
+ const auto memory = m_objects->memories.find(image->second.memory);
+ if (memory == m_objects->memories.end() || memory->second.address == 0) {
+ return false;
+ }
+ image->second.gpu_read = true;
+ offset = image->second.offset;
+ size = image->second.size;
+ dimensions = image->second.preference[2];
+ if (map) {
+ address = memory->second.address + offset;
+ register_mapping = image->second.gpu_maps++ == 0 &&
+ !Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address);
+ image->second.mapped_here |= register_mapping;
+ }
+ }
+ if (register_mapping) {
+ Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Register(address, size);
+ }
+ LOG_INFO(Service, "nx2:gsv {} decoder image {} at {:#x}+{:#x}{}", ctx.GetCommand(), handle,
+ address, size, register_mapping ? ", mapped for the GPU" : "");
+ std::array<u32, 37> words{};
+ if (map) {
+ const u64 last = address + size - 1;
+ words[0] = static_cast<u32>(address);
+ words[1] = static_cast<u32>(address >> 32);
+ words[2] = static_cast<u32>(last);
+ words[3] = static_cast<u32>(last >> 32);
+ }
+ u32* const desc = words.data() + 4;
+ desc[0] = 1;
+ desc[2] = dimensions;
+ desc[8] = 0x01000001;
+ desc[9] = 1;
+ desc[10] = map ? 0 : static_cast<u32>(offset);
+ desc[11] = static_cast<u32>(size);
+ IPC::ResponseBuilder rb{ctx, static_cast<u32>(2 + words.size())};
+ rb.PushRaw<u32>(ResultSuccess.raw);
+ for (const u32 w : words) {
+ rb.Push<u32>(w);
+ }
+ return true;
+ }
+
+ // 427(u32 handle, u32 address space, u64 address, u64 last address): undoes a 426. The last
+ // unmap of a decoder image drops the GPU mapping 426 made.
+ void UnmapNativeImage(HLERequestContext& ctx) {
+ const auto* w = ctx.CommandBuffer();
+ const u32 handle = w[2];
+ std::optional<std::pair<u64, u64>> unmap;
+ {
+ std::scoped_lock lk{m_objects->mutex};
+ if (const auto image = m_objects->images.find(handle);
+ image != m_objects->images.end() && image->second.gpu_maps != 0 &&
+ --image->second.gpu_maps == 0) {
+ unmap = TakeDecoderMapping(image->second);
+ }
+ }
+ LOG_DEBUG(Service, "nx2:gsv 427 image {} [{:#x}, {:#x}]", handle, Pair(w, 4), Pair(w, 6));
+ DropDecoderMapping(unmap);
+ TipcReplyWords(ctx, {});
+ }
+
+ // The range 426 mapped for an image, if it did; called with m_objects->mutex held.
+ std::optional<std::pair<u64, u64>> TakeDecoderMapping(Nx2NativeObjects::Image& image) {
+ image.gpu_maps = 0;
+ if (!std::exchange(image.mapped_here, false)) {
+ return std::nullopt;
+ }
+ const auto memory = m_objects->memories.find(image.memory);
+ if (memory == m_objects->memories.end() || memory->second.address == 0) {
+ return std::nullopt;
+ }
+ return std::pair{memory->second.address + image.offset, image.size};
+ }
+
+ static void DropDecoderMapping(const std::optional<std::pair<u64, u64>>& range) {
+ if (range) {
+ auto& registry = Nvidia::Devices::Nx2GpuMemoryRegistry::Get();
+ registry.Unmap(range->first, range->second);
+ registry.ReleaseBacking(range->first, range->second);
+ }
+ }
+
// 911(preference[16], u32 count, u32 memory id) (nn::codec::InitializeDecoderImagePool):
// result, the id of the first of `count` decoder output images; the client names image i of
// the pool id + i. They share the pool's memory, one image size apart.
@@ -2775,8 +3030,15 @@ private:
void DestroyNativeImage(HLERequestContext& ctx) {
const u32 id = ctx.CommandBuffer()[2];
- std::scoped_lock lk{m_objects->mutex};
- m_objects->images.erase(id);
+ std::optional<std::pair<u64, u64>> unmap;
+ {
+ std::scoped_lock lk{m_objects->mutex};
+ if (const auto image = m_objects->images.find(id); image != m_objects->images.end()) {
+ unmap = TakeDecoderMapping(image->second);
+ m_objects->images.erase(image);
+ }
+ }
+ DropDecoderMapping(unmap);
TipcReplyWords(ctx, {});
}
diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h
index 031f7fac75..489d003e60 100644
--- a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h
+++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h
@@ -56,7 +56,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
for (const auto& instruction : code) {
const u32 op = instruction.Bits(0, 12);
uses_uniform_bank |= op == 0x3c4 || op == 0x2bd || op == 0xc0c || op == 0x986 || op == 0x9a8 || op == 0xc12 || op == 0xc17 || op == 0xe24 || op == 0xc24 || op == 0xd00 || op == 0xd09 || op == 0x9c3 || op == 0xc45 || op == 0x2a4 || op == 0x891 || op == 0x890 || op == 0x290 || op == 0x28c || op == 0x88c || op == 0x287 || op == 0x887 || op == 0x958 || op == 0xab9 || op == 0xabb || op == 0x899 || op == 0xc20 || op == 0xe21 || op == 0xe23 || op == 0xc02 ||
- op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || op == 0x292 || op == 0x892 || op == 0x291 || op == 0x981 || op == 0xc13 || op == 0xc10 || op == 0xc23 || op == 0x526 || op == 0x2be || op == 0x8a4 || op == 0xd05 || (op == 0x984 && instruction.Bits(91, 1)) || op == 0x89c || op == 0x883 || op == 0x2a5 || op == 0xd06 || op == 0x299 || op == 0xc09 || (op == 0x81c && instruction.Bits(67, 1));
+ op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || op == 0x292 || op == 0x892 || op == 0x291 || op == 0x981 || op == 0xc13 || op == 0xc10 || op == 0xc23 || op == 0x526 || op == 0x2be || op == 0x8a4 || op == 0xd05 || (op == 0x984 && instruction.Bits(91, 1)) || op == 0x89c || op == 0x883 || op == 0x2a5 || op == 0xd06 || op == 0x299 || op == 0xc09 || (op == 0x81c && instruction.Bits(67, 1)) || op == 0xf60 || op == 0xc11 || op == 0xc0a;
}
const auto address_of = [](u32 index) { return 0x50U + index * 8 + ((index + 1) / 3) * 8; };
const auto fail = [&](u32 index, std::string_view reason) {
@@ -92,14 +92,20 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
active_link = call_links[call];
}
}
- // RET has no destination GPR; bits 16..23 are reserved, not a write to R0.
- if (active_link && opcode != 0x950 && insn.Bits(16, 8) == *active_link)
+ // RET, BRA, NOP and ISETP/FSETP (predicate outputs) have no destination GPR; bits 16..23
+ // are not a write to R0. IMAD.WIDE writes a register pair.
+ const bool no_destination = opcode == 0x950 || opcode == 0x947 || opcode == 0x918 ||
+ (opcode & 0x1ff) == 0x00c || (opcode & 0x1ff) == 0x00b;
+ const bool pair = opcode == 0x225 || opcode == 0x625 || opcode == 0x825;
+ if (active_link && !no_destination &&
+ (insn.Bits(16, 8) == *active_link || (pair && insn.Bits(16, 8) + 1 == *active_link)))
return fail(index, "native callee overwrites link register");
}
// Scratch values and emulated uniform state occupy reserved high GPRs.
// Reject native accesses that could alias that state.
for (u32 bit : {16U, 24U, 32U, 64U}) {
- if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66) continue;
+ if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66 && opcode != 0x361 &&
+ opcode != 0xf60) continue;
const u32 reg = insn.Bits(bit, 8);
if (reg >= (uses_uniform_bank ? 216U : 246U) && reg < 255) return fail(index, "native program uses lowering scratch registers");
}
@@ -163,6 +169,13 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
};
std::string_view operand_error;
const auto emit = [&](u64 word) { result.instructions.push_back(word); };
+ // Bound Maxwell texture instructions share one descriptor constant buffer per program.
+ const auto bind_texture_buffer = [&](u32 binding) {
+ if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
+ return false;
+ result.required_texture_buffer = binding;
+ return true;
+ };
// P(guard slot) while a uniform guard is swapped in; P6 for lowerings that use it as
// scratch (R2UR, MATCH, HFMA2.RELU), which save and restore it with the defaults below.
const u32 guard_mask = guard_slot < 7 ? 1U << guard_slot : 0U;
@@ -466,6 +479,20 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.Bits(64, 8)} << 20));
break;
}
+ if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 && (insn.words[2] & ~0x07800affU) == 0x000e0400 &&
+ (!rewritten_uniform || insn.Bits(87, 3) == 7)) {
+ // IMAD.X RZ, RZ, c, Pn: the high word of a 64-bit add (bit 75 inverts c).
+ u32 c = insn.Bits(64, 8);
+ mov_imm(250, 1);
+ emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ if (insn.Bits(75, 1)) {
+ emit(0x0400000000000000ULL | predicate | 251 | (u64{c} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53));
+ c = 251;
+ }
+ emit(base(0x5c10000000000000ULL) | (u64{c} << 8) | (u64{250} << 20));
+ break;
+ }
if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (insn.words[1] & ~0xffU))
return fail(index, "native integer register multiply mode");
multiply_low(insn.Bits(24, 8), insn.Bits(32, 8), insn.Bits(64, 8), false);
@@ -496,16 +523,87 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
emit(0x5c10000000070000ULL | d_low | (u64{250} << 8) | (u64{255} << 20));
break;
}
- case 0x827: { // IMAD.HI.U32, captured zero addend
- if (insn.words[2] != 0x078e00ff) return fail(index, "native high multiply mode");
- const u32 a = insn.Bits(24, 8), b = insn.words[1];
+ case 0x226: { // IDP.2A.LO.U16.U8: a.lo16 * b.byte0 + a.hi16 * b.byte1 + c
+ if ((insn.words[2] & ~0xffU) != 0x1000 || (insn.words[1] & ~0xffU))
+ return fail(index, "native packed dot-product mode");
+ const u32 a = insn.Bits(24, 8), b = insn.Bits(32, 8), c = insn.Bits(64, 8);
+ for (u32 component = 0; component < 2; ++component) {
+ emit(0x3828000000000000ULL | predicate | (250 + component) | (u64{b} << 8) | (u64{component * 8} << 20));
+ emit(0x0400000000000000ULL | predicate | (250 + component) | (u64{250 + component} << 8) | (u64{0xff} << 20));
+ }
+ xmad(252, a, 250, c, false, false, 0, false);
+ xmad(static_cast<u32>(dest), a, 251, 252, true, false, 0, false);
+ break;
+ }
+ case 0x225: case 0x227: case 0x625: case 0x825: case 0x827: case 0xa27: {
+ // IMAD.HI Rd / IMAD.WIDE Rd:Rd+1 = a * b + c:c+1 (.U32 or signed, bit 73), with an optional
+ // carry-out (bits 81-83). b is a register, an immediate (0x8xx) or a cbuf (0xa27); 0x625 takes
+ // b from bits 64-71 and adds a cbuf pair.
+ const bool wide = opcode == 0x225 || opcode == 0x625 || opcode == 0x825;
+ const bool immediate = opcode == 0x825 || opcode == 0x827, signed_product = insn.Bits(73, 1);
+ const u32 c = opcode == 0x625 ? 255U : insn.Bits(64, 8), carry_output = insn.Bits(81, 3);
+ if ((insn.words[2] & ~0x000e02ffU) != 0x07800000 ||
+ ((opcode == 0x225 || opcode == 0x227) && (insn.words[1] & ~0xffU)) ||
+ ((opcode == 0xa27 || opcode == 0x625) && (!valid_cbuf() || insn.Bits(62, 2))) ||
+ (opcode == 0x625 && insn.Bits(38, 16) > 0xfff8) || (c != 255 && (c & 1)) ||
+ (wide && (dest == 255 || (dest & 1))) || (signed_product && carry_output != 7))
+ return fail(index, "native high multiply mode");
+ const u32 a = insn.Bits(24, 8);
+ const u32 b = immediate ? insn.words[1] : opcode == 0xa27 ? 254U : insn.Bits(opcode == 0x625 ? 64 : 32, 8);
+ if (opcode == 0xa27) emit(0x4c98000000000000ULL | predicate | 254 | (u64{15} << 39) | cbuf());
+ if (signed_product) {
+ // The signed high word is the unsigned one minus (a < 0 ? b : 0) + (b < 0 ? a : 0),
+ // which R253 holds (R252 is free until the multiplication).
+ const u32 b_register = immediate ? 254U : b;
+ if (immediate) mov_imm(254, b);
+ emit(0x3828000000000000ULL | predicate | 253 | (u64{a} << 8) | (u64{31} << 20) | (u64{1} << 48));
+ emit(0x5c40000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{b_register} << 20) | (u64{7} << 48));
+ emit(0x3828000000000000ULL | predicate | 252 | (u64{b_register} << 8) | (u64{31} << 20) | (u64{1} << 48));
+ emit(0x5c40000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{a} << 20) | (u64{7} << 48));
+ emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{252} << 20));
+ }
// Base-2^16 long multiplication. Each intermediate sum fits U32.
- xmad(250, a, b, 255, false, false, 0, true);
- xmad(251, a, b, 250, true, false, 2, true);
- xmad(250, a, b, 251, false, true, 1, true);
- xmad(252, a, b, 251, true, true, 2, true);
+ xmad(250, a, b, 255, false, false, 0, immediate);
+ xmad(251, a, b, 250, true, false, 2, immediate);
+ xmad(250, a, b, 251, false, true, 1, immediate);
+ xmad(252, a, b, 251, true, true, 2, immediate);
emit(0x3828000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{16} << 20));
- emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20));
+ if (signed_product)
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{253} << 20) | (u64{1} << 48));
+ if (!wide && c == 255 && carry_output == 7) {
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20));
+ break;
+ }
+ // High word in R253 and low word in R250 (as UIMAD.WIDE.U32), then c:c+1 is added.
+ emit(0x5c10000000000000ULL | predicate | 253 | (u64{250} << 8) | (u64{252} << 20));
+ xmad(250, a, b, 255, false, false, 0, immediate);
+ xmad(251, a, b, 255, true, false, 0, immediate);
+ xmad(251, a, b, 251, false, true, 0, immediate);
+ emit(0x5c18000000000000ULL | predicate | 250 | (u64{251} << 8) | (u64{250} << 20) | (u64{16} << 39));
+ u32 c_low = c, c_high = c == 255 ? 255U : c + 1;
+ if (opcode == 0x625) {
+ emit(0x4c98000000000000ULL | predicate | 252 | (u64{15} << 39) | cbuf());
+ emit(0x4c98000000000000ULL | predicate | 254 | (u64{15} << 39) | (cbuf() + (u64{1} << 20)));
+ c_low = 252;
+ c_high = 254;
+ }
+ const u32 high = wide ? static_cast<u32>(dest) + 1 : static_cast<u32>(dest);
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{c_low} << 20) | (u64{1} << 47));
+ if (carry_output == 7) {
+ emit(0x5c10000000000000ULL | predicate | high | (u64{253} << 8) | (u64{c_high} << 20) | (u64{1} << 43));
+ if (wide) emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{255} << 20));
+ break;
+ }
+ // Maxwell IADD has no .X with .CC: R251 takes the low carry, R254 counts the high word's.
+ emit(0x5c10000000000000ULL | predicate | 251 | (u64{255} << 8) | (u64{255} << 20) | (u64{1} << 43));
+ emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{c_high} << 20) | (u64{1} << 47));
+ emit(0x5c10000000000000ULL | predicate | 254 | (u64{255} << 8) | (u64{255} << 20) | (u64{1} << 43));
+ emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{251} << 20) | (u64{1} << 47));
+ emit(0x5c10000000000000ULL | predicate | 254 | (u64{254} << 8) | (u64{255} << 20) | (u64{1} << 43));
+ emit(0x5c98000000000000ULL | predicate | high | (u64{253} << 20) | (u64{15} << 39));
+ if (wide) emit(base(0x5c98000000000000ULL) | (u64{250} << 20) | (u64{15} << 39));
+ emit(0x5b60000000000000ULL | predicate | 7 | (u64{carry_output} << 3) | (u64{254} << 8) |
+ (u64{255} << 20) | (u64{7} << 39) | (u64{5} << 49));
break;
}
case 0xa24: { // IMAD, constant-buffer multiplier
@@ -547,6 +645,21 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
}
case 0x824: { // Captured low-word IMAD forms; bit 75 negates the addend.
const bool negate_c = insn.Bits(75, 1) != 0;
+ if (insn.words[1] == 1 && (insn.words[2] & ~(0x07800800U | 0xffU)) == 0x000e0600U) {
+ // IMAD.X Rd, Ra, 1, Rc, Pn: Ra + Rc plus the carry predicate in bits 87-90 (~Rc with bit 75).
+ u32 c = insn.Bits(64, 8);
+ mov_imm(252, 1);
+ emit(0x5ca0000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{255} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ if (negate_c) {
+ emit(0x0400000000000000ULL | predicate | 251 | (u64{c} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53));
+ c = 251;
+ }
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{c} << 20));
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20));
+ break;
+ }
if (insn.words[1] == 1 && (insn.words[2] & 0xfffff700U) == 0x078e0200) {
emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
(u64{insn.Bits(64, 8)} << 20) | (u64{negate_c} << 48));
@@ -708,27 +821,27 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
}
case 0x899: { // USHF.L.U32; accept a zero word outside the selected half.
const u32 dst = insn.Bits(16, 8), low = insn.Bits(24, 8), high = insn.Bits(64, 8);
- if (insn.words[2] == (0x08011400U | high) && low == 63 && high < 16 &&
- dst < 16 && native_predicate == 7 && insn.words[1] <= 32) {
- emit(0x3828000000000000ULL | predicate | (dst + 216) |
- (u64{high + 216} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48));
+ if (insn.words[2] == (0x08011400U | high) && low == 63 && high < 28 &&
+ dst < 28 && native_predicate == 7 && insn.words[1] <= 32) {
+ emit(0x3828000000000000ULL | predicate | uniform_register(dst) |
+ (u64{uniform_register(high)} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48));
break;
}
// Logical right shifts of one register (bit 76 right, bit 80 high word, bits 73-74
// U32): USHF.R.U32 URd, URa, n, URZ and USHF.R.U32.HI URd, URZ, n, URb.
- const bool right_low = insn.words[2] == 0x0800163f && low < 16;
- const bool right_high = insn.words[2] == (0x08011600U | high) && low == 63 && high < 16;
- if ((right_low || right_high) && dst < 16 && native_predicate == 7 && insn.words[1] <= 32) {
- emit(0x3828000000000000ULL | predicate | (dst + 216) |
- (u64{(right_high ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20));
+ const bool right_low = insn.words[2] == 0x0800163f && low < 28;
+ const bool right_high = insn.words[2] == (0x08011600U | high) && low == 63 && high < 28;
+ if ((right_low || right_high) && dst < 28 && native_predicate == 7 && insn.words[1] <= 32) {
+ emit(0x3828000000000000ULL | predicate | uniform_register(dst) |
+ (u64{uniform_register(right_high ? high : low)} << 8) | (u64{insn.words[1]} << 20));
break;
}
- const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high < 16;
- const bool low_result = insn.words[2] == 0x0800063f && low < 16;
- if (dst >= 16 || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result))
+ const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high < 28;
+ const bool low_result = insn.words[2] == 0x0800063f && low < 28;
+ if (dst >= 28 || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result))
return fail(index, "native uniform shift mode");
- emit(0x3848000000000000ULL | predicate | (dst + 216) |
- (u64{(high_result ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20));
+ emit(0x3848000000000000ULL | predicate | uniform_register(dst) |
+ (u64{uniform_register(high_result ? high : low)} << 8) | (u64{insn.words[1]} << 20));
break;
}
case 0xb82: { // LDC.64 indexed byte offset
@@ -759,16 +872,18 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
(u64{insn.Bits(64, 8)} << 20) | (u64{maxwell} << 28));
break;
}
- case 0xb6c: { // TXD 2D or ARRAY_2D, four channels; the two destination pairs anywhere.
+ case 0xb6c: { // TXD 2D or ARRAY_2D, any channels; the two destination pairs anywhere.
const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14);
const u32 coordinates = insn.Bits(24, 8), derivatives = insn.Bits(32, 8);
// Ampere dimensions: 1 = 2D, 5 = ARRAY_2D; Maxwell TXD's are 2 and 3. The coordinates
// (x, y[, layer]) and the four gradients sit in the same registers for both.
const u32 dimension = insn.Bits(61, 3);
const u32 high = insn.Bits(64, 8); // destination of the third and fourth channels
- if ((insn.words[2] & ~0xffU) != 0x001e0f00 || insn.Bits(59, 2) ||
+ const u32 mask = insn.Bits(72, 4), channels = static_cast<u32>(std::popcount(mask));
+ if ((insn.words[2] & ~(0xfffU | (1U << 26))) != 0x001e0000 || mask == 0 || insn.Bits(59, 2) ||
(dimension != 1 && dimension != 5) || binding >= 18 || offset > 0x1fff ||
- (dest & 1) || dest >= 248 || (high & 1) || (high >= 248 && high != 255) ||
+ (channels > 1 && (dest & 1)) || dest >= 248 ||
+ (channels > 2 ? (high & 1) || high >= 248 : high != 255) ||
coordinates >= 248 || derivatives >= 247)
return fail(index, "native gradient texture mode");
if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
@@ -776,16 +891,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
result.required_texture_buffer = binding;
// Maxwell writes four consecutive registers: when the second pair is elsewhere (Splatoon
// Raiders: R12/R18, R18/R44), write them to scratch and move each pair into place.
- const bool consecutive = high == dest + 2;
+ const bool consecutive = channels <= 2 || high == dest + 2;
const u64 target = consecutive ? dest : 248;
emit(0xde00000000000000ULL | predicate | target | (u64{coordinates} << 8) |
- (u64{derivatives} << 20) | (u64{dimension == 5 ? 3U : 2U} << 28) | (u64{15} << 31) |
+ (u64{derivatives} << 20) | (u64{dimension == 5 ? 3U : 2U} << 28) | (u64{mask} << 31) |
(u64{offset} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51));
if (!consecutive) {
- for (u32 i = 0; i < 2; ++i) {
- emit(0x5c98000000000000ULL | predicate | (dest + i) | (u64{15} << 39) | (u64{248 + i} << 20));
- if (high != 255)
- emit(0x5c98000000000000ULL | predicate | (high + i) | (u64{15} << 39) | (u64{250 + i} << 20));
+ for (u32 i = 0; i < channels; ++i) {
+ emit(0x5c98000000000000ULL | predicate | (i < 2 ? dest + i : high + i - 2) |
+ (u64{15} << 39) | (u64{248 + i} << 20));
}
}
break;
@@ -836,44 +950,53 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
emit(0x3828000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 8) |
(u64{1} << 39) | (u64{1} << 40));
break;
- case 0x3a1: { // MATCH.ALL.U64 with only the all-equal predicate retained.
+ case 0x3a1: { // MATCH.ALL.U32/U64 (bit 73) with only the all-equal predicate retained.
const u32 source = insn.Bits(24, 8), target = insn.Bits(81, 3);
- if (dest != 255 || source >= 254 || (source & 1) || target >= 6 ||
- native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0000U) != 0x200)
+ const bool wide = insn.Bits(73, 1);
+ if (dest != 255 || source >= 254 || (wide && (source & 1)) || target >= 6 ||
+ native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0200U))
return fail(index, "native warp match mode");
save_predicate();
emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48));
emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49));
emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48));
emit(0x5c30000000070000ULL | 250 | (u64{250} << 20));
- for (u32 word = 0; word < 2; ++word)
+ for (u32 word = 0; word < (wide ? 2U : 1U); ++word)
emit(0xef10000000070000ULL | (251 + word) | (u64{source + word} << 8) |
(u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source} << 8) |
(u64{251} << 20) | (u64{7} << 39) | (u64{2} << 49));
- emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source + 1} << 8) |
- (u64{252} << 20) | (u64{6} << 39) | (u64{2} << 49));
+ if (wide)
+ emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source + 1} << 8) |
+ (u64{252} << 20) | (u64{6} << 39) | (u64{2} << 49));
emit(0x50d8000000070000ULL | 255 | (u64{6} << 39) | (u64{target} << 45));
restore_predicate();
break;
}
- case 0x3c4: { // REDUX.SUM (modulo 2^32) or .MAX (unsigned) over the actual active lanes.
+ case 0x3c4: { // REDUX.SUM (modulo 2^32), .OR, .MIN or .MAX (unsigned) over the actual active lanes.
const u32 source = insn.Bits(24, 8), target = uniform_register(static_cast<u32>(dest));
- const bool maximum = insn.words[2] == 0x14000;
+ const bool maximum = insn.words[2] == 0x14000, minimum = insn.words[2] == 0x10000;
+ const bool bitwise_or = insn.words[2] == 0x4000;
if (target == ~0U || target == 255 || insn.words[1] ||
- (insn.words[2] != 0xc000 && !maximum) || native_predicate != 7)
+ (insn.words[2] != 0xc000 && !maximum && !minimum && !bitwise_or) || native_predicate != 7)
return fail(index, "native warp sum mode");
emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48));
- mov_imm(target, 0);
+ mov_imm(target, minimum ? ~0U : 0U);
for (u32 lane = 0; lane < 32; ++lane) {
emit(0xef10000000070000ULL | 251 | (u64{source} << 8) | (u64{lane} << 20) |
(u64{1} << 28) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48));
emit(0x3828000000070000ULL | 252 | (u64{250} << 8) | (u64{lane} << 20));
emit(0x0400000000070000ULL | 252 | (u64{252} << 8) | (u64{1} << 20));
emit(0x5c10000000070000ULL | 252 | (u64{255} << 8) | (u64{252} << 20) | (u64{1} << 48));
- emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48));
- if (maximum) {
- emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | (u64{15} << 39));
+ // Inactive lanes give the identity: zero, or all ones for MIN (OR with ~mask).
+ emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) |
+ (minimum ? (u64{1} << 40) | (u64{1} << 41) : 0) | (u64{7} << 48));
+ if (maximum || minimum) {
+ emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) |
+ (u64{maximum ? 15U : 7U} << 39));
+ } else if (bitwise_or) {
+ emit(0x5c40000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) |
+ (u64{1} << 41) | (u64{7} << 48));
} else {
emit(0x5c10000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20));
}
@@ -894,12 +1017,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
emit(0xec00000000000000ULL | predicate | 255 | (u64{address} << 8) | (u64{250} << 20));
break;
}
- case 0x38c: { // ATOMS.ADD.U32, naturally aligned shared byte offset.
- const u32 offset = insn.Bits(40, 24);
- if (insn.words[2] || (offset & 3)) return fail(index, "native shared atomic mode");
+ case 0x38c: { // ATOMS.ADD/MIN/MAX/OR.U32, naturally aligned shared byte offset.
+ // The operation (bits 87-90) numbers ADD 0, MIN 1, MAX 2, OR 6 as Maxwell's (bits 52-55).
+ const u32 offset = insn.Bits(40, 24), operation = insn.Bits(87, 4);
+ if ((operation != 0 && operation != 1 && operation != 2 && operation != 6) ||
+ insn.words[2] != (operation << 23) || (offset & 3))
+ return fail(index, "native shared atomic mode");
emit(0xec00000000000000ULL | predicate | dest |
(u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) |
- (u64{offset >> 2} << 30));
+ (u64{offset >> 2} << 30) | (u64{operation} << 52));
break;
}
case 0x3a8: { // ATOMG.E.ADD.STRONG.GPU U32 at a 64-bit register address plus an offset.
@@ -944,14 +1070,20 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14);
const u32 data = insn.Bits(32, 8), coordinate = insn.Bits(24, 8);
const u32 dimension = insn.Bits(61, 3);
- if (insn.words[2] != 0x00100f00 || dimension < 3 || dimension > 5 ||
+ const bool strong = insn.words[2] == 0x0010af00; // .STRONG.GPU: fenced on both sides
+ if ((insn.words[2] != 0x00100f00 && !strong) || dimension < 3 || dimension > 5 ||
binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || data > 252 || coordinate > (dimension >= 4 ? 252U : 253U))
return fail(index, "native formatted surface store mode");
- if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
- return fail(index, "native surface descriptor buffer conflict");
- result.required_texture_buffer = binding;
+ // A second descriptor buffer: SUST.B with the handle, loaded from that buffer, in R250.
+ const bool bindless = !bind_texture_buffer(binding);
+ if (bindless) {
+ emit(0x4c98000000070000ULL | 250 | (u64{15} << 39) | (u64{offset} << 20) | (u64{binding} << 34));
+ }
+ if (strong) emit(0xef98000000000000ULL | predicate | (u64{1} << 8)); // MEMBAR.GL
emit(0xeb20000000000000ULL | predicate | data | (u64{coordinate} << 8) |
- (u64{15} << 20) | (u64{dimension} << 33) | (u64{offset} << 36) | (u64{1} << 51));
+ (u64{15} << 20) | (u64{dimension} << 33) |
+ (bindless ? u64{250} << 39 : (u64{offset} << 36) | (u64{1} << 51)));
+ if (strong) emit(0xef98000000000000ULL | predicate | (u64{1} << 8));
break;
}
case 0x301: // BREV register.
@@ -959,21 +1091,27 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
emit(base(0x3828000000000000ULL) | (u64{insn.words[1]} << 8) |
(u64{1} << 39) | (u64{1} << 40));
break;
- case 0x219: case 0xc19: { // U32 register (or uniform, 0xc19) shifts with a zero word outside the selected result.
+ case 0x219: case 0xa19: case 0xc19: { // U32 register (or cbuf, 0xa19, or uniform, 0xc19) shifts with a zero word outside the selected result.
const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8);
- const u32 shift = opcode == 0xc19 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8);
+ const bool constant = opcode == 0xa19;
+ // A constant shift amount is loaded into R251 once the form is known.
+ const u32 shift = opcode == 0xc19 ? uniform_register(insn.Bits(32, 8)) : constant ? 251U : insn.Bits(32, 8);
+ const bool shift_operand = constant ? valid_cbuf() && !insn.Bits(62, 2) : insn.words[1] <= 255;
+ const auto load_shift = [&] { if (constant) materialize(2, 0, 251, false, false); };
const bool right = insn.Bits(76, 1) != 0, high_result = insn.Bits(80, 1) != 0;
// SHF.R.U32 of the low word with a zero high word: a plain logical right shift.
if (right && !high_result && high == 255 && insn.Bits(73, 2) == 3 && shift != ~0U &&
- insn.words[1] <= 255 &&
+ shift_operand &&
(insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) == 0x00000600U) {
+ load_shift();
emit(base(0x5c28000000000000ULL) | (u64{low} << 8) | (u64{shift} << 20));
break;
}
- if (insn.words[1] > 255 || insn.Bits(73, 2) != 3 || shift == ~0U ||
+ if (!shift_operand || insn.Bits(73, 2) != 3 || shift == ~0U ||
(insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) != 0x00000600U ||
(right ? (!high_result || low != 255) : (high_result || high != 255)))
return fail(index, "native register shift mode");
+ load_shift();
emit(base(right ? 0x5cf8000000000000ULL : 0x5bf8000000000000ULL) |
(u64{right ? high : 255U} << 8) | (u64{shift} << 20) |
(u64{right ? 255U : low} << 39) | (u64{insn.Bits(75, 1)} << 50));
@@ -1012,8 +1150,8 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
if (insn.words[1] || insn.words[2]) return fail(index, "native error barrier mode");
emit(0xef98000000000000ULL | predicate | (u64{1} << 8));
break;
- case 0xb1d: // BAR.SYNC 0, full workgroup.
- if (insn.words[1] || insn.words[2] || native_predicate != 7)
+ case 0xb1d: // BAR.SYNC 0 (bit 80: .DEFER_BLOCKING), full workgroup.
+ if (insn.words[1] || (insn.words[2] & ~0x10000U) || native_predicate != 7)
return fail(index, "native workgroup barrier mode");
emit(0xf0a8000000000000ULL | predicate | (u64{1} << 43) |
(u64{1} << 44) | (u64{1} << 39) | (u64{7} << 39));
@@ -1257,10 +1395,17 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
// The uniform base sits where a register source would (bits 32-37).
const u32 uniform_address =
insn.Bits(38, 2) ? ~0U : uniform_register(insn.Bits(32, 6));
- if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U | 0x2000U)) != 0x081e8100 ||
+ // desc[URn][Ra.64]: the uniform pair is a memory (cache policy) descriptor and
+ // Ra the whole address.
+ const bool descriptor_address = (insn.words[2] & ~(0x3fU | 0xe00U)) == 0x0c1e1100;
+ if ((!descriptor_address &&
+ (insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U | 0x2000U)) != 0x081e8100) ||
uniform_address == ~0U || (uniform_address & 1))
return fail(index, "native global load uniform addressing mode");
- if (address == 255) {
+ if (descriptor_address) {
+ if ((address & 1) || address >= 246)
+ return fail(index, "native global load register address pair");
+ } else if (address == 255) {
address = uniform_address;
} else {
if (insn.Bits(90, 1) && ((address & 1) || address >= 246))
@@ -1293,10 +1438,16 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
bool extended = insn.Bits(72, 1) != 0;
if (opcode == 0x986) {
const u32 uniform_address = uniform_register(insn.Bits(64, 6));
- if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U)) != 0x0810a100 ||
+ // desc[URn][Ra.64]: the uniform pair is a memory (cache policy) descriptor and
+ // Ra the whole address.
+ const bool descriptor_address = (insn.words[2] & ~(0x3fU | 0xe00U)) == 0x0c101100;
+ if ((!descriptor_address && (insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U)) != 0x0810a100) ||
uniform_address == ~0U || (uniform_address & 1) || !extended)
return fail(index, "native global store uniform addressing mode");
- if (address == 255) address = uniform_address;
+ if (descriptor_address) {
+ if ((address & 1) || address >= 246)
+ return fail(index, "native global store register address pair");
+ } else if (address == 255) address = uniform_address;
else {
if (insn.Bits(90, 1) && ((address & 1) || address >= 254))
return fail(index, "native global store register address pair");
@@ -1319,13 +1470,29 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
return fail(index, "native fragment discard mode");
emit(0xe33000000000000fULL | predicate);
break;
+ case 0x311: { // F2I.FTZ.U64.TRUNC, register source, destination pair
+ if (insn.words[2] != 0x0021d800 || (insn.words[1] & ~0xffU) || insn.Bits(24, 8) ||
+ (dest & 1) || dest >= 246)
+ return fail(index, "native unsigned 64-bit float conversion mode");
+ emit(base(0x5cb0000000000000ULL) | (u64{3} << 8) | (u64{2} << 10) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{3} << 39) | (u64{1} << 44));
+ break;
+ }
+ case 0x312: { // I2F.F32.(U|S)64, register pair source; sign (74) and rounding (78-79) as Maxwell's
+ if ((insn.words[2] & ~0xc400U) != 0x00301000 || (insn.words[1] & ~0xffU) || insn.Bits(24, 8) ||
+ (insn.Bits(32, 8) & 1) || insn.Bits(32, 8) >= 254)
+ return fail(index, "native 64-bit integer-to-float mode");
+ emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{3} << 10) | (u64{insn.Bits(74, 1)} << 13) |
+ (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(78, 2)} << 39));
+ break;
+ }
case 0x305: case 0xb05: case 0xd05: { // F2I NTZ, register/constant-buffer/uniform source
const u32 mode = insn.words[2] & ~(opcode == 0xd05 ? 0x08000000U : 0U);
const bool half = opcode == 0x305 && mode == 0x0010f000;
const bool small = mode == 0x00212800;
const bool floor = (mode & ~0x10100U) == 0x00207000;
const bool nearest = (mode & ~0x10100U) == 0x00203000;
- if ((!half && !small && !floor && !nearest && (mode & ~0x100U) != 0x0021f000) ||
+ if ((!half && !small && !floor && !nearest && (mode & ~0x10100U) != 0x0020f000) ||
(opcode != 0xb05 && (insn.words[1] & 0xffffff00U)) ||
(opcode == 0xd05 && uniform_register(insn.Bits(32, 8)) == ~0U)) {
return fail(index, "native float-to-integer conversion mode");
@@ -1363,17 +1530,28 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
}
break;
}
- case 0xb6f: { // TXQ texture dimensions, two packed scalar outputs.
- if (insn.Bits(59, 5) || insn.Bits(64, 8) != 255 || insn.Bits(72, 4) == 0 || insn.Bits(72, 4) > 3 ||
- insn.Bits(76, 14) || insn.Bits(91, 1) || insn.Bits(54, 5) != 2 ||
- insn.Bits(40, 14) > 0x1fff || (insn.Bits(72, 4) == 3 && (dest & 1)) || dest >= 254)
+ case 0xb6f: { // TXQ texture dimensions (mask 1-3) or mip level count (mask 8).
+ const u32 mask = insn.Bits(72, 4);
+ if (insn.Bits(59, 5) || insn.Bits(64, 8) != 255 || (mask == 0 || (mask > 3 && mask != 8)) ||
+ insn.Bits(76, 14) || insn.Bits(91, 1) || insn.Bits(54, 5) >= 18 ||
+ insn.Bits(40, 14) > 0x1fff || (mask == 3 && (dest & 1)) || dest >= 254)
return fail(index, "native texture dimension query mode");
- result.required_texture_buffer = 2;
+ if (!bind_texture_buffer(insn.Bits(54, 5)))
+ return fail(index, "native texture descriptor buffer conflict");
emit(0xdf48000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) |
- (u64{1} << 22) | (u64{insn.Bits(72, 4)} << 31) | (u64{insn.Bits(40, 14)} << 36) |
+ (u64{1} << 22) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) |
(u64{insn.Bits(90, 1)} << 49));
break;
}
+ case 0x370: { // TXQ.B dimensions or mip count: handle in Ra, mip level in Ra+1.
+ const u32 mask = insn.Bits(72, 4), source = insn.Bits(24, 8);
+ if (insn.words[1] != 0x08000000 || insn.Bits(64, 8) != 255 || (mask == 0 || (mask > 3 && mask != 8)) ||
+ (insn.words[2] & ~0x04000f00U) != 255 || source > 245 || (mask == 3 && (dest & 1)) || dest >= 254)
+ return fail(index, "native bindless texture dimension query mode");
+ emit(0xdf50000000000000ULL | predicate | dest | (u64{source} << 8) |
+ (u64{1} << 22) | (u64{mask} << 31) | (u64{insn.Bits(90, 1)} << 49));
+ break;
+ }
case 0xb66: { // TLD.SCR.LZ, direct texel fetch
const u32 mask = insn.Bits(72, 4);
const u32 dimension = insn.Bits(61, 3);
@@ -1402,9 +1580,25 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
return fail(index, "native texel fetch mode or descriptor binding");
}
const u32 binding = insn.Bits(54, 5);
- if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
- return fail(index, "native texture descriptor buffer conflict");
- result.required_texture_buffer = binding;
+ if (!bind_texture_buffer(binding)) {
+ // A second descriptor buffer: TLD.B (LZ) with the coordinates in R250.. (array
+ // index or Z after Ra) and the handle, loaded from that buffer, in R253.
+ const auto copy = [&](u32 target, u32 source) {
+ emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20));
+ };
+ copy(250, source_a);
+ if (array || volume) copy(251, source_a + 1);
+ if (dimension != 0) copy(array || volume ? 252 : 251, source_b);
+ emit(0x4c98000000070000ULL | 253 | (u64{15} << 39) | (u64{insn.Bits(40, 14)} << 20) |
+ (u64{binding} << 34));
+ emit(0xdd00000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{253} << 20) |
+ (u64{array ? 3U : volume ? 4U : dimension == 0 ? 0U : 2U} << 28) | (u64{mask} << 31) |
+ (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51));
+ const u32 components = static_cast<u32>(std::popcount(mask));
+ for (u32 i = 0; i < components; ++i)
+ copy(i < 2 || !split ? static_cast<u32>(dest) + i : second_dest + (i - 2), 250 + i);
+ break;
+ }
if (array) {
emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a + 1} << 20));
emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{source_b} << 20));
@@ -1475,7 +1669,10 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
}
break;
}
+ case 0x361: // TEX with the descriptor handle in a register (after the coordinates)
+ case 0xf60: // TEX with the descriptor handle in a uniform register
case 0xb60: { // TEX, scalar 2D implicit-LOD sample
+ const bool register_handle = opcode == 0x361, uniform_handle = opcode == 0xf60;
const u32 mask = insn.Bits(72, 4);
const u32 second_dest = insn.Bits(64, 8);
constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12};
@@ -1485,12 +1682,13 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
const bool depth_compare = insn.Bits(78, 1);
const u32 dimension = insn.Bits(61, 3), lod = insn.Bits(87, 3);
const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8);
- if (dimension == 5 && depth_compare && lod == 0 && !split && mask == 1) {
+ if (dimension == 5 && depth_compare && lod == 0 && !split && mask == 1 && !register_handle &&
+ !uniform_handle) {
if (!insn.Bits(60, 1) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 ||
insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 ||
(source_a & 1) || (source_b & 1) || source_a >= 248 || source_b >= 248 || dest >= 248)
return fail(index, "native array depth comparison sample mode");
- result.required_texture_buffer = 2;
+ if (!bind_texture_buffer(2)) return fail(index, "native texture descriptor buffer conflict");
for (u32 i = 0; i < 3; ++i) {
const u32 source = i < 2 ? source_a + i : source_b;
emit(0x5c98000000000000ULL | predicate | (250 + i) |
@@ -1510,14 +1708,24 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
constexpr std::array<u32, 8> coordinate_count{1, 2, 3, 3, 2, 3, 0, 4};
constexpr std::array<u32, 8> maxwell_type{0, 2, 4, 6, 1, 3, 0, 7};
const bool offsets = insn.Bits(76, 1) != 0;
- if (dimension == 6 || insn.Bits(77, 1) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) ||
- insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 || (lod > 3 && lod != 6 && lod != 7) ||
- (depth_compare && mask != 1) || mask == 0)
+ // Handle forms: 0x361 has no descriptor fields (bit 59 set), 0xf60 names a UR
+ // in bits 40-45 (bit 91 set) and is only known with packed operands.
+ const u32 handle_register = uniform_handle ? uniform_register(insn.Bits(40, 6)) : ~0U;
+ if (register_handle ? !insn.Bits(59, 1) || insn.Bits(40, 19)
+ : uniform_handle ? !insn.Bits(60, 1) || insn.Bits(59, 1) || insn.Bits(46, 13) ||
+ !insn.Bits(91, 1) || handle_register == ~0U || handle_register == 255
+ : insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18)
+ return false;
+ if (dimension == 6 || insn.Bits(77, 1) || insn.Bits(81, 3) != 7 ||
+ (lod > 3 && lod != 6 && lod != 7) || (depth_compare && mask != 1) || mask == 0)
return false;
const u32 coordinates = coordinate_count[dimension];
const bool has_lod = lod == 2 || lod == 3 || lod == 6 || lod == 7;
- const u32 count = coordinates + (has_lod ? 1 : 0) + (offsets ? 1 : 0) + (depth_compare ? 1 : 0);
- const u32 in_a = count <= 4 ? (count + 1) / 2 : 4;
+ const u32 count = coordinates + (register_handle ? 1 : 0) + (has_lod ? 1 : 0) +
+ (offsets ? 1 : 0) + (depth_compare ? 1 : 0);
+ // Unpacked 0x361 (bit 60 clear): Ra holds the coordinates, Rb the handle and the rest.
+ const u32 in_a = register_handle && !insn.Bits(60, 1) ? coordinates
+ : count <= 4 ? (count + 1) / 2 : 4;
const auto value = [&](u32 i) -> u32 {
if (i < in_a) return source_a == 255 ? 255U : source_a + i;
return source_b == 255 ? 255U : source_b + (i - in_a);
@@ -1525,25 +1733,34 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
if ((source_a != 255 && source_a + in_a > 246) ||
(count > in_a && source_b != 255 && source_b + (count - in_a) > 246))
return false;
- const u32 binding = insn.Bits(54, 5);
- if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)
- return false;
- result.required_texture_buffer = binding;
+ // Handle forms and a second descriptor buffer are read bindless: TEX.B takes the
+ // handle (the register's, or loaded from that buffer) as its first meta register.
+ const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14);
+ const bool bindless = register_handle || uniform_handle || !bind_texture_buffer(binding);
const auto copy = [&](u32 target, u32 source) {
emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20));
};
- // Coordinates to R250.., then LOD/bias, offsets and reference to R246...
+ // Coordinates to R250.., then the handle, LOD/bias, offsets and reference to R246...
for (u32 i = 0; i < coordinates; ++i) copy(250 + i, value(i));
u32 meta = 246;
u32 next = coordinates;
+ // The handle is copied unpredicated, so it is the same value on every path.
+ if (register_handle || uniform_handle) {
+ emit(0x5c98000000070000ULL | meta++ | (u64{15} << 39) |
+ (u64{register_handle ? value(next++) : handle_register} << 20));
+ } else if (bindless) {
+ emit(0x4c98000000070000ULL | meta++ | (u64{15} << 39) | (u64{offset} << 20) |
+ (u64{binding} << 34));
+ }
if (has_lod) copy(meta++, value(next++));
if (offsets) copy(meta++, value(next++));
if (depth_compare) copy(meta++, value(next++));
- emit(0xc000000000000000ULL | predicate | 250 | (u64{250} << 8) |
+ emit((bindless ? 0xde80000000000000ULL : 0xc000000000000000ULL) | predicate | 250 | (u64{250} << 8) |
(u64{meta == 246 ? 255U : 246U} << 20) | (u64{maxwell_type[dimension]} << 28) |
- (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) |
- (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) | (u64{7} << 51) |
- (u64{offsets} << 54) | (u64{lod} << 55));
+ (u64{mask} << 31) | (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) |
+ (u64{7} << 51) |
+ (bindless ? (u64{offsets} << 36) | (u64{lod} << 37)
+ : (u64{offset} << 36) | (u64{offsets} << 54) | (u64{lod} << 55)));
const u32 components = static_cast<u32>(std::popcount(mask));
if (insn.Bits(79, 1)) {
// FP16 results: (c0, c1) packed into the first destination, (c2, c3) into
@@ -1587,11 +1804,11 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) ||
(split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) ||
(depth_compare && (dimension != 1 || split || mask != 1 || (source_a & 1) || source_a >= 249)) ||
- (dimension == 5 && ((source_a & 1) || source_a >= 249))) {
+ (dimension == 5 && ((source_a & 1) || source_a >= 249)) || register_handle || uniform_handle ||
+ !bind_texture_buffer(2)) {
if (!general_tex()) return fail(index, "native texture mode or descriptor binding");
break;
}
- result.required_texture_buffer = 2;
emit((0xd800000000000000ULL & ~(u64{insn.Bits(79, 1)} << 59)) | predicate | dest | (u64{insn.Bits(24, 8)} << 8) |
(u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(64, 8)} << 28) |
(u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) |
@@ -1739,12 +1956,22 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
break;
}
case 0x209: case 0x809: case 0xa09: case 0xc09: { // FMNMX, including a uniform source
- if (insn.Bits(74, 6) || insn.Bits(81, 6) || (opcode != 0xc09 && insn.Bits(91, 1)))
+ if (insn.Bits(74, 6) || insn.Bits(82, 5) || (opcode != 0xc09 && insn.Bits(91, 1)))
return fail(index, "native floating min/max mode");
const u32 a = operand(0), b = operand(1);
+ // .NAN (bit 81): an unordered pair gives the canonical NaN. R252 = ~0 when both are numbers.
+ const bool propagate_nan = insn.Bits(81, 1);
+ if (propagate_nan)
+ emit(0x5800000000000000ULL | predicate | 252 | (u64{a} << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{7} << 48));
emit(base(0x5c60000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
(u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) |
(u64{insn.Bits(80, 1)} << 44));
+ if (propagate_nan) {
+ emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{252} << 20) | (u64{7} << 48));
+ emit(0x0400000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53));
+ emit(0x0400000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{0x7fffffffU} << 20));
+ emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{252} << 20) | (u64{1} << 41) | (u64{7} << 48));
+ }
break;
}
case 0x308: case 0xb08: { // MUFU
@@ -1862,13 +2089,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
}
case 0x207: case 0x807: case 0xa07:
case 0x208: case 0x808: case 0xa08: { // Bitwise SEL / FSEL
- const u32 a = operand(0, false), b = operand(1, false);
+ // FSEL applies the floating operand modifiers (negate/absolute) before selecting.
+ const bool floating = (opcode & 0x1ff) == 0x008;
+ const u32 a = operand(0, floating), b = operand(1, floating);
emit(base(0x5ca0000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
(u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
break;
}
- case 0x20a: case 0x80a: case 0xa0a: { // FSET.BF, combined with Pc (bits 87-90) by bits 74-75
- if (insn.Bits(64, 8) || insn.Bits(74, 2) == 3 || insn.Bits(81, 6) || insn.Bits(91, 1))
+ case 0x20a: case 0x80a: case 0xa0a: case 0xc0a: { // FSET.BF, combined with Pc (bits 87-90) by bits 74-75 (0xc0a: uniform b)
+ if (insn.Bits(64, 8) || insn.Bits(74, 2) == 3 || insn.Bits(81, 6) || (opcode != 0xc0a && insn.Bits(91, 1)))
return fail(index, "native floating comparison output mode");
const u32 a = operand(0), b = operand(1);
emit(base(0x5800000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) |
@@ -1890,7 +2119,45 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
case 0x20b: case 0x80b: case 0xa0b: // FSETP
case 0x20c: case 0x80c: case 0xa0c: case 0xc0c: { // ISETP
const bool floating = (opcode & 0x1ff) == 0xb;
- if (!floating && insn.Bits(72, 1)) return fail(index, "native extended integer comparison");
+ if (!floating && insn.Bits(72, 1)) {
+ // ISETP.EX: the high words, chained to the low words' predicate in bits 68-71.
+ const u32 comparison = insn.Bits(76, 3), low_comparison = insn.Bits(68, 4);
+ if (insn.Bits(74, 2) || insn.Bits(84, 3) != 7 || insn.Bits(87, 4) != 7)
+ return fail(index, "native extended integer comparison");
+ if (comparison == 1 || comparison == 3 || comparison == 4 || comparison == 6) {
+ // Unsigned LT/LE/GT/GE: (a < b, or > for GT/GE) || (a == b && low predicate),
+ // built from 0/~0 ISET masks, then one ISETP.NE into the output predicate.
+ const u32 a = insn.Bits(24, 8), b = insn.Bits(32, 8);
+ if (opcode != 0x20c || insn.Bits(73, 1) || insn.Bits(64, 4) || insn.Bits(80, 1) ||
+ (insn.words[1] & ~0xffU))
+ return fail(index, "native extended ordered comparison mode");
+ const auto set_mask = [&](u32 target, u32 operation, u32 lhs, u32 rhs) {
+ emit(0x5b50000000000000ULL | predicate | target | (u64{lhs} << 8) |
+ (u64{rhs} << 20) | (u64{7} << 39) | (u64{operation} << 49));
+ };
+ if ((low_comparison & 7) == 7) {
+ mov_imm(252, (low_comparison & 8) ? 0U : ~0U);
+ } else {
+ emit(0x38e8000000000000ULL | predicate | 252 | (u64{255} << 8) |
+ (u64{1} << (20 + (low_comparison & 7))));
+ set_mask(252, (low_comparison & 8) ? 2U : 5U, 252, 255);
+ }
+ set_mask(250, comparison == 1 || comparison == 3 ? 1U : 4U, a, b);
+ set_mask(251, 2, a, b);
+ emit(0x5c40000000000000ULL | predicate | 251 | (u64{251} << 8) | (u64{252} << 20) |
+ (u64{7} << 48));
+ emit(0x5c40000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{251} << 20) |
+ (u64{1} << 41) | (u64{7} << 48));
+ emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) |
+ (u64{250} << 8) | (u64{255} << 20) | (u64{7} << 39) | (u64{5} << 49));
+ break;
+ }
+ // EQ/NE: the high comparison ANDed (EQ) or ORed (NE) with that predicate.
+ if (comparison != 2 && comparison != 5)
+ return fail(index, "native extended integer comparison");
+ insn.words[2] = (insn.words[2] & ~(0x100U | 0xc00U | 0x07800000U)) |
+ (insn.Bits(68, 4) << 23) | (comparison == 5 ? 0x400U : 0U);
+ }
const u32 a = operand(0, floating), b = operand(1, floating);
const u32 combine = insn.Bits(74, 2);
if (combine > 2) return fail(index, "native predicate combination");
@@ -1938,15 +2205,19 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
}
case 0x217: case 0x817: case 0xa17: case 0xc17: { // IMNMX register, immediate, cbuf or uniform operand.
if ((insn.words[2] & ~(0x04000200U | (opcode == 0xc17 ? 0x08000000U : 0U))) != 0x03800000 ||
- (opcode != 0xa17 && insn.words[1] > (opcode == 0x817 ? 0x7ffffU : 255U)) ||
+ (opcode == 0x817 && insn.words[1] > 0x7ffffU && insn.words[1] < 0xfff80000U) ||
+ (opcode != 0xa17 && opcode != 0x817 && insn.words[1] > 255U) ||
(opcode == 0xa17 && (!valid_cbuf() || insn.Bits(62, 2))))
return fail(index, "native integer min/max mode");
+ // An immediate is a signed 20-bit value: 19 bits at 20, the sign in bit 56.
const u32 source = opcode == 0xc17 ? uniform_register(insn.words[1])
: opcode == 0xa17 ? operand(1, false)
+ : opcode == 0x817 ? insn.words[1] & 0x7ffffU
: insn.words[1];
if (source == ~0U) return fail(index, "native uniform min/max operand");
emit(base(opcode == 0x817 ? 0x3820000000000000ULL : 0x5c20000000000000ULL) |
(u64{insn.Bits(24, 8)} << 8) | (u64{source} << 20) |
+ (u64{opcode == 0x817 ? insn.words[1] >> 31 : 0U} << 56) |
(u64{insn.Bits(87, 4)} << 39) | (u64{insn.Bits(73, 1)} << 48));
break;
}
@@ -1976,6 +2247,20 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
break;
}
case 0xa11: { // LEA with constant base and carry predicate.
+ if ((insn.words[2] & ~(0x07800000U | 0x0000f800U)) == 0x000f06ffU && !insn.Bits(62, 2) &&
+ insn.Bits(75, 5) != 0) {
+ // LEA.HI.X.SX32, no high source: (sign-extended Ra << shift) >> 32, plus the
+ // constant high word and the carry predicate (bits 87-90).
+ const u32 b = operand(1, false);
+ emit(0x3828000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) |
+ (u64{32 - insn.Bits(75, 5)} << 20) | (u64{1} << 48));
+ mov_imm(252, 1);
+ emit(0x5ca0000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{255} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{b} << 20));
+ emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20));
+ break;
+ }
if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) ||
insn.Bits(81, 3) == 7 || insn.Bits(62, 2))
return fail(index, "native constant LEA carry mode");
@@ -1986,35 +2271,65 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
(dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49));
break;
}
- case 0x211: case 0x811: { // LEA low word, no carry result
+ case 0x211: case 0x811: case 0xc11: { // LEA low word (0xc11: uniform Rb)
if (insn.Bits(80, 1)) {
- // LEA.HI Rd, Ra, Rb (or an immediate, 0x811), RZ, s: Rb + high word of (Ra << s);
- // bit 73 sign-extends Ra.
- if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(81, 3) != 7 ||
- insn.Bits(64, 8) != 255 || (opcode == 0x211 && insn.Bits(62, 2)))
+ // LEA.HI Rd, Ra, Rb (or an immediate, 0x811), Rc, s: Rb + high word of (Rc:Ra << s);
+ // bit 73 sign-extends Ra (Rc is RZ) and .X (bit 74) adds the carry predicate in bits 87-90.
+ const bool extended = insn.Bits(74, 1) != 0;
+ const u32 high = insn.Bits(64, 8);
+ if (insn.Bits(72, 1) || insn.Bits(81, 3) != 7 ||
+ (high != 255 && insn.Bits(73, 1)) || (opcode == 0x211 && insn.Bits(62, 2)) ||
+ (opcode == 0xc11 && (insn.words[1] & ~0x3fU)))
return fail(index, "native high effective address mode");
- u32 b = insn.Bits(32, 8);
+ u32 b = opcode == 0xc11 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8);
+ if (b == ~0U) return fail(index, "native uniform LEA register");
if (opcode == 0x811) {
b = insn.words[1] ? 251U : 255U;
if (b == 251) mov_imm(251, insn.words[1]);
}
const u64 shift = (u64{32 - insn.Bits(75, 5)} << 20) | (u64{insn.Bits(73, 1)} << 48);
- if (b == 255) {
- emit(base(0x3828000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | shift);
+ // The high word is a right shift of Ra, or with Rc a left funnel shift (SHF.L).
+ const auto high_word = [&](u64 target) {
+ if (high == 255) emit(target | 0x3828000000000000ULL | (u64{insn.Bits(24, 8)} << 8) | shift);
+ else emit(target | 0x36f8000000000000ULL | (u64{insn.Bits(24, 8)} << 8) | (u64{high} << 39) |
+ (u64{insn.Bits(75, 5)} << 20));
+ };
+ if (b == 255 && !extended) {
+ high_word(base(0));
} else {
- emit(0x3828000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | shift);
+ high_word(predicate | 250);
+ if (extended) {
+ mov_imm(252, 1);
+ emit(0x5ca0000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{255} << 20) |
+ (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{252} << 20));
+ }
emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{b} << 20));
}
break;
}
- if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || insn.Bits(81, 3) != 7 ||
- (opcode == 0x211 && insn.Bits(62, 1))) return fail(index, "native LEA carry/high mode");
- // 0x211 bit 63 negates Rb (LEA Rd, Ra, -Rb, s), as Maxwell ISCADD's bit 48.
- u32 b = insn.Bits(32, 8);
+ const u32 carry_output = insn.Bits(81, 3);
+ if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) ||
+ (carry_output != 7 && (rewritten_uniform || (opcode != 0x811 && insn.Bits(63, 1)))) ||
+ (opcode == 0x211 && insn.Bits(62, 1)) || (opcode == 0xc11 && (insn.words[1] & ~0x8000003fU)))
+ return fail(index, "native LEA carry/high mode");
+ // 0x211/0xc11 bit 63 negates Rb (LEA Rd, Ra, -Rb, s), as Maxwell ISCADD's bit 48.
+ u32 b = opcode == 0xc11 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8);
+ if (b == ~0U) return fail(index, "native uniform LEA register");
if (opcode == 0x811) { mov_imm(251, insn.words[1]); b = 251; }
+ if (carry_output != 7) {
+ // LEA Rd, Pu, Ra, Rb, s: Pu is the carry of (Ra << s) + Rb, set when the sum wraps
+ // below Ra << s. The guard may be Pu, so Rd is written first.
+ emit(0x3848000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(75, 5)} << 20));
+ emit(0x5c10000000000000ULL | predicate | 252 | (u64{250} << 8) | (u64{b} << 20));
+ emit(base(0x5c98000000000000ULL) | (u64{252} << 20) | (u64{15} << 39));
+ emit(0x5b60000000000000ULL | predicate | 7 | (u64{carry_output} << 3) | (u64{252} << 8) |
+ (u64{250} << 20) | (u64{7} << 39) | (u64{1} << 49));
+ break;
+ }
emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) |
(u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39) |
- (u64{opcode == 0x211 ? insn.Bits(63, 1) : 0U} << 48));
+ (u64{opcode != 0x811 ? insn.Bits(63, 1) : 0U} << 48));
break;
}
case 0x210: case 0x810: case 0xa10: case 0xc10: { // IADD3 with zero third source and no carry outputs
@@ -2022,7 +2337,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
// a carry-in predicate (bits 87-89, negate 90). Uniform forms keep UPn in R239 bit 6-n.
if ((opcode == 0x210 || opcode == 0x810) && insn.Bits(64, 8) == 255 && !insn.Bits(73, 1) &&
!insn.Bits(75, 2) && insn.Bits(84, 3) == 7 && insn.Bits(77, 4) == 15 && !insn.Bits(72, 1) &&
- (insn.Bits(81, 3) != 7 || insn.Bits(74, 1))) {
+ (insn.Bits(81, 3) != 7 || insn.Bits(74, 1)) && (opcode == 0x810 || !insn.Bits(63, 1))) {
const bool carry_in = insn.Bits(74, 1) != 0;
const u32 carry_out = insn.Bits(81, 3);
const u32 a = insn.Bits(24, 8);
@@ -2065,9 +2380,65 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{251} << 20));
break;
}
- if (opcode == 0xa10 && insn.Bits(81, 3) != 7) {
+ // Three sources, negations or two carries: the sum builds up in R253 (.CC) with the carry
+ // count in R254 (.X). Pu/Pv (bits 81/84) are set when the count reaches 1/2, and IADD3.X
+ // adds both carry-ins (bits 87, 77). A negated source is ~x, plus 1 unless extended.
+ const bool extended = insn.Bits(74, 1) != 0;
+ const bool negate_a = insn.Bits(72, 1), negate_b = opcode != 0x810 && insn.Bits(63, 1), negate_c = insn.Bits(75, 1);
+ if ((extended || insn.Bits(81, 6) != 0x3f) &&
+ (insn.Bits(64, 8) != 255 || insn.Bits(84, 3) != 7 || negate_a || negate_b || negate_c ||
+ (extended && insn.Bits(77, 4) != 15))) {
+ const u32 first_output = insn.Bits(81, 3), second_output = insn.Bits(84, 3);
+ if (rewritten_uniform || guarded_uniform || insn.Bits(73, 1) || insn.Bits(76, 1) ||
+ (first_output != 7 && first_output == second_output) ||
+ (!extended && (insn.Bits(77, 4) != 15 || insn.Bits(87, 4) != 15)) ||
+ (opcode == 0x210 && (insn.words[1] & ~0x800000ffU)) || (opcode == 0xa10 && insn.Bits(62, 1)) ||
+ (opcode == 0xc10 && (insn.words[1] & ~0x8000003fU)))
+ return fail(index, "native three-input carry mode");
+ u32 a = insn.Bits(24, 8), b = operand(1, false), c = insn.Bits(64, 8);
+ const auto invert = [&](u32& source, u32 target) {
+ emit(0x0400000000000000ULL | predicate | target | (u64{source} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53));
+ source = target;
+ };
+ if (negate_a) invert(a, 250);
+ if (negate_b) invert(b, 251);
+ if (negate_c) invert(c, 252);
+ const auto accumulate = [&](u32 source) {
+ emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{source} << 20) | (u64{1} << 47));
+ emit(0x5c10000000000000ULL | predicate | 254 | (u64{254} << 8) | (u64{255} << 20) | (u64{1} << 43));
+ };
+ emit(0x5c10000000000000ULL | predicate | 253 | (u64{a} << 8) | (u64{b} << 20) | (u64{1} << 47));
+ emit(0x5c10000000000000ULL | predicate | 254 | (u64{255} << 8) | (u64{255} << 20) | (u64{1} << 43));
+ accumulate(c);
+ if (extended) {
+ for (const u32 carry : {insn.Bits(87, 4), insn.Bits(77, 4)}) {
+ if (carry == 15) continue;
+ mov_imm(250, 1);
+ if (carry != 7)
+ emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) |
+ (u64{carry & 7} << 39) | (u64{carry >> 3} << 42));
+ accumulate(250);
+ }
+ } else if (const u32 ones = u32{negate_a} + u32{negate_b} + u32{negate_c}) {
+ mov_imm(250, ones);
+ accumulate(250);
+ }
+ if (dest != 255) emit(base(0x5c98000000000000ULL) | (u64{253} << 20) | (u64{15} << 39));
+ // An output that is also the guard predicate is written last.
+ const bool guard_first = (native_predicate & 7) != 7 && first_output == (native_predicate & 7);
+ for (const u32 output : {guard_first ? 1U : 0U, guard_first ? 0U : 1U}) {
+ const u32 target = output ? second_output : first_output;
+ if (target == 7) continue;
+ mov_imm(250, output + 1);
+ emit(0x5b60000000000000ULL | predicate | 7 | (u64{target} << 3) | (u64{254} << 8) |
+ (u64{250} << 20) | (u64{7} << 39) | (u64{6} << 49));
+ }
+ break;
+ }
+ if ((opcode == 0xa10 || opcode == 0xc10) && insn.Bits(81, 3) != 7) { // cbuf or uniform b
if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(84, 7) != 0x7f ||
- insn.Bits(77, 4) != 15 || insn.Bits(63, 1) || insn.Bits(72, 1))
+ insn.Bits(77, 4) != 15 || insn.Bits(63, 1) || insn.Bits(72, 1) ||
+ (opcode == 0xc10 && (insn.words[1] & ~0x3fU)))
return fail(index, "native constant add carry mode");
const u32 b = operand(1, false);
emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20));
@@ -2075,13 +2446,16 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) {
(dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49));
break;
}
- if (opcode == 0xa10 && insn.Bits(74, 1)) {
- if (insn.Bits(24, 8) != 255 || (insn.words[2] & ~0x07800000U) != 0x007fe4ffU)
+ if ((opcode == 0xa10 || opcode == 0xc10) && insn.Bits(74, 1)) { // IADD3.X Ra + cbuf or UR + carry predicate
+ if ((insn.words[2] & ~(0x07800000U | (opcode == 0xc10 ? 0x08000000U : 0U))) != 0x007fe4ffU ||
+ (opcode == 0xc10 && (insn.words[1] & ~0x3fU)))
return fail(index, "native extended constant add mode");
const u32 b = operand(1, false);
mov_imm(250, 1);
emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) |
(u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42));
+ if (insn.Bits(24, 8) != 255)
+ emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{insn.Bits(24, 8)} << 20));
emit(base(0x5c10000000000000ULL) | (u64{b} << 8) | (u64{250} << 20));
break;
}
diff --git a/src/shader_recompiler/ir_opt/position_pass.cpp b/src/shader_recompiler/ir_opt/position_pass.cpp
index 4890346944..600a596bf3 100644
--- a/src/shader_recompiler/ir_opt/position_pass.cpp
+++ b/src/shader_recompiler/ir_opt/position_pass.cpp
@@ -83,6 +83,37 @@ void BlitConstantsPass(Environment& env, IR::Program& program) {
if (env.ShaderStage() != Stage::VertexB && env.ShaderStage() != Stage::Fragment) {
return;
}
+ // Bank 24 is root table 0, which games may load themselves (3D methods 0x504/0x508; the buffer
+ // cache binds it as bank 17): Fire Emblem: Fortune's Weave keeps its material push constants
+ // there. A shader reading bank 17 outside the blit block reads such a table and keeps its reads,
+ // as does one reading only words the blit block leaves zero (an unbound bank reads zero anyway;
+ // the game's enemy health gauges read only 0x40 of their table).
+ const auto in_blit_block = [](u32 offset) {
+ return offset < 0x20 || (offset >= 0x30 && offset < 0x4c) || (offset >= 0x50 && offset < 0x60);
+ };
+ const auto is_nonzero_word = [](u32 offset) {
+ return offset < 0x20 || offset == 0x38 || offset == 0x3c || offset == 0x58 || offset == 0x5c;
+ };
+ bool reads_nonzero_word{};
+ for (IR::Block* const block : program.post_order_blocks) {
+ for (IR::Inst& inst : block->Instructions()) {
+ const IR::Opcode opcode{inst.GetOpcode()};
+ if ((opcode != IR::Opcode::GetCbufU32 && opcode != IR::Opcode::GetCbufF32 &&
+ opcode != IR::Opcode::GetCbufU32x2) ||
+ !inst.Arg(0).IsImmediate() || inst.Arg(0).U32() != 17 || !inst.Arg(1).IsImmediate()) {
+ continue;
+ }
+ const u32 offset{inst.Arg(1).U32()};
+ const bool pair{opcode == IR::Opcode::GetCbufU32x2};
+ if (!in_blit_block(offset) || (pair && !in_blit_block(offset + 4))) {
+ return;
+ }
+ reads_nonzero_word |= is_nonzero_word(offset) || (pair && is_nonzero_word(offset + 4));
+ }
+ }
+ if (!reads_nonzero_word) {
+ return;
+ }
bool used{};
for (IR::Block* const block : program.post_order_blocks) {
for (IR::Inst& inst : block->Instructions()) {
diff --git a/src/tests/video_core/ampere_returns.cpp b/src/tests/video_core/ampere_returns.cpp
index 56b8b374da..163758e07e 100644
--- a/src/tests/video_core/ampere_returns.cpp
+++ b/src/tests/video_core/ampere_returns.cpp
@@ -48,4 +48,31 @@ TEST_CASE("Ampere callees still reject writes to the link", "[video_core][ampere
REQUIRE(lowered.error == "native callee overwrites link register");
REQUIRE(lowered.error_offset == 0x30);
}
+
+TEST_CASE("Ampere callees can compare into predicates with an R0 link", "[video_core][ampere]") {
+ // ISETP's zero destination field is not a write to R0.
+ const std::array<Instruction, 5> code{{
+ {{0x00007802, 0x20, 0x00000f00, 0}},
+ {{0x00007944, 0x10, 0x03c00000, 0}},
+ {{0x0000794d, 0, 0x03800000, 0}},
+ {{0x0100720c, 0x00000002, 0x03f06070, 0}},
+ {{0x00007950, 0xffffffb0, 0x03c3ffff, 0}},
+ }};
+ REQUIRE(static_cast<bool>(LowerToMaxwell(code)));
+}
+
+TEST_CASE("Ampere callees reject a wide multiply over the link", "[video_core][ampere]") {
+ // IMAD.WIDE.U32 R54:R55 writes the R55 link.
+ const std::array<Instruction, 5> code{{
+ {{0x00377802, 0x20, 0x00000f00, 0}},
+ {{0x00007944, 0x10, 0x03c00000, 0}},
+ {{0x0000794d, 0, 0x03800000, 0}},
+ {{0x01367225, 0x00000002, 0x078e00ff, 0}},
+ {{0x37007950, 0xffffffb0, 0x03c3ffff, 0}},
+ }};
+ const auto lowered = LowerToMaxwell(code);
+ REQUIRE(!static_cast<bool>(lowered));
+ REQUIRE(lowered.error == "native callee overwrites link register");
+ REQUIRE(lowered.error_offset == 0x30);
+}
} // namespace
diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index e90af3214d..2fc74cb6c8 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -994,6 +994,26 @@ void BufferCache<P>::BindHostGraphicsUniformBuffers(size_t stage) {
template <class P>
void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind) {
++channel_state->uniform_cache_shots[0];
+ // Bank 24 (root table 0, NVN2's push constants) is read from bank 17, which such games do not
+ // bind: stream the table's current words as that buffer.
+ if (index == 17 && maxwell3d->root_tables.loaded && channel_state->uniform_buffers[stage][index].size == 0) {
+ const auto& words = maxwell3d->root_tables.tables[0];
+ const u32 size = (*channel_state->uniform_buffer_sizes)[stage][index];
+ const u32 copied = (std::min)(size, static_cast<u32>(sizeof(words)));
+ if constexpr (IS_OPENGL) {
+ std::array<u8, sizeof(words)> data{};
+ std::memcpy(data.data(), words.data(), copied);
+ runtime.BindFastUniformBuffer(stage, binding_index, copied);
+ runtime.PushFastUniformBuffer(stage, binding_index, std::span<const u8>(data.data(), copied));
+ } else {
+ const std::span<u8> span = runtime.BindMappedUniformBuffer(stage, binding_index, size);
+ std::memset(span.data(), 0, span.size());
+ std::memcpy(span.data(), words.data(), copied);
+ }
+ channel_state->fast_bound_uniform_buffers[stage] |= 1u << binding_index;
+ channel_state->uniform_buffer_binding_sizes[stage][binding_index] = size;
+ return;
+ }
const Binding& binding = channel_state->uniform_buffers[stage][index];
const DAddr device_addr = binding.device_addr;
const u32 size = (std::min)(binding.size, (*channel_state->uniform_buffer_sizes)[stage][index]);
diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp
index ab2ba10162..b8a6567907 100644
--- a/src/video_core/engines/kepler_compute.cpp
+++ b/src/video_core/engines/kepler_compute.cpp
@@ -6,6 +6,7 @@
#include <algorithm>
#include <atomic>
+#include <bit>
#include <bitset>
#include <cstdlib>
#include <fstream>
@@ -557,6 +558,7 @@ bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor&
// The console runs this on tensor cores; stand in with a plain upscale of its input.
return UpscaleInsteadOfNativeProgram();
}
+ RememberDlssMotionVectors();
return false;
}
return true;
@@ -588,9 +590,35 @@ void KeplerCompute::DescribeNativeResources(const NativeQmdScheduler::Descriptor
std::atomic<u32> g_nx2_trace_draws_after_upscale{};
+// Fire Emblem: Fortune's Weave's first DLSS pass (rejected, before the network) has the input colour
+// at cbuf0+0x278, its motion vectors (R16G16 float, the input's size) at +0x290 and this frame's
+// camera jitter (x, y; input pixels, Halton bases 2 and 3 less 0.5) at +0x1b0.
+void KeplerCompute::RememberDlssMotionVectors() {
+ const auto& cbuf = launch_description.const_buffer_config[0];
+ if (cbuf.size.Value() < 0x294) return;
+ const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low;
+ const auto image = [&](u32 offset) -> std::optional<Texture::TICEntry> {
+ const u32 tic_index = memory_manager.Read<u32>(cbuf_address + offset) & 0xfffff;
+ if (tic_index > regs.tic.limit) return std::nullopt;
+ return GetTICEntry(tic_index);
+ };
+ const auto color = image(0x278), motion = image(0x290);
+ if (!color || !motion || color->format != Texture::TextureFormat::B10G11R11 ||
+ motion->format != Texture::TextureFormat::R16G16 || motion->Address() == 0 ||
+ motion->Width() != color->Width() || motion->Height() != color->Height() ||
+ !motion->IsBlockLinear())
+ return;
+ dlss_motion_vectors = *motion;
+ dlss_motion_color = color->Address();
+ dlss_jitter = {std::bit_cast<f32>(memory_manager.Read<u32>(cbuf_address + 0x1b0)),
+ std::bit_cast<f32>(memory_manager.Read<u32>(cbuf_address + 0x1b4))};
+}
+
// Title 4's upscaler reads the tonemapped frame through the texture handle at cbuf0+0x90 and
// writes the output resolution image through the surface handle at cbuf0+0xa0 (from its
-// TLD/SUST operands). Blit one to the other with bilinear filtering when both are plain 2D images
+// TLD/SUST operands). Fire Emblem: Fortune's Weave's DLSS network ends with a pass that reads the
+// input colour through cbuf0+0x328 and writes the DLSS output through cbuf0+0x348 (B10G11R11; 720p
+// to 1440p docked). Blit one to the other with bilinear filtering when both are plain 2D images
// of the same format and the output is larger.
bool KeplerCompute::UpscaleInsteadOfNativeProgram() {
const auto& cbuf = launch_description.const_buffer_config[0];
@@ -600,11 +628,20 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() {
if (tic_index > regs.tic.limit) return std::nullopt;
return GetTICEntry(tic_index);
};
- const auto input = image(0x90), output = image(0xa0);
- if (!input || !output || input->format != output->format || input->Address() == 0 ||
- output->Address() == 0 || output->Width() <= input->Width() || !input->IsBlockLinear() ||
- !output->IsBlockLinear())
- return false;
+ const auto usable = [](const std::optional<Texture::TICEntry>& input,
+ const std::optional<Texture::TICEntry>& output) {
+ return input && output && input->format == output->format && input->Address() != 0 &&
+ output->Address() != 0 && output->Width() > input->Width() && input->IsBlockLinear() &&
+ output->IsBlockLinear();
+ };
+ auto input = image(0x90), output = image(0xa0);
+ bool fortunes_weave = false;
+ if (!usable(input, output)) {
+ input = image(0x328);
+ output = image(0x348);
+ if (!usable(input, output)) return false;
+ fortunes_weave = true;
+ }
using RTFormat = Tegra::RenderTargetFormat;
RTFormat format{};
switch (input->format) {
@@ -700,7 +737,24 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() {
source.format = static_cast<RTFormat>(std::strtoul(value, nullptr, 16));
}
}
- const bool done = rasterizer->AccelerateSurfaceCopy(source, surface(*output), config);
+ // Fortune's Weave: a temporal upscale along its motion vectors that undoes this frame's camera
+ // jitter (both from the frame's first DLSS pass) and accumulates the frames as DLSS would.
+ // EDEN_NX2_TEMPORAL_UPSCALE=0 blits instead.
+ const auto upscale_temporally = [&] {
+ static const bool enabled = [] {
+ const char* value = std::getenv("EDEN_NX2_TEMPORAL_UPSCALE");
+ return value == nullptr || std::string_view{value} != "0";
+ }();
+ if (!fortunes_weave || !enabled || !dlss_motion_vectors ||
+ dlss_motion_color != input->Address())
+ return false;
+ Fermi2D::Surface motion = surface(*dlss_motion_vectors);
+ motion.format = RTFormat::R16G16_FLOAT;
+ return rasterizer->AccelerateTemporalUpscale(source, surface(*output), motion, dlss_jitter,
+ {1.0f, 1.0f});
+ };
+ const bool done = upscale_temporally() ||
+ rasterizer->AccelerateSurfaceCopy(source, surface(*output), config);
if (static u32 checked{}; checked++ == sample_at) {
const u64 size = u64{output->Width()} * output->Height() * 4;
if (const auto device = memory_manager.GpuToCpuAddress(output->Address())) {
diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h
index b525636dde..b5aee3706a 100644
--- a/src/video_core/engines/kepler_compute.h
+++ b/src/video_core/engines/kepler_compute.h
@@ -266,7 +266,15 @@ private:
bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor);
void ProcessNativeReportSemaphore(u32 argument);
void DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd);
+ void RememberDlssMotionVectors();
bool UpscaleInsteadOfNativeProgram();
+
+ /// Motion vectors of Fire Emblem: Fortune's Weave's DLSS evaluation, from its first pass (run
+ /// earlier in the frame than the network pass the stand-in replaces), with the input colour
+ /// image and camera jitter they go with.
+ std::optional<Texture::TICEntry> dlss_motion_vectors;
+ GPUVAddr dlss_motion_color{};
+ std::array<f32, 2> dlss_jitter{};
bool ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call);
NativeQmdScheduler native_qmd;
std::array<u32, 0xc00>* native_data_ram{};
diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp
index 6adfcf89b8..4001284127 100644
--- a/src/video_core/engines/maxwell_3d.cpp
+++ b/src/video_core/engines/maxwell_3d.cpp
@@ -164,6 +164,8 @@ bool Maxwell3D::IsMethodExecutable(u32 method) {
case MAXWELL3D_REG_INDEX(vertex_array_instance_first):
case MAXWELL3D_REG_INDEX(vertex_array_instance_subsequent):
case MAXWELL3D_REG_INDEX(draw_texture.src_y0):
+ case RootTables::SelectMethod:
+ case RootTables::LoadMethod:
case MAXWELL3D_REG_INDEX(wait_for_idle):
case MAXWELL3D_REG_INDEX(shadow_ram_control):
case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
@@ -346,6 +348,17 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume
LOG_DEBUG(HW_GPU, "Native GPU surface clear: value={:#x}", argument);
}
switch (method) {
+ case RootTables::SelectMethod:
+ root_tables.selected = argument & 7;
+ root_tables.offset = (argument >> 8) & 0xff;
+ return;
+ case RootTables::LoadMethod:
+ if (root_tables.offset < 256 && root_tables.offset % 4 == 0) {
+ root_tables.tables[root_tables.selected][root_tables.offset / 4] = argument;
+ root_tables.loaded = true;
+ }
+ root_tables.offset += 4;
+ return;
case MAXWELL3D_REG_INDEX(wait_for_idle):
return rasterizer->WaitForIdle();
case MAXWELL3D_REG_INDEX(shadow_ram_control):
@@ -819,6 +832,8 @@ bool FrontHandledOutsideDrawManager(u32 method) {
return true;
}
switch (method) {
+ case Maxwell3D::RootTables::SelectMethod:
+ case Maxwell3D::RootTables::LoadMethod:
case MAXWELL3D_REG_INDEX(wait_for_idle):
case MAXWELL3D_REG_INDEX(shadow_ram_control):
case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr):
diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h
index 0121f1a01c..a32b31fd1e 100644
--- a/src/video_core/engines/maxwell_3d.h
+++ b/src/video_core/engines/maxwell_3d.h
@@ -3160,6 +3160,18 @@ public:
std::array<u32, 0xc00>& GetNativeDataRam() { return native_data_ram; }
std::span<u32> GetNativeMacroPositions() { return native_macro_context ? std::span<u32>{native_macro_context->positions} : std::span<u32>{macro_positions}; }
+ /// Root tables: eight 256-byte tables written inline (method 0x504 selects a table, bits 0-2,
+ /// and a byte offset, bits 8-15; each 0x508 writes the next word) that shaders read as constant
+ /// banks 24-31. NVN2 keeps push constants in bank 24, which the Ampere lowering reads as bank 17.
+ struct RootTables {
+ static constexpr u32 SelectMethod = 0x504 / 4;
+ static constexpr u32 LoadMethod = 0x508 / 4;
+ std::array<std::array<u32, 64>, 8> tables{};
+ u32 selected{};
+ u32 offset{};
+ bool loaded{};
+ } root_tables;
+
/// Write the value to the register identified by method.
void CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) override;
diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt
index 35b350c2cd..ada5c779dc 100644
--- a/src/video_core/host_shaders/CMakeLists.txt
+++ b/src/video_core/host_shaders/CMakeLists.txt
@@ -71,6 +71,7 @@ set(SHADER_FILES
${CMAKE_CURRENT_SOURCE_DIR}/smaa_blending_weight_calculation.frag
${CMAKE_CURRENT_SOURCE_DIR}/smaa_neighborhood_blending.vert
${CMAKE_CURRENT_SOURCE_DIR}/smaa_neighborhood_blending.frag
+ ${CMAKE_CURRENT_SOURCE_DIR}/temporal_upscale.frag
${CMAKE_CURRENT_SOURCE_DIR}/vulkan_blit_depth_stencil.frag
${CMAKE_CURRENT_SOURCE_DIR}/vulkan_color_clear.frag
${CMAKE_CURRENT_SOURCE_DIR}/vulkan_color_clear.vert
diff --git a/src/video_core/host_shaders/temporal_upscale.frag b/src/video_core/host_shaders/temporal_upscale.frag
new file mode 100644
index 0000000000..0e76df2b43
--- /dev/null
+++ b/src/video_core/host_shaders/temporal_upscale.frag
@@ -0,0 +1,132 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Stands in for a game's DLSS evaluation (the tensor-core network cannot run here): a temporal
+// upscale. Each frame's input samples are placed where the camera jitter put them and weighted by
+// their distance to the output pixel; they accumulate with the previous output, reprojected along
+// the game's motion vectors (bicubic) and clipped to the current 3x3 neighbourhood.
+
+#version 450
+
+layout(binding = 0) uniform sampler2D input_color;
+layout(binding = 1) uniform sampler2D motion_vectors;
+layout(binding = 2) uniform sampler2D history;
+
+layout(push_constant) uniform PushConstants {
+ // The vertex stage's texture scale and offset come first.
+ layout(offset = 16) vec2 jitter; // this frame's sample offset, in input pixels
+ vec2 motion_scale; // motion vector -> UV offset to the previous frame
+ float current_weight; // largest weight of this frame; 1 ignores the history
+};
+
+layout(location = 0) in vec2 texcoord; // 0 to 1 across the output
+layout(location = 0) out vec4 color;
+
+// The blit helper's samplers take unnormalized (texel) coordinates.
+
+// Blending in a range-compressed space keeps bright pixels from dominating the average.
+vec3 Compress(vec3 c) {
+ return c / (1.0 + max(max(c.r, c.g), c.b));
+}
+
+vec3 Expand(vec3 c) {
+ c = clamp(c, vec3(0.0), vec3(1.0 - 1.0 / 65504.0));
+ return c / (1.0 - max(max(c.r, c.g), c.b));
+}
+
+vec3 RgbToYCoCg(vec3 c) {
+ return vec3(dot(c, vec3(0.25, 0.5, 0.25)), dot(c, vec3(0.5, 0.0, -0.5)),
+ dot(c, vec3(-0.25, 0.5, -0.25)));
+}
+
+vec3 YCoCgToRgb(vec3 c) {
+ return vec3(c.x + c.y - c.z, c.x + c.z, c.x - c.y - c.z);
+}
+
+// Moves value towards the box centre until it lies inside the box.
+vec3 ClipToBox(vec3 value, vec3 low, vec3 high) {
+ const vec3 center = 0.5 * (low + high);
+ const vec3 extent = 0.5 * (high - low) + 1.0 / 65504.0;
+ const vec3 offset = value - center;
+ const vec3 units = abs(offset / extent);
+ const float largest = max(units.x, max(units.y, units.z));
+ return largest > 1.0 ? center + offset / largest : value;
+}
+
+// Catmull-Rom filtered history from five bilinear fetches: repeated bilinear reprojection would
+// blur the accumulated image a little more every frame.
+vec3 SampleHistory(vec2 position) {
+ const vec2 texel1 = floor(position - 0.5) + 0.5;
+ const vec2 f = position - texel1;
+ const vec2 w0 = f * (-0.5 + f * (1.0 - 0.5 * f));
+ const vec2 w1 = 1.0 + f * f * (-2.5 + 1.5 * f);
+ const vec2 w2 = f * (0.5 + f * (2.0 - 1.5 * f));
+ const vec2 w3 = f * f * (-0.5 + 0.5 * f);
+ const vec2 w12 = w1 + w2;
+ const vec2 texel0 = texel1 - 1.0;
+ const vec2 texel3 = texel1 + 2.0;
+ const vec2 texel12 = texel1 + w2 / w12;
+ vec3 sum = texture(history, vec2(texel12.x, texel0.y)).rgb * (w12.x * w0.y);
+ sum += texture(history, vec2(texel0.x, texel12.y)).rgb * (w0.x * w12.y);
+ sum += texture(history, texel12).rgb * (w12.x * w12.y);
+ sum += texture(history, vec2(texel3.x, texel12.y)).rgb * (w3.x * w12.y);
+ sum += texture(history, vec2(texel12.x, texel3.y)).rgb * (w12.x * w3.y);
+ const float weight = w12.x * w0.y + w0.x * w12.y + w12.x * w12.y + w3.x * w12.y + w12.x * w3.y;
+ return max(sum / weight, vec3(0.0));
+}
+
+void main() {
+ const ivec2 input_size = textureSize(input_color, 0);
+ const ivec2 motion_size = textureSize(motion_vectors, 0);
+ // The output pixel's centre in input pixels. Input texel k sampled the scene at k + 0.5 - jitter
+ // (the sign measured against consecutive frames).
+ const vec2 target = texcoord * vec2(input_size);
+ const ivec2 nearest = ivec2(floor(target + jitter));
+
+ vec3 filtered = vec3(0.0);
+ float total_weight = 0.0;
+ float nearest_weight = 0.0;
+ vec3 sum = vec3(0.0);
+ vec3 squares = vec3(0.0);
+ vec2 motion = vec2(0.0);
+ float longest = -1.0;
+ for (int y = -1; y <= 1; ++y) {
+ for (int x = -1; x <= 1; ++x) {
+ const ivec2 texel = clamp(nearest + ivec2(x, y), ivec2(0), input_size - 1);
+ const vec3 sample_color = RgbToYCoCg(Compress(texelFetch(input_color, texel, 0).rgb));
+ // A Blackman-Harris-like window over the sample's distance to the output pixel.
+ const vec2 distance = vec2(texel) + 0.5 - jitter - target;
+ const float weight = exp(-2.29 * dot(distance, distance));
+ filtered += sample_color * weight;
+ total_weight += weight;
+ nearest_weight = max(nearest_weight, weight);
+ sum += sample_color;
+ squares += sample_color * sample_color;
+ // The longest motion around the pixel, so that an edge follows the object in front.
+ const vec2 candidate =
+ texelFetch(motion_vectors, clamp(texel, ivec2(0), motion_size - 1), 0).rg;
+ const float length_squared = dot(candidate, candidate);
+ if (length_squared > longest) {
+ longest = length_squared;
+ motion = candidate;
+ }
+ }
+ }
+ const vec3 current = filtered / total_weight;
+ const vec3 mean = sum / 9.0;
+ const vec3 deviation = sqrt(max(squares / 9.0 - mean * mean, vec3(0.0)));
+ const vec3 low = mean - deviation;
+ const vec3 high = mean + deviation;
+
+ const vec2 previous = texcoord + motion * motion_scale;
+ // A frame whose samples land close to this pixel counts more; one whose samples land between
+ // pixels mostly keeps the history (from other jitter positions) there.
+ float weight = current_weight * nearest_weight;
+ if (current_weight >= 1.0 || any(lessThan(previous, vec2(0.0))) ||
+ any(greaterThan(previous, vec2(1.0)))) {
+ weight = 1.0;
+ }
+ const vec2 history_position = previous * vec2(textureSize(history, 0));
+ const vec3 past = ClipToBox(RgbToYCoCg(Compress(SampleHistory(history_position))), low, high);
+ color = vec4(Expand(YCoCgToRgb(mix(past, current, weight))), 1.0);
+}
diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h
index b527c0e2a9..8486d44ac9 100644
--- a/src/video_core/rasterizer_interface.h
+++ b/src/video_core/rasterizer_interface.h
@@ -6,6 +6,7 @@
#pragma once
+#include <array>
#include <functional>
#include <optional>
#include <span>
@@ -162,6 +163,16 @@ public:
return false;
}
+ /// Stand-in for a DLSS evaluation: temporal upscale of src into dst along motion_vectors
+ /// (jitter in src pixels; motion_scale turns a motion vector into a UV offset to the previous
+ /// frame). False when unsupported; the caller then falls back to AccelerateSurfaceCopy.
+ [[nodiscard]] virtual bool AccelerateTemporalUpscale(
+ const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst,
+ const Tegra::Engines::Fermi2D::Surface& motion_vectors, std::array<f32, 2> jitter,
+ std::array<f32, 2> motion_scale) {
+ return false;
+ }
+
[[nodiscard]] virtual Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() = 0;
virtual void AccelerateInlineToMemory(GPUVAddr address, size_t copy_size,
diff --git a/src/video_core/renderer_opengl/present/layer.cpp b/src/video_core/renderer_opengl/present/layer.cpp
index 9e4a0f854a..2656892f4a 100644
--- a/src/video_core/renderer_opengl/present/layer.cpp
+++ b/src/video_core/renderer_opengl/present/layer.cpp
@@ -183,6 +183,11 @@ void Layer::ConfigureFramebufferTexture(const Tegra::FramebufferConfig& framebuf
framebuffer_texture.gl_format = GL_RGBA;
framebuffer_texture.gl_type = GL_UNSIGNED_INT_8_8_8_8_REV;
break;
+ case Service::android::PixelFormat::Rgba1010102:
+ internal_format = GL_RGB10_A2;
+ framebuffer_texture.gl_format = GL_RGBA;
+ framebuffer_texture.gl_type = GL_UNSIGNED_INT_2_10_10_10_REV;
+ break;
case Service::android::PixelFormat::Rgb565:
internal_format = GL_RGB565;
framebuffer_texture.gl_format = GL_RGB;
diff --git a/src/video_core/renderer_vulkan/blit_image.cpp b/src/video_core/renderer_vulkan/blit_image.cpp
index ddd52e9426..edd151b563 100644
--- a/src/video_core/renderer_vulkan/blit_image.cpp
+++ b/src/video_core/renderer_vulkan/blit_image.cpp
@@ -33,6 +33,7 @@
#include "video_core/host_shaders/convert_non_msaa_to_msaa_depth_stencil_frag_spv.h"
#include "video_core/host_shaders/convert_s8d24_to_abgr8_frag_spv.h"
#include "video_core/host_shaders/full_screen_triangle_vert_spv.h"
+#include "video_core/host_shaders/temporal_upscale_frag_spv.h"
#include "video_core/host_shaders/vulkan_blit_depth_stencil_frag_spv.h"
#include "video_core/host_shaders/vulkan_color_clear_frag_spv.h"
#include "video_core/host_shaders/vulkan_color_clear_vert_spv.h"
@@ -91,6 +92,15 @@ struct PushConstants {
std::array<float, 2> tex_offset;
};
+// The vertex stage's PushConstants first, then those of temporal_upscale.frag.
+struct TemporalUpscalePushConstants {
+ PushConstants vertex;
+ std::array<float, 2> jitter;
+ std::array<float, 2> motion_scale;
+ float current_weight;
+ float padding;
+};
+
struct MSAACopyPushConstants {
std::array<s32, 2> dst_offset;
std::array<s32, 2> src_offset;
@@ -109,6 +119,11 @@ constexpr std::array TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{
TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>,
TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>,
};
+constexpr std::array THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{
+ TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>,
+ TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>,
+ TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<2>,
+};
constexpr VkDescriptorSetLayoutCreateInfo ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.pNext = nullptr,
@@ -133,6 +148,13 @@ constexpr VkDescriptorSetLayoutCreateInfo TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CRE
.bindingCount = static_cast<u32>(TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()),
.pBindings = TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(),
};
+constexpr VkDescriptorSetLayoutCreateInfo THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{
+ .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .bindingCount = static_cast<u32>(THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()),
+ .pBindings = THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(),
+};
template <VkShaderStageFlags stageFlags, size_t size>
inline constexpr VkPushConstantRange PUSH_CONSTANT_RANGE{
.stageFlags = stageFlags,
@@ -357,6 +379,32 @@ void UpdateOneTextureDescriptorSet(const Device& device, VkDescriptorSet descrip
device.GetLogical().UpdateDescriptorSets(write_descriptor_set, nullptr);
}
+void UpdateThreeTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set,
+ VkSampler sampler, std::array<VkImageView, 3> image_views) {
+ std::array<VkDescriptorImageInfo, 3> image_infos{};
+ std::array<VkWriteDescriptorSet, 3> write_descriptor_sets{};
+ for (u32 binding = 0; binding < 3; ++binding) {
+ image_infos[binding] = VkDescriptorImageInfo{
+ .sampler = sampler,
+ .imageView = image_views[binding],
+ .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
+ };
+ write_descriptor_sets[binding] = VkWriteDescriptorSet{
+ .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
+ .pNext = nullptr,
+ .dstSet = descriptor_set,
+ .dstBinding = binding,
+ .dstArrayElement = 0,
+ .descriptorCount = 1,
+ .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
+ .pImageInfo = &image_infos[binding],
+ .pBufferInfo = nullptr,
+ .pTexelBufferView = nullptr,
+ };
+ }
+ device.GetLogical().UpdateDescriptorSets(write_descriptor_sets, nullptr);
+}
+
void UpdateTwoTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set,
VkSampler sampler, VkImageView image_view_0,
VkImageView image_view_1) {
@@ -590,10 +638,14 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_,
ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)),
two_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout(
TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)),
+ three_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout(
+ THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)),
one_texture_descriptor_allocator{
descriptor_pool.Allocator(device_, scheduler_, *one_texture_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<1>)},
two_textures_descriptor_allocator{
descriptor_pool.Allocator(device_, scheduler_, *two_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<2>)},
+ three_textures_descriptor_allocator{
+ descriptor_pool.Allocator(device_, scheduler_, *three_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<3>)},
one_texture_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo(
one_texture_set_layout.address(),
PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT, sizeof(PushConstants)>))),
@@ -601,6 +653,11 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_,
device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo(
two_textures_set_layout.address(),
PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT, sizeof(PushConstants)>))),
+ temporal_upscale_pipeline_layout(
+ device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo(
+ three_textures_set_layout.address(),
+ PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT,
+ sizeof(TemporalUpscalePushConstants)>))),
clear_color_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo(
nullptr, PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(float) * 4>))),
msaa_copy_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo(
@@ -612,6 +669,7 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_,
PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(MSAACopyPushConstants)>))),
full_screen_vert(BuildShader(device, FULL_SCREEN_TRIANGLE_VERT_SPV)),
blit_color_to_color_frag(BuildShader(device, BLIT_COLOR_FLOAT_FRAG_SPV)),
+ temporal_upscale_frag(BuildShader(device, TEMPORAL_UPSCALE_FRAG_SPV)),
blit_color_msaa_frag(BuildShader(device, BLIT_COLOR_MSAA_FRAG_SPV)),
blit_depth_stencil_frag(device.IsExtShaderStencilExportSupported()
? BuildShader(device, VULKAN_BLIT_DEPTH_STENCIL_FRAG_SPV)
@@ -702,6 +760,42 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView
});
}
+void BlitImageHelper::TemporalUpscale(const Framebuffer* dst_framebuffer,
+ const ImageView& input_color, const ImageView& motion_vectors,
+ VkImageView history, const Region2D& dst_region,
+ std::array<f32, 2> jitter, std::array<f32, 2> motion_scale,
+ f32 current_weight) {
+ const VkPipelineLayout layout = *temporal_upscale_pipeline_layout;
+ const VkPipeline pipeline = FindOrEmplaceTemporalUpscalePipeline(dst_framebuffer->RenderPass());
+ const VkSampler sampler = *linear_sampler;
+ const std::array<VkImageView, 3> views{input_color.Handle(Shader::TextureType::Color2D),
+ motion_vectors.Handle(Shader::TextureType::Color2D),
+ history};
+ const TemporalUpscalePushConstants push_constants{
+ .vertex = {.tex_scale = {1.0f, 1.0f}, .tex_offset = {0.0f, 0.0f}},
+ .jitter = jitter,
+ .motion_scale = motion_scale,
+ .current_weight = current_weight,
+ .padding = 0.0f,
+ };
+ RecordShaderReadBarrier(scheduler, input_color);
+ RecordShaderReadBarrier(scheduler, motion_vectors);
+ scheduler.RequestRenderpass(dst_framebuffer);
+ scheduler.Record([this, dst_region, pipeline, layout, sampler, views,
+ push_constants](vk::CommandBuffer cmdbuf) {
+ const VkDescriptorSet descriptor_set = three_textures_descriptor_allocator.Commit();
+ UpdateThreeTexturesDescriptorSet(device, descriptor_set, sampler, views);
+ cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
+ cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, descriptor_set,
+ nullptr);
+ BindBlitState(cmdbuf, dst_region);
+ cmdbuf.PushConstants(layout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT,
+ push_constants);
+ cmdbuf.Draw(3, 1, 0, 0);
+ });
+ scheduler.InvalidateState();
+}
+
void BlitImageHelper::BlitImpl(const Framebuffer* dst_framebuffer,
const ImageView& src_image_view, const Region2D& dst_region,
const Region2D& src_region, VkPipeline pipeline, VkSampler sampler,
@@ -1285,6 +1379,59 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe
return *blit_color_pipelines.back();
}
+VkPipeline BlitImageHelper::FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass) {
+ const auto it = std::ranges::find(temporal_upscale_keys, renderpass);
+ if (it != temporal_upscale_keys.end()) {
+ return *temporal_upscale_pipelines[std::distance(temporal_upscale_keys.begin(), it)];
+ }
+ temporal_upscale_keys.push_back(renderpass);
+ const std::array stages = MakeStages(*full_screen_vert, *temporal_upscale_frag);
+ const VkPipelineColorBlendAttachmentState blend_attachment{
+ .blendEnable = VK_FALSE,
+ .srcColorBlendFactor = VK_BLEND_FACTOR_ZERO,
+ .dstColorBlendFactor = VK_BLEND_FACTOR_ZERO,
+ .colorBlendOp = VK_BLEND_OP_ADD,
+ .srcAlphaBlendFactor = VK_BLEND_FACTOR_ZERO,
+ .dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO,
+ .alphaBlendOp = VK_BLEND_OP_ADD,
+ .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
+ VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
+ };
+ const VkPipelineColorBlendStateCreateInfo color_blend_create_info{
+ .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .logicOpEnable = VK_FALSE,
+ .logicOp = VK_LOGIC_OP_CLEAR,
+ .attachmentCount = 1,
+ .pAttachments = &blend_attachment,
+ .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f},
+ };
+ const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device);
+ temporal_upscale_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({
+ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .stageCount = static_cast<u32>(stages.size()),
+ .pStages = stages.data(),
+ .pVertexInputState = &PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
+ .pInputAssemblyState = &input_assembly_ci,
+ .pTessellationState = nullptr,
+ .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO,
+ .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
+ .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
+ .pDepthStencilState = nullptr,
+ .pColorBlendState = &color_blend_create_info,
+ .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO,
+ .layout = *temporal_upscale_pipeline_layout,
+ .renderPass = renderpass,
+ .subpass = 0,
+ .basePipelineHandle = VK_NULL_HANDLE,
+ .basePipelineIndex = 0,
+ }, device.StaticPipelineCache()));
+ return *temporal_upscale_pipelines.back();
+}
+
VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key) {
const auto it = std::ranges::find(blit_depth_stencil_keys, key);
if (it != blit_depth_stencil_keys.end()) {
diff --git a/src/video_core/renderer_vulkan/blit_image.h b/src/video_core/renderer_vulkan/blit_image.h
index 62e6743a23..303beeea5c 100644
--- a/src/video_core/renderer_vulkan/blit_image.h
+++ b/src/video_core/renderer_vulkan/blit_image.h
@@ -82,6 +82,13 @@ public:
VkImage src_image, VkSampler src_sampler, const Region2D& dst_region,
const Region2D& src_region, const Extent3D& src_size);
+ /// Temporal upscale (temporal_upscale.frag) into dst_framebuffer: the jittered input colour,
+ /// accumulated with the history image reprojected along the motion vectors.
+ void TemporalUpscale(const Framebuffer* dst_framebuffer, const ImageView& input_color,
+ const ImageView& motion_vectors, VkImageView history,
+ const Region2D& dst_region, std::array<f32, 2> jitter,
+ std::array<f32, 2> motion_scale, f32 current_weight);
+
void BlitColorMSAA(const Framebuffer* dst_framebuffer, const ImageView& src_image_view,
const Region2D& dst_region, const Region2D& src_region);
@@ -170,6 +177,8 @@ private:
[[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key);
+ [[nodiscard]] VkPipeline FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass);
+
[[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key);
[[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key);
@@ -207,15 +216,19 @@ private:
vk::DescriptorSetLayout one_texture_set_layout;
vk::DescriptorSetLayout two_textures_set_layout;
+ vk::DescriptorSetLayout three_textures_set_layout;
DescriptorAllocator one_texture_descriptor_allocator;
DescriptorAllocator two_textures_descriptor_allocator;
+ DescriptorAllocator three_textures_descriptor_allocator;
vk::PipelineLayout one_texture_pipeline_layout;
vk::PipelineLayout two_textures_pipeline_layout;
+ vk::PipelineLayout temporal_upscale_pipeline_layout;
vk::PipelineLayout clear_color_pipeline_layout;
vk::PipelineLayout msaa_copy_pipeline_layout;
vk::PipelineLayout msaa_copy_depth_stencil_pipeline_layout;
vk::ShaderModule full_screen_vert;
vk::ShaderModule blit_color_to_color_frag;
+ vk::ShaderModule temporal_upscale_frag;
vk::ShaderModule blit_color_msaa_frag;
vk::ShaderModule blit_depth_stencil_frag;
vk::ShaderModule blit_depth_frag;
@@ -246,6 +259,8 @@ private:
std::vector<BlitImagePipelineKey> blit_color_keys;
std::vector<vk::Pipeline> blit_color_pipelines;
+ std::vector<VkRenderPass> temporal_upscale_keys;
+ std::vector<vk::Pipeline> temporal_upscale_pipelines;
std::vector<BlitImagePipelineKey> blit_depth_stencil_keys;
std::vector<vk::Pipeline> blit_depth_stencil_pipelines;
std::vector<BlitImagePipelineKey> clear_color_keys;
diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp
index fa24c02f6d..5652090bd7 100644
--- a/src/video_core/renderer_vulkan/present/layer.cpp
+++ b/src/video_core/renderer_vulkan/present/layer.cpp
@@ -52,6 +52,8 @@ VkFormat GetFormat(const Tegra::FramebufferConfig& framebuffer) {
return VK_FORMAT_R5G6B5_UNORM_PACK16;
case Service::android::PixelFormat::Bgra8888:
return VK_FORMAT_B8G8R8A8_UNORM;
+ case Service::android::PixelFormat::Rgba1010102:
+ return VK_FORMAT_A2B10G10R10_UNORM_PACK32;
default:
UNIMPLEMENTED_MSG("Unknown framebuffer pixel format: {}",
static_cast<u32>(framebuffer.pixel_format));
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index cb31a0e7e4..c0e8825bac 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -1231,6 +1231,14 @@ bool RasterizerVulkan::AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surf
return texture_cache.BlitImage(dst, src, copy_config);
}
+bool RasterizerVulkan::AccelerateTemporalUpscale(
+ const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst,
+ const Tegra::Engines::Fermi2D::Surface& motion_vectors, std::array<f32, 2> jitter,
+ std::array<f32, 2> motion_scale) {
+ std::scoped_lock lock{texture_cache.mutex};
+ return texture_cache.TemporalUpscale(dst, src, motion_vectors, jitter, motion_scale);
+}
+
Tegra::Engines::AccelerateDMAInterface& RasterizerVulkan::AccessAccelerateDMA() {
return accelerate_dma;
}
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h
index ae9929ca0b..7aca376614 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.h
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.h
@@ -128,6 +128,11 @@ public:
bool AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surface& src,
const Tegra::Engines::Fermi2D::Surface& dst,
const Tegra::Engines::Fermi2D::Config& copy_config) override;
+ bool AccelerateTemporalUpscale(const Tegra::Engines::Fermi2D::Surface& src,
+ const Tegra::Engines::Fermi2D::Surface& dst,
+ const Tegra::Engines::Fermi2D::Surface& motion_vectors,
+ std::array<f32, 2> jitter,
+ std::array<f32, 2> motion_scale) override;
Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() override;
void AccelerateInlineToMemory(GPUVAddr address, size_t copy_size,
std::span<const u8> memory) override;
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index ddd55de690..4e428229f6 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -1491,6 +1491,154 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src,
});
}
+bool TextureCacheRuntime::TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst,
+ ImageView& input_color, ImageView& motion_vectors,
+ std::array<f32, 2> jitter,
+ std::array<f32, 2> motion_scale) {
+ if (dst.Samples() != VK_SAMPLE_COUNT_1_BIT || ImageAspectMask(dst.format) != VK_IMAGE_ASPECT_COLOR_BIT) {
+ return false;
+ }
+ const VkFormat format =
+ MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, false, dst.format).format;
+ const VkExtent2D extent{dst.size.width, dst.size.height};
+ const VkImage output = dst.ImageHandle();
+ bool reset = false;
+ if (!temporal_history.image || temporal_history.format != format ||
+ temporal_history.extent.width != extent.width ||
+ temporal_history.extent.height != extent.height) {
+ if (temporal_history.image) {
+ scheduler.Finish(); // the old history may still be sampled
+ }
+ temporal_history.image = memory_allocator.CreateImage(VkImageCreateInfo{
+ .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .imageType = VK_IMAGE_TYPE_2D,
+ .format = format,
+ .extent = {extent.width, extent.height, 1},
+ .mipLevels = 1,
+ .arrayLayers = 1,
+ .samples = VK_SAMPLE_COUNT_1_BIT,
+ .tiling = VK_IMAGE_TILING_OPTIMAL,
+ .usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT,
+ .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
+ .queueFamilyIndexCount = 0,
+ .pQueueFamilyIndices = nullptr,
+ .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
+ });
+ temporal_history.view = device.GetLogical().CreateImageView(VkImageViewCreateInfo{
+ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .image = *temporal_history.image,
+ .viewType = VK_IMAGE_VIEW_TYPE_2D,
+ .format = format,
+ .components{},
+ .subresourceRange{
+ .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+ .baseMipLevel = 0,
+ .levelCount = 1,
+ .baseArrayLayer = 0,
+ .layerCount = 1,
+ },
+ });
+ temporal_history.format = format;
+ temporal_history.extent = extent;
+ reset = true;
+ LOG_INFO(Render_Vulkan, "Temporal upscale into {}x{} (format {})", extent.width, extent.height,
+ static_cast<u32>(format));
+ }
+
+ // The history holds this pass's previous result, copied right after drawing it: the game may
+ // draw into its DLSS output afterwards (post-processing), which must not feed back. The history
+ // stays in the GENERAL layout, which the blit helper's descriptors use.
+ const VkImage history = *temporal_history.image;
+ const VkImageSubresourceLayers output_layers{
+ .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+ .mipLevel = static_cast<u32>(dst.range.base.level),
+ .baseArrayLayer = static_cast<u32>(dst.range.base.layer),
+ .layerCount = 1,
+ };
+ static constexpr VkImageSubresourceLayers history_layers{
+ .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+ .mipLevel = 0,
+ .baseArrayLayer = 0,
+ .layerCount = 1,
+ };
+ const auto barrier = [](VkImage image, VkAccessFlags src_access, VkAccessFlags dst_access,
+ VkImageLayout old_layout, const VkImageSubresourceLayers& layers) {
+ return VkImageMemoryBarrier{
+ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+ .pNext = nullptr,
+ .srcAccessMask = src_access,
+ .dstAccessMask = dst_access,
+ .oldLayout = old_layout,
+ .newLayout = VK_IMAGE_LAYOUT_GENERAL,
+ .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .image = image,
+ .subresourceRange{
+ .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+ .baseMipLevel = layers.mipLevel,
+ .levelCount = 1,
+ .baseArrayLayer = layers.baseArrayLayer,
+ .layerCount = 1,
+ },
+ };
+ };
+ if (reset) {
+ // A new history image has no contents yet; the first frame does not read it.
+ scheduler.RequestOutsideRenderPassOperationContext();
+ scheduler.Record([history, barrier](vk::CommandBuffer cmdbuf) {
+ cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
+ 0, barrier(history, 0, VK_ACCESS_SHADER_READ_BIT,
+ VK_IMAGE_LAYOUT_UNDEFINED, history_layers));
+ });
+ }
+ const Region2D dst_region{
+ .start = {0, 0},
+ .end = {static_cast<s32>(extent.width), static_cast<s32>(extent.height)},
+ };
+ // Each frame weighs at most 0.1 against the history (scaled by how close its samples land to
+ // the pixel; the neighbourhood clip limits what older samples contribute).
+ constexpr f32 current_weight = 0.1f;
+ blit_image_helper.TemporalUpscale(dst_framebuffer, input_color, motion_vectors,
+ *temporal_history.view, dst_region, jitter, motion_scale,
+ reset ? 1.0f : current_weight);
+ scheduler.RequestOutsideRenderPassOperationContext();
+ scheduler.Record([output, history, extent, output_layers, barrier](vk::CommandBuffer cmdbuf) {
+ constexpr VkAccessFlags written = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
+ const std::array before{
+ barrier(output, written, VK_ACCESS_TRANSFER_READ_BIT, VK_IMAGE_LAYOUT_GENERAL,
+ output_layers),
+ barrier(history, VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
+ VK_IMAGE_LAYOUT_GENERAL, history_layers),
+ };
+ cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT |
+ VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
+ VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, {}, before);
+ const VkImageCopy copy{
+ .srcSubresource = output_layers,
+ .srcOffset = {0, 0, 0},
+ .dstSubresource = history_layers,
+ .dstOffset = {0, 0, 0},
+ .extent = {extent.width, extent.height, 1},
+ };
+ cmdbuf.CopyImage(output, VK_IMAGE_LAYOUT_GENERAL, history, VK_IMAGE_LAYOUT_GENERAL, copy);
+ const std::array after{
+ barrier(output, VK_ACCESS_TRANSFER_READ_BIT,
+ VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
+ VK_ACCESS_TRANSFER_READ_BIT,
+ VK_IMAGE_LAYOUT_GENERAL, output_layers),
+ barrier(history, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT,
+ VK_IMAGE_LAYOUT_GENERAL, history_layers),
+ };
+ cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0,
+ {}, {}, after);
+ });
+ return true;
+}
+
void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& src,
const Region2D& dst_region, const Region2D& src_region,
Tegra::Engines::Fermi2D::Filter filter,
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h
index 43218d6ad3..25a6ce7c1e 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.h
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.h
@@ -87,6 +87,12 @@ public:
Tegra::Engines::Fermi2D::Filter filter,
Tegra::Engines::Fermi2D::Operation operation);
+ /// Stands in for a DLSS evaluation: temporal upscale of input_color into dst, accumulated with
+ /// a copy of dst's previous contents (the last output). False when dst cannot take it.
+ bool TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& input_color,
+ ImageView& motion_vectors, std::array<f32, 2> jitter,
+ std::array<f32, 2> motion_scale);
+
void CopyImage(Image& dst, Image& src, std::span<const VideoCommon::ImageCopy> copies);
void CopyImageMSAA(Image& dst, Image& src, std::span<const VideoCommon::ImageCopy> copies);
@@ -169,6 +175,15 @@ public:
StagingBufferPool& staging_buffer_pool;
BlitImageHelper& blit_image_helper;
RenderPassCache& render_pass_cache;
+
+ /// TemporalUpscale's history: its previous result, copied right after drawing it.
+ struct TemporalHistory {
+ vk::Image image;
+ vk::ImageView view;
+ VkFormat format{};
+ VkExtent2D extent{};
+ } temporal_history;
+
std::optional<ASTCDecoderPass> astc_decoder_pass;
std::optional<BcEncoderPass> bc_encoder_pass;
diff --git a/src/video_core/surface.cpp b/src/video_core/surface.cpp
index 66b9577cb5..ef4859ea3e 100644
--- a/src/video_core/surface.cpp
+++ b/src/video_core/surface.cpp
@@ -214,6 +214,8 @@ PixelFormat PixelFormatFromGPUPixelFormat(Service::android::PixelFormat format)
return PixelFormat::R5G6B5_UNORM;
case Service::android::PixelFormat::Bgra8888:
return PixelFormat::B8G8R8A8_UNORM;
+ case Service::android::PixelFormat::Rgba1010102:
+ return PixelFormat::A2B10G10R10_UNORM;
default:
UNIMPLEMENTED_MSG("Unimplemented format={}", format);
return PixelFormat::A8B8G8R8_UNORM;
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 33ca051482..32775b8719 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -957,6 +957,73 @@ bool TextureCache<P>::BlitImage(const Tegra::Engines::Fermi2D::Surface& dst,
}
template <class P>
+bool TextureCache<P>::TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst,
+ const Tegra::Engines::Fermi2D::Surface& src,
+ const Tegra::Engines::Fermi2D::Surface& motion_vectors,
+ std::array<f32, 2> jitter, std::array<f32, 2> motion_scale) {
+ if constexpr (FRAMEBUFFER_BLITS) {
+ return false;
+ } else {
+ const Tegra::Engines::Fermi2D::Config copy{
+ .operation = Tegra::Engines::Fermi2D::Operation::SrcCopy,
+ .filter = Tegra::Engines::Fermi2D::Filter::Bilinear,
+ .must_accelerate = true,
+ .dst_x0 = 0,
+ .dst_y0 = 0,
+ .dst_x1 = static_cast<s32>(dst.width),
+ .dst_y1 = static_cast<s32>(dst.height),
+ .src_x0 = 0,
+ .src_y0 = 0,
+ .src_x1 = static_cast<s32>(src.width),
+ .src_y1 = static_cast<s32>(src.height),
+ };
+ const auto result = GetBlitImages(dst, src, copy);
+ if (!result) {
+ return false;
+ }
+ const ImageInfo motion_info(motion_vectors);
+ has_deleted_images = false;
+ const ImageId motion_id =
+ FindOrInsertImage(motion_info, motion_vectors.Address(), RelaxedOptions{});
+ if (has_deleted_images || !motion_id) {
+ return false; // the colour and output ids may be stale; the caller blits instead
+ }
+ const ImageId dst_id = result->dst_id;
+ const ImageId src_id = result->src_id;
+ PrepareImage(src_id, false, false);
+ PrepareImage(motion_id, false, false);
+ PrepareImage(dst_id, true, false);
+ for (const ImageId id : {src_id, motion_id, dst_id}) {
+ if (True(slot_images[id].flags & ImageFlagBits::Rescaled) ||
+ slot_images[id].info.num_samples != 1) {
+ return false;
+ }
+ }
+ const auto view = [&](ImageId id, const Tegra::Engines::Fermi2D::Surface& surface,
+ PixelFormat format) -> std::optional<std::pair<FramebufferId, ImageViewId>> {
+ const std::optional base = slot_images[id].TryFindBase(surface.Address());
+ if (!base) {
+ return std::nullopt;
+ }
+ const SubresourceRange range{.base = *base, .extent = {1, 1}};
+ return RenderTargetFromImage(id, ImageViewInfo(ImageViewType::e2D, format, range));
+ };
+ const auto src_view = view(src_id, src, result->src_format);
+ const auto motion_view = view(motion_id, motion_vectors, motion_info.format);
+ const auto dst_view = view(dst_id, dst, result->dst_format);
+ if (!src_view || !motion_view || !dst_view) {
+ return false;
+ }
+ // After the views: creating them may reallocate the framebuffer slots.
+ Framebuffer* const dst_framebuffer = &slot_framebuffers[dst_view->first];
+ return runtime.TemporalUpscale(dst_framebuffer, slot_image_views[dst_view->second],
+ slot_image_views[src_view->second],
+ slot_image_views[motion_view->second], jitter,
+ motion_scale);
+ }
+}
+
+template <class P>
std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImageView(
const Tegra::FramebufferConfig& config, DAddr cpu_addr) {
// TODO: Properly implement this
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index 141e5b76d5..d4b0f597ae 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -276,6 +276,13 @@ public:
const Tegra::Engines::Fermi2D::Surface& src,
const Tegra::Engines::Fermi2D::Config& copy);
+ /// Stand-in for a DLSS evaluation: temporal upscale of src into dst (see the runtime's
+ /// TemporalUpscale). False when the images cannot take it; the caller then blits.
+ bool TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst,
+ const Tegra::Engines::Fermi2D::Surface& src,
+ const Tegra::Engines::Fermi2D::Surface& motion_vectors,
+ std::array<f32, 2> jitter, std::array<f32, 2> motion_scale);
+
/// Try to find a cached image view in the given CPU address
[[nodiscard]] std::pair<ImageView*, bool> TryFindFramebufferImageView(
const Tegra::FramebufferConfig& config, DAddr cpu_addr);