diff options
| author | kraibse <33869366+kraibse@users.noreply.github.com> | 2026-11-02 15:39:20 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-11-02 15:39:20 +0100 |
| commit | ce1b1de6f1beed36958368f89cda466937ff9716 (patch) | |
| tree | e3f6d670018b57004915e69d1fc741b4efecb325 /src | |
| parent | a407703f6d29a5ab73328b4ededb12cf29b2758d (diff) | |
| parent | c0cb3325e22756edb65c87d33ef298349ade0ad3 (diff) | |
Merge pull request #291 from Djihads80/nx2-experimental
[Update] Bring the mainline release up to speed with the performance updates from nx2-experimental
Diffstat (limited to 'src')
154 files changed, 5659 insertions, 940 deletions
diff --git a/src/audio_core/audio_render_manager.cpp b/src/audio_core/audio_render_manager.cpp index 8c4e74c4be..4034de1ec3 100644 --- a/src/audio_core/audio_render_manager.cpp +++ b/src/audio_core/audio_render_manager.cpp @@ -33,6 +33,8 @@ SystemManager& Manager::GetSystemManager() { Result Manager::GetWorkBufferSize(const AudioRendererParameterInternal& params, u64& out_count) const { if (!CheckValidRevision(params.revision)) { + LOG_ERROR(Service_Audio, "Audio renderer revision {:#x} ({}) is not supported", + params.revision, GetRevisionNum(params.revision)); return Service::Audio::ResultInvalidRevision; } diff --git a/src/audio_core/renderer/sink/circular_buffer_sink_info.cpp b/src/audio_core/renderer/sink/circular_buffer_sink_info.cpp index 0ede02b6b4..7e25413fe1 100644 --- a/src/audio_core/renderer/sink/circular_buffer_sink_info.cpp +++ b/src/audio_core/renderer/sink/circular_buffer_sink_info.cpp @@ -3,7 +3,6 @@ #include "audio_core/renderer/memory/pool_mapper.h" #include "audio_core/renderer/sink/circular_buffer_sink_info.h" -#include "audio_core/renderer/upsampler/upsampler_manager.h" namespace AudioCore::Renderer { @@ -17,13 +16,9 @@ CircularBufferSinkInfo::CircularBufferSinkInfo() { } void CircularBufferSinkInfo::CleanUp() { - auto state_{reinterpret_cast<DeviceState*>(state.data())}; - - if (state_->upsampler_info) { - state_->upsampler_info->manager->Free(state_->upsampler_info); - state_->upsampler_info = nullptr; - } - + // Only device sinks own an upsampler. This sink's state is a CircularBufferState, whose + // first words are buffer positions: reading them as DeviceState::upsampler_info freed a + // garbage pointer when a game changed or removed its circular buffer sink. parameter.fill(0); type = Type::Invalid; } diff --git a/src/common/fiber.cpp b/src/common/fiber.cpp index 08558a0026..613ac97e22 100644 --- a/src/common/fiber.cpp +++ b/src/common/fiber.cpp @@ -25,8 +25,10 @@ struct Fiber::FiberImpl { FiberImpl() {} u32 canary_1 = CANARY_VALUE; - std::array<u8, DEFAULT_STACK_SIZE> stack; - std::array<u8, DEFAULT_STACK_SIZE> rewind_stack; + // Left uninitialized: the host gives a stack's pages memory only as they are used. Zero-filled + // std::arrays (and a second stack nothing switched to) made every fiber, one per guest thread, + // hold 4 MiB: ~750 MiB for title 1's ~190 threads. + std::unique_ptr<u8[]> stack; u32 canary_2 = CANARY_VALUE; boost::context::detail::fcontext_t context{}; @@ -38,7 +40,6 @@ struct Fiber::FiberImpl { std::shared_ptr<Fiber> previous_fiber; u8* stack_limit = nullptr; - u8* rewind_stack_limit = nullptr; bool is_thread_fiber = false; bool released = false; }; @@ -49,10 +50,10 @@ void Fiber::SetRewindPoint(std::function<void()>&& rewind_func) { Fiber::Fiber(std::function<void()>&& entry_point_func) : impl{std::make_unique<FiberImpl>()} { impl->entry_point = std::move(entry_point_func); - impl->stack_limit = impl->stack.data(); - impl->rewind_stack_limit = impl->rewind_stack.data(); + impl->stack.reset(new u8[DEFAULT_STACK_SIZE]); + impl->stack_limit = impl->stack.get(); u8* stack_base = impl->stack_limit + DEFAULT_STACK_SIZE; - impl->context = boost::context::detail::make_fcontext(stack_base, impl->stack.size(), [](boost::context::detail::transfer_t transfer) -> void { + impl->context = boost::context::detail::make_fcontext(stack_base, DEFAULT_STACK_SIZE, [](boost::context::detail::transfer_t transfer) -> void { auto* fiber = static_cast<Fiber*>(transfer.data); ASSERT(fiber && fiber->impl && fiber->impl->previous_fiber && fiber->impl->previous_fiber->impl); ASSERT(fiber->impl->canary_1 == CANARY_VALUE); diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 147eba8477..2ef1982cbc 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -57,6 +57,7 @@ #include "common/alignment.h" #include "common/assert.h" +#include "common/div_ceil.h" #include "common/free_region_manager.h" #include "common/host_memory.h" #include "common/logging.h" @@ -716,7 +717,9 @@ private: HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) : backing_size(backing_size_) , virtual_size(virtual_size_) + , num_granules(Common::DivCeil(backing_size_, size_t{1} << GranuleShift)) { + granules = std::make_unique<std::atomic<u8>[]>(num_granules); #if defined(__OPENORBIS__) || defined(__managarm__) LOG_WARNING(HW_Memory, "Platform doesn't support fastmem"); backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); @@ -799,21 +802,98 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, MemoryPermission #endif } +void HostMemory::HintCold(size_t physical_offset, size_t length, + std::optional<size_t> virtual_offset) const { + if (length == 0 || backing_base == nullptr) { + return; + } + const auto hint = [length](u8* address) { +#ifdef _WIN32 + // Unlocking pages that are not locked takes them out of the working set. + VirtualUnlock(address, length); +#elif defined(__linux__) && defined(MADV_PAGEOUT) + madvise(address, length, MADV_PAGEOUT); +#else + static_cast<void>(address); +#endif + }; + hint(backing_base + physical_offset); + if (virtual_offset && virtual_base != nullptr && *virtual_offset + length <= virtual_size) { + hint(virtual_base + *virtual_offset); + } +} + void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value) { + if (length == 0) { + return; + } + MarkUsed(physical_offset, length); + if (fill_value != 0) { + std::memset(backing_base + physical_offset, fill_value, length); + MarkDirty(physical_offset, length); + return; + } + // Fresh backing memory is zero and the host gives it no RAM until it's touched: clear only + // the granules that may hold data (title 1's heap made the host commit every page of it). + // A granule cleared in full is zero again. #ifdef __linux__ - const size_t first = AlignUp(physical_offset, PageAlignment); - const size_t last = AlignDown(physical_offset + length, PageAlignment); + // Handing a whole granule back to the kernel frees its RAM; a memset would commit it. static const bool punch = [] { const char* value = std::getenv("EDEN_PUNCH_HOLE"); return value == nullptr || value[0] != '0'; }(); - if (punch && fill_value == 0 && impl && last > first && impl->PunchHole(first, last - first)) { - std::memset(backing_base + physical_offset, 0, first - physical_offset); - std::memset(backing_base + last, 0, physical_offset + length - last); +#endif + const size_t end = physical_offset + length; + for (size_t offset = physical_offset; offset < end;) { + const size_t index = offset >> GranuleShift; + const size_t granule_begin = index << GranuleShift; + const size_t granule_end = granule_begin + (size_t{1} << GranuleShift); + const size_t clear_end = (std::min)(granule_end, end); + if ((granules[index].load(std::memory_order_acquire) & GranuleDirty) != 0) { + const bool whole = offset == granule_begin && clear_end == granule_end; + bool cleared = false; +#ifdef __linux__ + cleared = whole && punch && impl && impl->PunchHole(granule_begin, granule_end - granule_begin); +#endif + if (!cleared) { + std::memset(backing_base + offset, 0, clear_end - offset); + } + if (whole) { + granules[index].fetch_and(static_cast<u8>(~GranuleDirty), std::memory_order_release); + } + } + offset = clear_end; + } +} + +void HostMemory::MarkUsed(size_t physical_offset, size_t length) { + if (length == 0) { return; } -#endif - std::memset(backing_base + physical_offset, fill_value, length); + const size_t last = (physical_offset + length - 1) >> GranuleShift; + for (size_t index = physical_offset >> GranuleShift; index <= last; ++index) { + if ((granules[index].load(std::memory_order_relaxed) & GranuleUsed) == 0) { + granules[index].fetch_or(GranuleUsed, std::memory_order_release); + } + } +} + +void HostMemory::MarkDirty(size_t physical_offset, size_t length) { + if (length == 0) { + return; + } + const size_t last = (physical_offset + length - 1) >> GranuleShift; + for (size_t index = physical_offset >> GranuleShift; index <= last; ++index) { + granules[index].fetch_or(GranuleDirty, std::memory_order_release); + } +} + +void HostMemory::ForgetContents() { + for (size_t index = 0; index < num_granules; ++index) { + if ((granules[index].load(std::memory_order_relaxed) & GranuleUsed) != 0) { + granules[index].fetch_or(GranuleDirty, std::memory_order_release); + } + } } void HostMemory::EnableDirectMappedAddress() { diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 8747a1850f..84315091c5 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -6,6 +6,7 @@ #pragma once +#include <atomic> #include <memory> #include <optional> #include "common/common_funcs.h" @@ -52,8 +53,24 @@ public: void EnableDirectMappedAddress(); + /// Clears guest memory being handed out. Memory still zero since it was created (or since its + /// last full clear) is skipped: clearing it only made the host give every page real memory. void ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value); + /// Memory written once and read again rarely: its pages leave the working set (backing and, + /// when given, its view at virtual_offset), so the OS compresses or pages them out first when + /// it needs RAM. The contents stay; using the pages brings them back. + void HintCold(size_t physical_offset, size_t length, + std::optional<size_t> virtual_offset) const; + /// Pages the kernel hands out (cleared or not). + void MarkUsed(size_t physical_offset, size_t length); + + /// Pages given back to the kernel: their contents are no longer known to be zero. + void MarkDirty(size_t physical_offset, size_t length); + + /// A new kernel on this memory: everything a previous one used may hold data. + void ForgetContents(); + [[nodiscard]] u8* BackingBasePointer() noexcept { return backing_base; } @@ -88,6 +105,12 @@ private: u8* backing_base{}; u8* virtual_base{}; size_t virtual_base_offset{}; + + static constexpr size_t GranuleShift = 16; ///< 64 KiB of backing per state byte + static constexpr u8 GranuleUsed = 1 << 0; ///< Handed out at least once + static constexpr u8 GranuleDirty = 1 << 1; ///< May hold non-zero data + std::unique_ptr<std::atomic<u8>[]> granules; + size_t num_granules{}; // Windows requires it for kernels whom lack proper support for some functions! bool fallback_buffer{false}; }; diff --git a/src/common/settings.h b/src/common/settings.h index 97103f457d..74e2006d15 100644 --- a/src/common/settings.h +++ b/src/common/settings.h @@ -573,8 +573,10 @@ struct Values { true, true}; + // BC3 by default: GPUs without ASTC otherwise keep ASTC textures as RGBA8, four times the + // VRAM (Mario Kart World: 7.3-7.8 GB of an 8 GB card, the rest paged into system RAM). SwitchableSetting<AstcRecompression, true> astc_recompression{linkage, - AstcRecompression::Uncompressed, + AstcRecompression::Bc3, "astc_recompression", Category::RendererAdvanced}; diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp index b96340999d..34aba73d78 100644 --- a/src/common/sparse_large_vector.cpp +++ b/src/common/sparse_large_vector.cpp @@ -39,12 +39,12 @@ static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { std::lock_guard lock(vector_regions_mutex); for (auto region: vector_regions) { auto addr_shifted = exception_addr >> HostPageBits; - if (region.first <= addr_shifted && addr_shifted <= region.second) { + if (region.first <= addr_shifted && addr_shifted < region.second) { addr = addr_shifted; } // Page-boundary accesses - if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ <= region.second) { + if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ < region.second) { addr2 = addr_; } @@ -164,7 +164,8 @@ void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { return; #ifdef _WIN32 std::lock_guard lock(vector_regions_mutex); - std::erase_if(vector_regions, [base](const auto& r) {return r.first == reinterpret_cast<u64>(base); }); + // Regions are kept in host pages. + std::erase_if(vector_regions, [base](const auto& r) {return r.first == reinterpret_cast<u64>(base) >> HostPageBits; }); ASSERT(VirtualFree(base, 0, MEM_RELEASE)); #else ASSERT(munmap(base, size) == 0); diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index fd45642876..eeb3202d12 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -531,6 +531,7 @@ add_library(core STATIC hle/service/audio/audio_out.h hle/service/audio/audio_renderer_manager.cpp hle/service/audio/audio_renderer_manager.h + hle/service/audio/nx2_audio_device_manager.h hle/service/audio/nx2_audio_user_client.h hle/service/audio/audio_renderer.cpp hle/service/audio/audio_renderer.h diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.cpp b/src/core/arm/dynarmic/arm_dynarmic_32.cpp index a59aa3cac3..3a875493c6 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_32.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_32.cpp @@ -416,6 +416,10 @@ void ArmDynarmic32::ClearInstructionCache() { } void ArmDynarmic32::InvalidateCacheRange(u64 addr, std::size_t size) { + // Drop the cached code page too: blocks recompiled after the invalidation would + // otherwise still decode the old instructions (e.g. a gdbstub breakpoint written + // into or removed from that page). + m_cb->last_code_addr = u64(-1); m_jit->InvalidateCacheRange(static_cast<u32>(addr), size); } diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index e65e38dd9e..c5be4d2520 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -481,6 +481,10 @@ void ArmDynarmic64::ClearInstructionCache() { } void ArmDynarmic64::InvalidateCacheRange(u64 addr, std::size_t size) { + // Drop the cached code page too: blocks recompiled after the invalidation would + // otherwise still decode the old instructions (e.g. a gdbstub breakpoint written + // into or removed from that page). + m_cb->last_code_addr = u64(-1); m_jit->InvalidateCacheRange(addr, size); } diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 27ff342213..d275bdbf2d 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -18,6 +18,7 @@ #include "common/common_types.h" #include "common/range_mutex.h" #include "common/scratch_buffer.h" +#include "common/thread_worker.h" #include "common/sparse_large_vector.h" namespace Core { @@ -53,6 +54,10 @@ public: void BindInterface(DeviceInterface* device_inter); + /// Marks the memory behind [address, address + size) cold (Common::HostMemory::HintCold), + /// from a low-priority thread: data the host GPU has its own copy of, e.g. uploaded textures. + void HintCold(DAddr address, size_t size); + DAddr Allocate(size_t size); void AllocateFixed(DAddr start, size_t size); void Free(DAddr start, size_t size); @@ -197,7 +202,9 @@ private: std::unique_ptr<DeviceMemoryManagerAllocator<Traits>> impl; const uintptr_t physical_base; + const DeviceMemory* device_memory_ptr; DeviceInterface* device_inter; + std::unique_ptr<Common::ThreadWorker> cold_hint_worker; struct TrackedEntry { VAddr cpu_backing_address; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 83faa1b3f3..3523e3cf1c 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -171,6 +171,7 @@ struct DeviceMemoryManagerAllocator { template <typename Traits> DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memory_) : physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())} + , device_memory_ptr{&device_memory_} , device_inter{nullptr} , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS)) , tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS) @@ -183,6 +184,49 @@ template <typename Traits> DeviceMemoryManager<Traits>::~DeviceMemoryManager() = default; template <typename Traits> +void DeviceMemoryManager<Traits>::HintCold(DAddr address, size_t size) { + struct Run { + u64 physical_offset; + VAddr virtual_address; ///< 0: no CPU mapping known + u64 size; + }; + std::vector<Run> runs; + const size_t page_end = Common::DivCeil(address + size, Memory::YUZU_PAGESIZE); + for (size_t page = address >> Memory::YUZU_PAGEBITS; page < page_end; ++page) { + const u32 compressed = tracked_entries[page].compressed_physical_ptr; + if (compressed == 0) { + continue; + } + const u64 physical = u64{compressed - 1} << Memory::YUZU_PAGEBITS; + const VAddr vaddr = ExtractCPUBacking(page).second; + if (!runs.empty()) { + Run& last = runs.back(); + const bool contiguous_view = + last.virtual_address == 0 ? vaddr == 0 : last.virtual_address + last.size == vaddr; + if (last.physical_offset + last.size == physical && contiguous_view) { + last.size += Memory::YUZU_PAGESIZE; + continue; + } + } + runs.push_back({physical, vaddr, Memory::YUZU_PAGESIZE}); + } + if (runs.empty()) { + return; + } + if (!cold_hint_worker) { + cold_hint_worker = std::make_unique<Common::ThreadWorker>(1, "DeviceMemory:ColdHint"); + } + cold_hint_worker->QueueWork([this, runs = std::move(runs)] { + for (const Run& run : runs) { + device_memory_ptr->buffer.HintCold( + run.physical_offset, run.size, + run.virtual_address != 0 ? std::optional<size_t>{run.virtual_address} + : std::nullopt); + } + }); +} + +template <typename Traits> void DeviceMemoryManager<Traits>::BindInterface(DeviceInterface* device_inter_) { device_inter = device_inter_; } diff --git a/src/core/file_sys/fsmitm_romfsbuild.cpp b/src/core/file_sys/fsmitm_romfsbuild.cpp index 8807bbd0f7..d34e24f177 100644 --- a/src/core/file_sys/fsmitm_romfsbuild.cpp +++ b/src/core/file_sys/fsmitm_romfsbuild.cpp @@ -55,15 +55,18 @@ static_assert(sizeof(RomFSFileEntry) == 0x20, "RomFSFileEntry has incorrect size struct RomFSBuildFileContext; +// The context's directories and files own the nodes; the links between them don't, or every +// parent and child would own each other and no build would ever be freed (title 1's loose RomFS: +// 206k files, ~50 MiB per build). struct RomFSBuildDirectoryContext { std::string path; u32 cur_path_ofs = 0; u32 path_len = 0; u32 entry_offset = 0; - std::shared_ptr<RomFSBuildDirectoryContext> parent; - std::shared_ptr<RomFSBuildDirectoryContext> child; - std::shared_ptr<RomFSBuildDirectoryContext> sibling; - std::shared_ptr<RomFSBuildFileContext> file; + RomFSBuildDirectoryContext* parent = nullptr; + RomFSBuildDirectoryContext* child = nullptr; + RomFSBuildDirectoryContext* sibling = nullptr; + RomFSBuildFileContext* file = nullptr; }; struct RomFSBuildFileContext { @@ -73,8 +76,8 @@ struct RomFSBuildFileContext { u32 entry_offset = 0; u64 offset = 0; u64 size = 0; - std::shared_ptr<RomFSBuildDirectoryContext> parent; - std::shared_ptr<RomFSBuildFileContext> sibling; + RomFSBuildDirectoryContext* parent = nullptr; + RomFSBuildFileContext* sibling = nullptr; VirtualFile source; }; @@ -168,7 +171,7 @@ bool RomFSBuildContext::AddDirectory(std::shared_ptr<RomFSBuildDirectoryContext> num_dirs++; dir_table_size += sizeof(RomFSDirectoryEntry) + Common::AlignUp(dir_ctx->path_len - dir_ctx->cur_path_ofs, 4); - dir_ctx->parent = std::move(parent_dir_ctx); + dir_ctx->parent = parent_dir_ctx.get(); directories.emplace_back(std::move(dir_ctx)); return true; @@ -180,7 +183,7 @@ bool RomFSBuildContext::AddFile(std::shared_ptr<RomFSBuildDirectoryContext> pare num_files++; file_table_size += sizeof(RomFSFileEntry) + Common::AlignUp(file_ctx->path_len - file_ctx->cur_path_ofs, 4); - file_ctx->parent = std::move(parent_dir_ctx); + file_ctx->parent = parent_dir_ctx.get(); files.emplace_back(std::move(file_ctx)); return true; @@ -245,11 +248,11 @@ std::vector<std::pair<u64, VirtualFile>> RomFSBuildContext::Build() { Common::AlignUp(cur_file->path_len - cur_file->cur_path_ofs, 4)); prev_file = cur_file; } - // Assign deferred parent/sibling ownership. + // Assign deferred parent/sibling links. for (auto it = files.rbegin(); it != files.rend(); ++it) { auto& cur_file = *it; cur_file->sibling = cur_file->parent->file; - cur_file->parent->file = cur_file; + cur_file->parent->file = cur_file.get(); } // Determine directory offsets. @@ -260,11 +263,11 @@ std::vector<std::pair<u64, VirtualFile>> RomFSBuildContext::Build() { static_cast<u32>(sizeof(RomFSDirectoryEntry) + Common::AlignUp(cur_dir->path_len - cur_dir->cur_path_ofs, 4)); } - // Assign deferred parent/sibling ownership. + // Assign deferred parent/sibling links. for (auto it = directories.rbegin(); (*it) != root; ++it) { auto& cur_dir = *it; cur_dir->sibling = cur_dir->parent->child; - cur_dir->parent->child = cur_dir; + cur_dir->parent->child = cur_dir.get(); } // Create output map. diff --git a/src/core/file_sys/system_archive/system_archive.cpp b/src/core/file_sys/system_archive/system_archive.cpp index 1743d01b12..c3931b6f62 100644 --- a/src/core/file_sys/system_archive/system_archive.cpp +++ b/src/core/file_sys/system_archive/system_archive.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> + #include "common/logging.h" #include "core/file_sys/romfs.h" #include "core/file_sys/system_archive/mii_model.h" @@ -12,6 +14,7 @@ #include "core/file_sys/system_archive/system_archive.h" #include "core/file_sys/system_archive/system_version.h" #include "core/file_sys/system_archive/time_zone_binary.h" +#include "core/file_sys/vfs/vfs_vector.h" namespace FileSys::SystemArchive { @@ -76,6 +79,22 @@ constexpr inline SystemArchiveDescriptor GetSystemArchive(u64 title_id) { VirtualFile SynthesizeSystemArchive(const u64 title_id) { auto const desc = GetSystemArchive(title_id); + if (desc.name == nullptr) { + // Not a system archive Eden knows (fmt throws on a null string, ending the process). + // Switch 2 system data (ids 04...) is not installed with Switch 1 firmware, and titles + // abort when it cannot be mounted (Nintendo Switch Sports Resort, 0400000000000849): + // serve an empty archive. EDEN_NX2_EMPTY_SYSTEM_DATA=0 fails the request instead. + static const bool empty_nx2 = [] { + const char* setting = std::getenv("EDEN_NX2_EMPTY_SYSTEM_DATA"); + return setting == nullptr || setting[0] != '0'; + }(); + if (empty_nx2 && (title_id >> 56) == 0x04) { + LOG_WARNING(Service_FS, "Empty archive for Switch 2 system data {:#016x}", title_id); + return CreateRomFS(std::make_shared<VectorVfsDirectory>()); + } + LOG_WARNING(Service_FS, "No system archive {:#016x} to synthesize", title_id); + return nullptr; + } LOG_INFO(Service_FS, "Synthesizing system archive '{}' ({:#016x}).", desc.name, title_id); if (desc.supplier != nullptr) { if (auto const dir = desc.supplier(); dir != nullptr) { diff --git a/src/core/file_sys/vfs/vfs_real.cpp b/src/core/file_sys/vfs/vfs_real.cpp index 5b5b54017c..7cc5315a70 100644 --- a/src/core/file_sys/vfs/vfs_real.cpp +++ b/src/core/file_sys/vfs/vfs_real.cpp @@ -114,7 +114,6 @@ VfsEntryType RealVfsFilesystem::GetEntryType(std::string_view path_) const { } VirtualFile RealVfsFilesystem::OpenFileFromEntry(std::string_view path_, std::optional<u64> size, - std::optional<std::string> parent_path, OpenMode perms) { const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault); std::scoped_lock lk{list_lock}; @@ -132,14 +131,14 @@ VirtualFile RealVfsFilesystem::OpenFileFromEntry(std::string_view path_, std::op auto reference = std::make_unique<FileReference>(); this->InsertReferenceIntoListLocked(*reference); - auto file = std::make_shared<RealVfsFile>(*this, std::move(reference), path, perms, size, std::move(parent_path)); + auto file = std::make_shared<RealVfsFile>(*this, std::move(reference), path, perms, size); cache[path] = file; return file; } VirtualFile RealVfsFilesystem::OpenFile(std::string_view path_, OpenMode perms) { - return OpenFileFromEntry(path_, {}, {}, perms); + return OpenFileFromEntry(path_, {}, perms); } VirtualFile RealVfsFilesystem::CreateFile(std::string_view path_, OpenMode perms) { @@ -257,7 +256,8 @@ std::unique_lock<std::mutex> RealVfsFilesystem::RefreshReference(const std::stri return lk; } -void RealVfsFilesystem::DropReference(std::unique_ptr<FileReference>&& reference) { +void RealVfsFilesystem::DropReference(std::unique_ptr<FileReference>&& reference, + const std::string& path) { if (in_dtor) return; @@ -266,6 +266,11 @@ void RealVfsFilesystem::DropReference(std::unique_ptr<FileReference>&& reference // Remove from list. this->RemoveReferenceFromListLocked(*reference); + // Forget the path, unless it was opened again since this file's last reference went away. + if (const auto it = cache.find(path); it != cache.end() && it->second.expired()) { + cache.erase(it); + } + // Close the file. if (reference->file) { reference->file.reset(); @@ -309,14 +314,11 @@ void RealVfsFilesystem::RemoveReferenceFromListLocked(FileReference& reference) } RealVfsFile::RealVfsFile(RealVfsFilesystem& base_, std::unique_ptr<FileReference> reference_, - const std::string& path_, OpenMode perms_, std::optional<u64> size_, - std::optional<std::string> parent_path_) - : base(base_), reference(std::move(reference_)), path(path_), - parent_path(parent_path_ ? std::move(*parent_path_) : FS::GetParentPath(path_)), - path_components(FS::SplitPathComponentsCopy(path_)), size(size_), perms(perms_) {} + const std::string& path_, OpenMode perms_, std::optional<u64> size_) + : base(base_), reference(std::move(reference_)), path(path_), size(size_), perms(perms_) {} RealVfsFile::~RealVfsFile() { - base.DropReference(std::move(reference)); + base.DropReference(std::move(reference), path); } std::string RealVfsFile::GetName() const { @@ -325,7 +327,14 @@ std::string RealVfsFile::GetName() const { return FS::Android::GetFilename(path); } #endif - return path_components.empty() ? "" : std::string(path_components.back()); + // The last path component. + const auto end = path.find_last_not_of("\\/"); + if (end == std::string::npos) { + return ""; + } + const auto separator = path.find_last_of("\\/", end); + const auto begin = separator == std::string::npos ? 0 : separator + 1; + return path.substr(begin, end + 1 - begin); } std::size_t RealVfsFile::GetSize() const { @@ -343,7 +352,7 @@ bool RealVfsFile::Resize(std::size_t new_size) { } VirtualDir RealVfsFile::GetContainingDirectory() const { - return base.OpenDirectory(parent_path, perms); + return base.OpenDirectory(FS::GetParentPath(path), perms); } bool RealVfsFile::IsWritable() const { @@ -372,7 +381,7 @@ std::size_t RealVfsFile::Write(const u8* data, std::size_t length, std::size_t o } bool RealVfsFile::Rename(std::string_view name) { - return base.MoveFile(path, parent_path + '/' + std::string(name)) != nullptr; + return base.MoveFile(path, FS::GetParentPath(path) + '/' + std::string(name)) != nullptr; } // TODO(DarkLordZach): MSVC would not let me combine the following two functions using 'if @@ -390,7 +399,7 @@ std::vector<VirtualFile> RealVfsDirectory::IterateEntries<RealVfsFile, VfsFile>( &out](const std::filesystem::directory_entry& entry) { const auto full_path_string = FS::PathToUTF8String(entry.path()); - out.emplace_back(base.OpenFileFromEntry(full_path_string, entry.file_size(), path, perms)); + out.emplace_back(base.OpenFileFromEntry(full_path_string, entry.file_size(), perms)); return true; }; diff --git a/src/core/file_sys/vfs/vfs_real.h b/src/core/file_sys/vfs/vfs_real.h index a97f7d32ea..2df125b7eb 100644 --- a/src/core/file_sys/vfs/vfs_real.h +++ b/src/core/file_sys/vfs/vfs_real.h @@ -62,12 +62,11 @@ private: friend class RealVfsFile; std::unique_lock<std::mutex> RefreshReference(const std::string& path, OpenMode perms, FileReference& reference); - void DropReference(std::unique_ptr<FileReference>&& reference); + void DropReference(std::unique_ptr<FileReference>&& reference, const std::string& path); private: friend class RealVfsDirectory; VirtualFile OpenFileFromEntry(std::string_view path, std::optional<u64> size, - std::optional<std::string> parent_path, OpenMode perms = OpenMode::Read); private: @@ -84,7 +83,7 @@ class RealVfsFile : public VfsFile { public: RealVfsFile(RealVfsFilesystem& base, std::unique_ptr<FileReference> reference, const std::string& path, OpenMode perms = OpenMode::Read, - std::optional<u64> size = {}, std::optional<std::string> parent_path = {}); + std::optional<u64> size = {}); ~RealVfsFile() override; std::string GetName() const override; @@ -101,9 +100,8 @@ private: RealVfsFilesystem& base; std::unique_ptr<FileReference> reference; + // Only the path: a loose RomFS keeps one of these per file (title 1: 206k). std::string path; - std::string parent_path; - std::vector<std::string> path_components; std::optional<u64> size; OpenMode perms; }; diff --git a/src/core/hle/kernel/board/nintendo/nx/k_system_control.h b/src/core/hle/kernel/board/nintendo/nx/k_system_control.h index cc5572bde8..9aa527adb1 100644 --- a/src/core/hle/kernel/board/nintendo/nx/k_system_control.h +++ b/src/core/hle/kernel/board/nintendo/nx/k_system_control.h @@ -23,7 +23,10 @@ public: // Extra DRAM on the 12 GB layout for memory the Switch 2 graphics server keeps itself (nx2:gpu // 214 GPU-only regions). It sits in the non-secure system pool, outside the application pool. - static constexpr size_t Nx2GraphicsServerMemorySize = 8ULL * 1024 * 1024 * 1024; // 8_GiB + // GPU-only regions get memory only where guest memory is written and sparse regions take + // their tiles from the game's own memory (216), so much less of it is used (title 1: ~1.1 GiB, + // all of it at boot); all of it counts against the host's commit limit from boot. + static constexpr size_t Nx2GraphicsServerMemorySize = 2ULL * 1024 * 1024 * 1024; // 2_GiB public: class Init { diff --git a/src/core/hle/kernel/k_memory_manager.cpp b/src/core/hle/kernel/k_memory_manager.cpp index 743e290541..bed3dd819d 100644 --- a/src/core/hle/kernel/k_memory_manager.cpp +++ b/src/core/hle/kernel/k_memory_manager.cpp @@ -48,6 +48,8 @@ KMemoryManager::KMemoryManager(Core::System& system) } {} void KMemoryManager::Initialize(KVirtualAddress management_region, size_t management_region_size) { + // Memory a previous kernel handed out may still hold its data. + m_system.DeviceMemory().buffer.ForgetContents(); // Clear the management region to zero. const KVirtualAddress management_region_end = management_region + management_region_size; @@ -242,10 +244,21 @@ KPhysicalAddress KMemoryManager::AllocateAndOpenContinuous(size_t num_pages, siz // Open the first reference to the pages. chosen_manager->OpenFirst(allocated_block, num_pages); + this->MarkUsed(allocated_block, num_pages); return allocated_block; } +void KMemoryManager::MarkUsed(KPhysicalAddress address, size_t num_pages) { + m_system.DeviceMemory().buffer.MarkUsed(GetInteger(address) - Core::DramMemoryMap::Base, + num_pages * PageSize); +} + +void KMemoryManager::MarkDirty(KPhysicalAddress address, size_t num_pages) { + m_system.DeviceMemory().buffer.MarkDirty(GetInteger(address) - Core::DramMemoryMap::Base, + num_pages * PageSize); +} + Result KMemoryManager::AllocatePageGroupImpl(KPageGroup* out, size_t num_pages, Pool pool, Direction dir, bool unoptimized, bool random) { // Choose a heap based on our page size request. @@ -297,6 +310,10 @@ Result KMemoryManager::AllocatePageGroupImpl(KPageGroup* out, size_t num_pages, // Only succeed if we allocated as many pages as we wanted. R_UNLESS(num_pages == 0, ResultOutOfMemory); + for (const auto& block : *out) { + this->MarkUsed(block.GetAddress(), block.GetNumPages()); + } + // We succeeded! R_SUCCEED(); } @@ -525,9 +542,10 @@ bool KMemoryManager::Impl::ProcessOptimizedAllocation(KernelCore& kernel, KPhysi // If not, it's new. any_new = true; - // Fill the page. - auto* ptr = device_memory.GetPointer<u8>(m_heap.GetAddress()); - std::memset(ptr + offset * PageSize, fill_pattern, PageSize); + // Fill the page (a zero fill skips pages that are still zero). + device_memory.buffer.ClearBackingRegion( + GetInteger(m_heap.GetAddress()) + offset * PageSize - Core::DramMemoryMap::Base, + PageSize, fill_pattern); } offset++; diff --git a/src/core/hle/kernel/k_memory_manager.h b/src/core/hle/kernel/k_memory_manager.h index 2f4ae7f04f..66b97fef8a 100644 --- a/src/core/hle/kernel/k_memory_manager.h +++ b/src/core/hle/kernel/k_memory_manager.h @@ -100,6 +100,9 @@ public: } void Close(KPhysicalAddress address, size_t num_pages) { + // Pages that may be freed here may hold data: the next allocation clears them. + this->MarkDirty(address, num_pages); + // Repeatedly close references until we've done so for all pages. while (num_pages) { auto& manager = this->GetManager(address); @@ -333,6 +336,9 @@ private: }; private: + void MarkUsed(KPhysicalAddress address, size_t num_pages); + void MarkDirty(KPhysicalAddress address, size_t num_pages); + Impl& GetManager(KPhysicalAddress address) { return m_managers[m_memory_layout.GetPhysicalLinearRegion(address).GetAttributes()]; } diff --git a/src/core/hle/kernel/k_shared_memory.cpp b/src/core/hle/kernel/k_shared_memory.cpp index 61f2872560..cb8877b8b2 100644 --- a/src/core/hle/kernel/k_shared_memory.cpp +++ b/src/core/hle/kernel/k_shared_memory.cpp @@ -59,9 +59,10 @@ Result KSharedMemory::Initialize(KernelCore& kernel, Core::DeviceMemory& device_ // Mark initialized. m_is_initialized = true; - // Clear all pages in the memory. + // Clear all pages in the memory (pages still zero are skipped). for (const auto& block : *m_page_group) { - std::memset(m_device_memory->GetPointer<void>(block.GetAddress()), 0, block.GetSize()); + m_device_memory->buffer.ClearBackingRegion( + GetInteger(block.GetAddress()) - Core::DramMemoryMap::Base, block.GetSize(), 0); } R_SUCCEED(); diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp index 53d964ecbc..925eafc472 100644 --- a/src/core/hle/kernel/svc.cpp +++ b/src/core/hle/kernel/svc.cpp @@ -14,6 +14,8 @@ #include "core/core.h" #include "core/hle/kernel/k_process.h" #include "core/hle/kernel/svc.h" +#include "core/hle/kernel/k_readable_event.h" +#include "core/hle/service/kernel_helpers.h" namespace Kernel::Svc { @@ -2522,6 +2524,21 @@ void Call(Core::System& system, u32 imm) { LOG_INFO(Kernel_SVC, "svc {:#04x} in t{} [0]={:#x} [1]={:#x} [2]={:#x} [3]={:#x}", imm, GetCurrentThread(kernel).GetThreadId(), GetArg64(args, 0), GetArg64(args, 1), GetArg64(args, 2), GetArg64(args, 3)); + if (imm == 0x18) { + // WaitSynchronization: the handles, with the names services gave their events. + std::string list; + const s32 count = static_cast<s32>(GetArg64(args, 2)); + for (s32 i = 0; i < std::min(count, 8); ++i) { + const u32 handle = process.GetMemory().Read32(GetArg64(args, 1) + 4 * i); + KScopedAutoObject event = + process.GetHandleTable().GetObject<KReadableEvent>(kernel, handle); + list += fmt::format(" {:#x}{}", handle, + event.IsNotNull() + ? "=" + Service::KernelHelpers::DebugEventName(event.GetPointerUnsafe()) + : std::string{}); + } + LOG_INFO(Kernel_SVC, "svc 0x18 t{} waits on{}", GetCurrentThread(kernel).GetThreadId(), list); + } } if (imm == 0x80) { // Switch 2 only: svcSetGpuMemoryAttribute(address, size, mask, attr). nn::gpu::sdk and diff --git a/src/core/hle/service/acc/acc.cpp b/src/core/hle/service/acc/acc.cpp index d68b6bde0d..772da97ef6 100644 --- a/src/core/hle/service/acc/acc.cpp +++ b/src/core/hle/service/acc/acc.cpp @@ -696,9 +696,9 @@ public: {4, &IManagerForApplication::LoadIdTokenCache, "LoadIdTokenCache"}, {130, &IManagerForApplication::GetNintendoAccountUserResourceCacheForApplication, "GetNintendoAccountUserResourceCacheForApplication"}, {136, &IManagerForApplication::GetNintendoAccountUserResourceCacheForApplication, "GetNintendoAccountUserResourceCache"}, // 19.0.0+ - {150, nullptr, "CreateAuthorizationRequest"}, + {150, &IManagerForApplication::StubReplyInterface, "CreateAuthorizationRequest"}, {160, &IManagerForApplication::StoreOpenContext, "StoreOpenContext"}, - {170, nullptr, "LoadNetworkServiceLicenseKindAsync"}, + {170, &IManagerForApplication::StubReplyInterface, "LoadNetworkServiceLicenseKindAsync"}, }; // clang-format on diff --git a/src/core/hle/service/am/service/application_functions.cpp b/src/core/hle/service/am/service/application_functions.cpp index 37dfbfa344..c63d29b4e9 100644 --- a/src/core/hle/service/am/service/application_functions.cpp +++ b/src/core/hle/service/am/service/application_functions.cpp @@ -58,18 +58,19 @@ IApplicationFunctions::IApplicationFunctions(Core::System& system_, std::shared_ : ServiceFramework{system_, "IApplicationFunctions"}, m_applet{std::move(applet)} { // clang-format off static const FunctionInfo functions[] = { + {0, &IApplicationFunctions::StubReplyZeroed, "Unknown0"}, {1, D<&IApplicationFunctions::PopLaunchParameter>, "PopLaunchParameter"}, - {10, nullptr, "CreateApplicationAndPushAndRequestToStart"}, - {11, nullptr, "CreateApplicationAndPushAndRequestToStartForQuest"}, + {10, &IApplicationFunctions::StubReplyZeroed, "CreateApplicationAndPushAndRequestToStart"}, + {11, &IApplicationFunctions::StubReplyZeroed, "CreateApplicationAndPushAndRequestToStartForQuest"}, {12, D<&IApplicationFunctions::CreateApplicationAndRequestToStart>, "CreateApplicationAndRequestToStart"}, - {13, nullptr, "CreateApplicationAndRequestToStartForQuest"}, - {14, nullptr, "CreateApplicationWithAttributeAndPushAndRequestToStartForQuest"}, - {15, nullptr, "CreateApplicationWithAttributeAndRequestToStartForQuest"}, + {13, &IApplicationFunctions::StubReplyZeroed, "CreateApplicationAndRequestToStartForQuest"}, + {14, &IApplicationFunctions::StubReplyZeroed, "CreateApplicationWithAttributeAndPushAndRequestToStartForQuest"}, + {15, &IApplicationFunctions::StubReplyZeroed, "CreateApplicationWithAttributeAndRequestToStartForQuest"}, {20, D<&IApplicationFunctions::EnsureSaveData>, "EnsureSaveData"}, {21, D<&IApplicationFunctions::GetDesiredLanguage>, "GetDesiredLanguage"}, {22, D<&IApplicationFunctions::SetTerminateResult>, "SetTerminateResult"}, {23, D<&IApplicationFunctions::GetDisplayVersion>, "GetDisplayVersion"}, - {24, nullptr, "GetLaunchStorageInfoForDebug"}, + {24, &IApplicationFunctions::StubReplyZeroed, "GetLaunchStorageInfoForDebug"}, {25, D<&IApplicationFunctions::ExtendSaveData>, "ExtendSaveData"}, {26, D<&IApplicationFunctions::GetSaveDataSize>, "GetSaveDataSize"}, {27, D<&IApplicationFunctions::CreateCacheStorage>, "CreateCacheStorage"}, @@ -80,53 +81,57 @@ IApplicationFunctions::IApplicationFunctions(Core::System& system_, std::shared_ {32, D<&IApplicationFunctions::BeginBlockingHomeButton>, "BeginBlockingHomeButton"}, {33, D<&IApplicationFunctions::EndBlockingHomeButton>, "EndBlockingHomeButton"}, {34, nullptr, "SelectApplicationLicense"}, - {35, nullptr, "GetDeviceSaveDataSizeMax"}, - {36, nullptr, "GetLimitedApplicationLicense"}, - {37, nullptr, "GetLimitedApplicationLicenseUpgradableEvent"}, + {35, &IApplicationFunctions::StubReplyZeroed, "GetDeviceSaveDataSizeMax"}, + {36, &IApplicationFunctions::StubReplyZeroed, "GetLimitedApplicationLicense"}, + {37, &IApplicationFunctions::StubReplyEvent, "GetLimitedApplicationLicenseUpgradableEvent"}, {40, D<&IApplicationFunctions::NotifyRunning>, "NotifyRunning"}, {50, D<&IApplicationFunctions::GetPseudoDeviceId>, "GetPseudoDeviceId"}, {60, D<&IApplicationFunctions::SetMediaPlaybackStateForApplication>, "SetMediaPlaybackStateForApplication"}, {65, D<&IApplicationFunctions::IsGamePlayRecordingSupported>, "IsGamePlayRecordingSupported"}, {66, D<&IApplicationFunctions::InitializeGamePlayRecording>, "InitializeGamePlayRecording"}, {67, D<&IApplicationFunctions::SetGamePlayRecordingState>, "SetGamePlayRecordingState"}, - {68, nullptr, "RequestFlushGamePlayingMovieForDebug"}, - {70, nullptr, "RequestToShutdown"}, - {71, nullptr, "RequestToReboot"}, - {72, nullptr, "RequestToSleep"}, - {80, nullptr, "ExitAndRequestToShowThanksMessage"}, + {68, &IApplicationFunctions::StubReplyZeroed, "RequestFlushGamePlayingMovieForDebug"}, + {70, &IApplicationFunctions::StubReplyZeroed, "RequestToShutdown"}, + {71, &IApplicationFunctions::StubReplyZeroed, "RequestToReboot"}, + {72, &IApplicationFunctions::StubReplyZeroed, "RequestToSleep"}, + {80, &IApplicationFunctions::StubReplyZeroed, "ExitAndRequestToShowThanksMessage"}, {90, D<&IApplicationFunctions::EnableApplicationCrashReport>, "EnableApplicationCrashReport"}, {100, D<&IApplicationFunctions::InitializeApplicationCopyrightFrameBuffer>, "InitializeApplicationCopyrightFrameBuffer"}, {101, D<&IApplicationFunctions::SetApplicationCopyrightImage>, "SetApplicationCopyrightImage"}, {102, D<&IApplicationFunctions::SetApplicationCopyrightVisibility>, "SetApplicationCopyrightVisibility"}, {110, D<&IApplicationFunctions::QueryApplicationPlayStatistics>, "QueryApplicationPlayStatistics"}, {111, D<&IApplicationFunctions::QueryApplicationPlayStatisticsByUid>, "QueryApplicationPlayStatisticsByUid"}, + {112, &IApplicationFunctions::StubReplyZeroed, "Unknown112"}, + {113, &IApplicationFunctions::StubReplyZeroed, "Unknown113"}, {120, D<&IApplicationFunctions::ExecuteProgram>, "ExecuteProgram"}, {121, D<&IApplicationFunctions::ClearUserChannel>, "ClearUserChannel"}, {122, D<&IApplicationFunctions::UnpopToUserChannel>, "UnpopToUserChannel"}, {123, D<&IApplicationFunctions::GetPreviousProgramIndex>, "GetPreviousProgramIndex"}, - {124, nullptr, "EnableApplicationAllThreadDumpOnCrash"}, + {124, &IApplicationFunctions::StubReplyZeroed, "EnableApplicationAllThreadDumpOnCrash"}, {130, D<&IApplicationFunctions::GetGpuErrorDetectedSystemEvent>, "GetGpuErrorDetectedSystemEvent"}, - {131, nullptr, "SetDelayTimeToAbortOnGpuError"}, + {131, &IApplicationFunctions::StubReplyZeroed, "SetDelayTimeToAbortOnGpuError"}, {140, D<&IApplicationFunctions::GetFriendInvitationStorageChannelEvent>, "GetFriendInvitationStorageChannelEvent"}, {141, D<&IApplicationFunctions::TryPopFromFriendInvitationStorageChannel>, "TryPopFromFriendInvitationStorageChannel"}, {150, D<&IApplicationFunctions::GetNotificationStorageChannelEvent>, "GetNotificationStorageChannelEvent"}, - {151, nullptr, "TryPopFromNotificationStorageChannel"}, + {151, &IApplicationFunctions::StubReplyInterface, "TryPopFromNotificationStorageChannel"}, {160, D<&IApplicationFunctions::GetHealthWarningDisappearedSystemEvent>, "GetHealthWarningDisappearedSystemEvent"}, - {170, nullptr, "SetHdcpAuthenticationActivated"}, - {180, nullptr, "GetLaunchRequiredVersion"}, - {181, nullptr, "UpgradeLaunchRequiredVersion"}, - {190, nullptr, "SendServerMaintenanceOverlayNotification"}, - {200, nullptr, "GetLastApplicationExitReason"}, + {170, &IApplicationFunctions::StubReplyZeroed, "SetHdcpAuthenticationActivated"}, + {180, &IApplicationFunctions::StubReplyZeroed, "GetLaunchRequiredVersion"}, + {181, &IApplicationFunctions::StubReplyZeroed, "UpgradeLaunchRequiredVersion"}, + {190, &IApplicationFunctions::StubReplyZeroed, "SendServerMaintenanceOverlayNotification"}, + {200, &IApplicationFunctions::StubReplyZeroed, "GetLastApplicationExitReason"}, {210, D<&IApplicationFunctions::GetOperationModeRefreshEvent>, "GetOperationModeRefreshEvent (guessed)"}, - {220, nullptr, "Unknown220"}, // [20.0.0+] - {300, nullptr, "CreateMovieWriter"}, // [19.0.0+] - {310, nullptr, "Unknown310"}, // [20.0.0+] + {220, &IApplicationFunctions::StubReplyZeroed, "Unknown220"}, // [20.0.0+] + {230, &IApplicationFunctions::StubReplyZeroed, "Unknown230"}, + {300, &IApplicationFunctions::StubReplyInterface, "CreateMovieWriter"}, // [19.0.0+] + {310, &IApplicationFunctions::StubReplyZeroed, "Unknown310"}, // [20.0.0+] {320, D<&IApplicationFunctions::Unknown320>, "Unknown320"}, // [20.0.0+] {321, D<&IApplicationFunctions::Unknown321>, "Unknown321"}, // Switch 2 {330, D<&IApplicationFunctions::Unknown330>, "Unknown330"}, // [20.0.0+] - {500, nullptr, "StartContinuousRecordingFlushForDebug"}, - {1000, nullptr, "CreateMovieMaker"}, + {500, &IApplicationFunctions::StubReplyEvent, "StartContinuousRecordingFlushForDebug"}, + {1000, &IApplicationFunctions::StubReplyInterface, "CreateMovieMaker"}, {1001, D<&IApplicationFunctions::PrepareForJit>, "PrepareForJit"}, + {2000, &IApplicationFunctions::StubReplyZeroed, "Unknown2000"}, }; // clang-format on @@ -571,15 +576,18 @@ Result IApplicationFunctions::GetOperationModeRefreshEvent( R_SUCCEED(); } -Result IApplicationFunctions::Unknown320() { - // Called by Switch 2 titles during nn::oe::Initialize; arguments unknown. +Result IApplicationFunctions::Unknown320(Out<u16> out) { + // Called by Switch 2 titles during nn::oe::Initialize. It returns two bytes of data (meaning + // unknown); a reply without them left the caller reading stale message words. LOG_WARNING(Service_AM, "(STUBBED) called"); + *out = 0; R_SUCCEED(); } -Result IApplicationFunctions::Unknown321() { - // Called by Switch 2 titles right after 320; arguments unknown. +Result IApplicationFunctions::Unknown321(Out<u8> out) { + // Called by Switch 2 titles right after 320; returns one byte (meaning unknown). LOG_WARNING(Service_AM, "(STUBBED) called"); + *out = 0; R_SUCCEED(); } diff --git a/src/core/hle/service/am/service/application_functions.h b/src/core/hle/service/am/service/application_functions.h index 44b5fadff0..aca2f8c694 100644 --- a/src/core/hle/service/am/service/application_functions.h +++ b/src/core/hle/service/am/service/application_functions.h @@ -83,8 +83,8 @@ private: Result GetHealthWarningDisappearedSystemEvent(OutCopyHandle<Kernel::KReadableEvent> out_event); Result CreateApplicationAndRequestToStart(u64 application_id); Result GetOperationModeRefreshEvent(OutCopyHandle<Kernel::KReadableEvent> out_event); - Result Unknown320(); - Result Unknown321(); + Result Unknown320(Out<u16> out); + Result Unknown321(Out<u8> out); Result Unknown330(Out<u8> out); Result PrepareForJit(); diff --git a/src/core/hle/service/am/service/application_proxy_service.cpp b/src/core/hle/service/am/service/application_proxy_service.cpp index 67306aadc0..9d332798b4 100644 --- a/src/core/hle/service/am/service/application_proxy_service.cpp +++ b/src/core/hle/service/am/service/application_proxy_service.cpp @@ -21,6 +21,7 @@ IApplicationProxyService::IApplicationProxyService(Core::System& system_, {0, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy"}, // Switch 2 SDKs open the proxy with command 10, same arguments. {10, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy (Switch 2)"}, + {2000, &IApplicationProxyService::StubReplyInterface, "Unknown2000"}, }; RegisterHandlers(functions); } diff --git a/src/core/hle/service/am/service/audio_controller.cpp b/src/core/hle/service/am/service/audio_controller.cpp index 4e9af756f2..7901f67c27 100644 --- a/src/core/hle/service/am/service/audio_controller.cpp +++ b/src/core/hle/service/am/service/audio_controller.cpp @@ -18,7 +18,7 @@ IAudioController::IAudioController(Core::System& system_) {2, D<&IAudioController::GetLibraryAppletExpectedMasterVolume>, "GetLibraryAppletExpectedMasterVolume"}, {3, D<&IAudioController::ChangeMainAppletMasterVolume>, "ChangeMainAppletMasterVolume"}, {4, D<&IAudioController::SetTransparentVolumeRate>, "SetTransparentVolumeRate"}, - {5, nullptr, "Unknown5"}, //20.0.0+ + {5, &IAudioController::StubReplyZeroed, "Unknown5"}, //20.0.0+ }; // clang-format on diff --git a/src/core/hle/service/am/service/common_state_getter.cpp b/src/core/hle/service/am/service/common_state_getter.cpp index 044929e434..2c5e536414 100644 --- a/src/core/hle/service/am/service/common_state_getter.cpp +++ b/src/core/hle/service/am/service/common_state_getter.cpp @@ -24,67 +24,73 @@ ICommonStateGetter::ICommonStateGetter(Core::System& system_, std::shared_ptr<Ap static const FunctionInfo functions[] = { {0, D<&ICommonStateGetter::GetEventHandle>, "GetEventHandle"}, {1, D<&ICommonStateGetter::ReceiveMessage>, "ReceiveMessage"}, - {2, nullptr, "GetThisAppletKind"}, - {3, nullptr, "AllowToEnterSleep"}, - {4, nullptr, "DisallowToEnterSleep"}, + {2, &ICommonStateGetter::StubReplyZeroed, "GetThisAppletKind"}, + {3, &ICommonStateGetter::StubReplyZeroed, "AllowToEnterSleep"}, + {4, &ICommonStateGetter::StubReplyZeroed, "DisallowToEnterSleep"}, {5, D<&ICommonStateGetter::GetOperationMode>, "GetOperationMode"}, {6, D<&ICommonStateGetter::GetPerformanceMode>, "GetPerformanceMode"}, - {7, nullptr, "GetCradleStatus"}, + {7, &ICommonStateGetter::StubReplyZeroed, "GetCradleStatus"}, {8, D<&ICommonStateGetter::GetBootMode>, "GetBootMode"}, {9, D<&ICommonStateGetter::GetCurrentFocusState>, "GetCurrentFocusState"}, {10, D<&ICommonStateGetter::RequestToAcquireSleepLock>, "RequestToAcquireSleepLock"}, {11, D<&ICommonStateGetter::ReleaseSleepLock>, "ReleaseSleepLock"}, {12, D<&ICommonStateGetter::ReleaseSleepLockTransiently>, "ReleaseSleepLockTransiently"}, {13, D<&ICommonStateGetter::GetAcquiredSleepLockEvent>, "GetAcquiredSleepLockEvent"}, - {14, nullptr, "GetWakeupCount"}, //11.0.0+ - {15, nullptr, "Unknown15"}, //19.0.0+ + {14, &ICommonStateGetter::StubReplyZeroed, "GetWakeupCount"}, //11.0.0+ + {15, &ICommonStateGetter::StubReplyEvent, "Unknown15"}, //19.0.0+ {20, D<&ICommonStateGetter::PushToGeneralChannel>, "PushToGeneralChannel"}, {30, D<&ICommonStateGetter::GetHomeButtonReaderLockAccessor>, "GetHomeButtonReaderLockAccessor"}, {31, D<&ICommonStateGetter::GetReaderLockAccessorEx>, "GetReaderLockAccessorEx"}, //2.0.0+ {32, D<&ICommonStateGetter::GetWriterLockAccessorEx>, "GetWriterLockAccessorEx"}, //7.0.0+ - {40, nullptr, "GetCradleFwVersion"}, //2.0.0+ + {40, &ICommonStateGetter::StubReplyZeroed, "GetCradleFwVersion"}, //2.0.0+ {50, D<&ICommonStateGetter::IsVrModeEnabled>, "IsVrModeEnabled"}, //3.0.0+ {51, D<&ICommonStateGetter::SetVrModeEnabled>, "SetVrModeEnabled"}, //3.0.0+ {52, D<&ICommonStateGetter::SetLcdBacklighOffEnabled>, "SetLcdBacklighOffEnabled"}, //4.0.0+ {53, D<&ICommonStateGetter::BeginVrModeEx>, "BeginVrModeEx"}, //7.0.0+ {54, D<&ICommonStateGetter::EndVrModeEx>, "EndVrModeEx"}, //7.0.0+ {55, D<&ICommonStateGetter::IsInControllerFirmwareUpdateSection>, "IsInControllerFirmwareUpdateSection"}, //3.0.0+ - {59, nullptr, "SetVrPositionForDebug"}, //1.0.0+ + {59, &ICommonStateGetter::StubReplyZeroed, "SetVrPositionForDebug"}, //1.0.0+ {60, D<&ICommonStateGetter::GetDefaultDisplayResolution>, "GetDefaultDisplayResolution"}, {61, D<&ICommonStateGetter::GetDefaultDisplayResolutionChangeEvent>, "GetDefaultDisplayResolutionChangeEvent"}, {62, D<&ICommonStateGetter::GetHdcpAuthenticationState>, "GetHdcpAuthenticationState"}, {63, D<&ICommonStateGetter::GetHdcpAuthenticationStateChangeEvent>, "GetHdcpAuthenticationStateChangeEvent"}, - {64, nullptr, "SetTvPowerStateMatchingMode"}, - {65, nullptr, "GetApplicationIdByContentActionName"}, + {64, &ICommonStateGetter::StubReplyZeroed, "SetTvPowerStateMatchingMode"}, + {65, &ICommonStateGetter::StubReplyZeroed, "GetApplicationIdByContentActionName"}, {66, &ICommonStateGetter::SetCpuBoostMode, "SetCpuBoostMode"}, - {67, nullptr, "CancelCpuBoostMode"}, + {67, &ICommonStateGetter::StubReplyZeroed, "CancelCpuBoostMode"}, {68, D<&ICommonStateGetter::GetBuiltInDisplayType>, "GetBuiltInDisplayType"}, {80, D<&ICommonStateGetter::PerformSystemButtonPressingIfInFocus>, "PerformSystemButtonPressingIfInFocus"}, - {90, nullptr, "SetPerformanceConfigurationChangedNotification"}, - {91, nullptr, "GetCurrentPerformanceConfiguration"}, + {90, &ICommonStateGetter::StubReplyZeroed, "SetPerformanceConfigurationChangedNotification"}, + {91, &ICommonStateGetter::StubReplyZeroed, "GetCurrentPerformanceConfiguration"}, {100, D<&ICommonStateGetter::SetHandlingHomeButtonShortPressedEnabled>, "SetHandlingHomeButtonShortPressedEnabled"}, - {110, nullptr, "OpenMyGpuErrorHandler"}, + {110, &ICommonStateGetter::StubReplyInterface, "OpenMyGpuErrorHandler"}, {120, D<&ICommonStateGetter::GetAppletLaunchedHistory>, "GetAppletLaunchedHistory"}, //13.0.0+ {130, D<&ICommonStateGetter::EnableStartupLogoDisappearedMessage>, "EnableStartupLogoDisappearedMessage"}, //21.0.0+ {200, D<&ICommonStateGetter::GetOperationModeSystemInfo>, "GetOperationModeSystemInfo"}, {300, D<&ICommonStateGetter::GetSettingsPlatformRegion>, "GetSettingsPlatformRegion"}, - {400, nullptr, "ActivateMigrationService"}, - {401, nullptr, "DeactivateMigrationService"}, - {500, nullptr, "DisableSleepTillShutdown"}, - {501, nullptr, "SuppressDisablingSleepTemporarily"}, - {502, nullptr, "IsSleepEnabled"}, - {503, nullptr, "IsDisablingSleepSuppressed"}, - {600, nullptr, "SetHidInputMagnificationForApplication"}, //20.0.0+ + {310, &ICommonStateGetter::StubReplyZeroed, "Unknown310"}, + {320, &ICommonStateGetter::StubReplyZeroed, "Unknown320"}, + {400, &ICommonStateGetter::StubReplyZeroed, "ActivateMigrationService"}, + {401, &ICommonStateGetter::StubReplyZeroed, "DeactivateMigrationService"}, + {500, &ICommonStateGetter::StubReplyZeroed, "DisableSleepTillShutdown"}, + {501, &ICommonStateGetter::StubReplyZeroed, "SuppressDisablingSleepTemporarily"}, + {502, &ICommonStateGetter::StubReplyZeroed, "IsSleepEnabled"}, + {503, &ICommonStateGetter::StubReplyZeroed, "IsDisablingSleepSuppressed"}, + {600, &ICommonStateGetter::StubReplyZeroed, "SetHidInputMagnificationForApplication"}, //20.0.0+ {610, D<&ICommonStateGetter::Unknown610>, "Unknown610"}, //21.0.0+ {611, D<&ICommonStateGetter::Unknown611>, "Unknown611"}, //22.0.0+ + {650, &ICommonStateGetter::StubReplyZeroed, "Unknown650"}, + {651, &ICommonStateGetter::StubReplyZeroed, "Unknown651"}, {900, D<&ICommonStateGetter::SetRequestExitToLibraryAppletAtExecuteNextProgramEnabled>, "SetRequestExitToLibraryAppletAtExecuteNextProgramEnabled"}, //11.0.0+ - {910, nullptr, "GetLaunchRequiredTick"}, //17.0.0+ + {910, &ICommonStateGetter::StubReplyZeroed, "GetLaunchRequiredTick"}, //17.0.0+ {1000, D<&ICommonStateGetter::BeginVrMode3d>, "BeginVrMode3d"}, //19.0.0+ {1001, D<&ICommonStateGetter::EndVrMode3d>, "EndVrMode3d"}, //19.0.0+ {1002, D<&ICommonStateGetter::IsVrModeEnabled3d>, "IsVrModeEnabled3d"}, //19.0.0+ {1003, D<&ICommonStateGetter::GetVrLaboGoggleViewport>, "GetVrLaboGoggleViewport"}, //21.0.0+ {1004, D<&ICommonStateGetter::GetPanelPhysicalSizeForSpecificTitle>, "GetPanelPhysicalSizeForSpecificTitle"}, //21.0.0+ {1005, D<&ICommonStateGetter::GetPanelResolutionForSpecificTitle>, "GetPanelResolutionForSpecificTitle"}, //21.0.0+ + {2000, &ICommonStateGetter::StubReplyZeroed, "Unknown2000"}, + {2010, &ICommonStateGetter::StubReplyZeroed, "Unknown2010"}, }; // clang-format on diff --git a/src/core/hle/service/am/service/display_controller.cpp b/src/core/hle/service/am/service/display_controller.cpp index 8f87327b17..7e7db23708 100644 --- a/src/core/hle/service/am/service/display_controller.cpp +++ b/src/core/hle/service/am/service/display_controller.cpp @@ -16,34 +16,35 @@ IDisplayController::IDisplayController(Core::System& system_, std::shared_ptr<Ap : ServiceFramework{system_, "IDisplayController"}, applet(std::move(applet_)) { // clang-format off static const FunctionInfo functions[] = { - {0, nullptr, "GetLastForegroundCaptureImage"}, - {1, nullptr, "UpdateLastForegroundCaptureImage"}, - {2, nullptr, "GetLastApplicationCaptureImage"}, - {3, nullptr, "GetCallerAppletCaptureImage"}, - {4, nullptr, "UpdateCallerAppletCaptureImage"}, + {0, &IDisplayController::StubReplyZeroed, "GetLastForegroundCaptureImage"}, + {1, &IDisplayController::StubReplyZeroed, "UpdateLastForegroundCaptureImage"}, + {2, &IDisplayController::StubReplyZeroed, "GetLastApplicationCaptureImage"}, + {3, &IDisplayController::StubReplyZeroed, "GetCallerAppletCaptureImage"}, + {4, &IDisplayController::StubReplyZeroed, "UpdateCallerAppletCaptureImage"}, {5, D<&IDisplayController::GetLastForegroundCaptureImageEx>, "GetLastForegroundCaptureImageEx"}, - {6, nullptr, "GetLastApplicationCaptureImageEx"}, + {6, &IDisplayController::StubReplyZeroed, "GetLastApplicationCaptureImageEx"}, {7, D<&IDisplayController::GetCallerAppletCaptureImageEx>, "GetCallerAppletCaptureImageEx"}, {8, D<&IDisplayController::TakeScreenShotOfOwnLayer>, "TakeScreenShotOfOwnLayer"}, - {9, nullptr, "CopyBetweenCaptureBuffers"}, - {10, nullptr, "AcquireLastApplicationCaptureBuffer"}, - {11, nullptr, "ReleaseLastApplicationCaptureBuffer"}, - {12, nullptr, "AcquireLastForegroundCaptureBuffer"}, - {13, nullptr, "ReleaseLastForegroundCaptureBuffer"}, - {14, nullptr, "AcquireCallerAppletCaptureBuffer"}, - {15, nullptr, "ReleaseCallerAppletCaptureBuffer"}, - {16, nullptr, "AcquireLastApplicationCaptureBufferEx"}, - {17, nullptr, "AcquireLastForegroundCaptureBufferEx"}, - {18, nullptr, "AcquireCallerAppletCaptureBufferEx"}, + {9, &IDisplayController::StubReplyZeroed, "CopyBetweenCaptureBuffers"}, + {10, &IDisplayController::StubReplyEvent, "AcquireLastApplicationCaptureBuffer"}, + {11, &IDisplayController::StubReplyZeroed, "ReleaseLastApplicationCaptureBuffer"}, + {12, &IDisplayController::StubReplyEvent, "AcquireLastForegroundCaptureBuffer"}, + {13, &IDisplayController::StubReplyZeroed, "ReleaseLastForegroundCaptureBuffer"}, + {14, &IDisplayController::StubReplyEvent, "AcquireCallerAppletCaptureBuffer"}, + {15, &IDisplayController::StubReplyZeroed, "ReleaseCallerAppletCaptureBuffer"}, + {16, &IDisplayController::StubReplyEvent, "AcquireLastApplicationCaptureBufferEx"}, + {17, &IDisplayController::StubReplyEvent, "AcquireLastForegroundCaptureBufferEx"}, + {18, &IDisplayController::StubReplyEvent, "AcquireCallerAppletCaptureBufferEx"}, + {19, &IDisplayController::StubReplyZeroed, "Unknown19"}, {20, D<&IDisplayController::ClearCaptureBuffer>, "ClearCaptureBuffer"}, - {21, nullptr, "ClearAppletTransitionBuffer"}, + {21, &IDisplayController::StubReplyZeroed, "ClearAppletTransitionBuffer"}, {22, D<&IDisplayController::AcquireLastApplicationCaptureSharedBuffer>, "AcquireLastApplicationCaptureSharedBuffer"}, {23, D<&IDisplayController::ReleaseLastApplicationCaptureSharedBuffer>, "ReleaseLastApplicationCaptureSharedBuffer"}, {24, D<&IDisplayController::AcquireLastForegroundCaptureSharedBuffer>, "AcquireLastForegroundCaptureSharedBuffer"}, {25, D<&IDisplayController::ReleaseLastForegroundCaptureSharedBuffer>, "ReleaseLastForegroundCaptureSharedBuffer"}, {26, D<&IDisplayController::AcquireCallerAppletCaptureSharedBuffer>, "AcquireCallerAppletCaptureSharedBuffer"}, {27, D<&IDisplayController::ReleaseCallerAppletCaptureSharedBuffer>, "ReleaseCallerAppletCaptureSharedBuffer"}, - {28, nullptr, "TakeScreenShotOfOwnLayerEx"}, + {28, &IDisplayController::StubReplyZeroed, "TakeScreenShotOfOwnLayerEx"}, }; // clang-format on diff --git a/src/core/hle/service/am/service/graphics_scale_controller.h b/src/core/hle/service/am/service/graphics_scale_controller.h index 07e38c6ac8..b31858d3ed 100644 --- a/src/core/hle/service/am/service/graphics_scale_controller.h +++ b/src/core/hle/service/am/service/graphics_scale_controller.h @@ -25,9 +25,13 @@ public: } private: - Result GetCurrentGraphicsScale(Out<u32> out_scale) { + // The reply is 12 bytes; only the first word (the scale) is known. The other two used to be + // left to the message buffer's leftovers. + Result GetCurrentGraphicsScale(Out<u32> out_scale, Out<u32> out_unknown1, Out<u32> out_unknown2) { LOG_WARNING(Service_AM, "(STUBBED) called, returning 10"); *out_scale = 10; + *out_unknown1 = 0; + *out_unknown2 = 0; R_SUCCEED(); } }; diff --git a/src/core/hle/service/am/service/library_applet_accessor.cpp b/src/core/hle/service/am/service/library_applet_accessor.cpp index 849e5794cf..7119c60596 100644 --- a/src/core/hle/service/am/service/library_applet_accessor.cpp +++ b/src/core/hle/service/am/service/library_applet_accessor.cpp @@ -62,21 +62,22 @@ ILibraryAppletAccessor::ILibraryAppletAccessor(Core::System& system_, {20, D<&ILibraryAppletAccessor::RequestExit>, "RequestExit"}, {25, D<&ILibraryAppletAccessor::Terminate>, "Terminate"}, {30, D<&ILibraryAppletAccessor::GetResult>, "GetResult"}, - {50, nullptr, "SetOutOfFocusApplicationSuspendingEnabled"}, + {50, &ILibraryAppletAccessor::StubReplyZeroed, "SetOutOfFocusApplicationSuspendingEnabled"}, {60, D<&ILibraryAppletAccessor::PresetLibraryAppletGpuTimeSliceZero>, "PresetLibraryAppletGpuTimeSliceZero"}, //10.0.0+ - {80, nullptr, "RequestForLibraryAppletToGetForeground"}, //19.0.0+ - {81, nullptr, "GetCurrentChildLibraryApplet"}, //19.0.0+ + {70, &ILibraryAppletAccessor::StubReplyZeroed, "Unknown70"}, + {80, &ILibraryAppletAccessor::StubReplyZeroed, "RequestForLibraryAppletToGetForeground"}, //19.0.0+ + {81, &ILibraryAppletAccessor::StubReplyInterface, "GetCurrentChildLibraryApplet"}, //19.0.0+ {90, D<&ILibraryAppletAccessor::Unknown90>, "Unknown90"}, //20.0.0+ {100, D<&ILibraryAppletAccessor::PushInData>, "PushInData"}, {101, D<&ILibraryAppletAccessor::PopOutData>, "PopOutData"}, - {102, nullptr, "PushExtraStorage"}, + {102, &ILibraryAppletAccessor::StubReplyZeroed, "PushExtraStorage"}, {103, D<&ILibraryAppletAccessor::PushInteractiveInData>, "PushInteractiveInData"}, {104, D<&ILibraryAppletAccessor::PopInteractiveOutData>, "PopInteractiveOutData"}, {105, D<&ILibraryAppletAccessor::GetPopOutDataEvent>, "GetPopOutDataEvent"}, {106, D<&ILibraryAppletAccessor::GetPopInteractiveOutDataEvent>, "GetPopInteractiveOutDataEvent"}, - {110, nullptr, "NeedsToExitProcess"}, + {110, &ILibraryAppletAccessor::StubReplyZeroed, "NeedsToExitProcess"}, {120, D<&ILibraryAppletAccessor::GetLibraryAppletInfo>, "GetLibraryAppletInfo"}, - {150, nullptr, "RequestForAppletToGetForeground"}, + {150, &ILibraryAppletAccessor::StubReplyZeroed, "RequestForAppletToGetForeground"}, {160, D<&ILibraryAppletAccessor::GetIndirectLayerConsumerHandle>, "GetIndirectLayerConsumerHandle"}, //2.0.0+ {170, D<&ILibraryAppletAccessor::Unknown170>, "Unknown170"}, //22.0.0+ }; diff --git a/src/core/hle/service/am/service/library_applet_creator.cpp b/src/core/hle/service/am/service/library_applet_creator.cpp index afdb609c00..2993ed7436 100644 --- a/src/core/hle/service/am/service/library_applet_creator.cpp +++ b/src/core/hle/service/am/service/library_applet_creator.cpp @@ -204,8 +204,8 @@ ILibraryAppletCreator::ILibraryAppletCreator(Core::System& system_, std::shared_ m_window_system{window_system}, m_applet{std::move(applet)} { static const FunctionInfo functions[] = { {0, D<&ILibraryAppletCreator::CreateLibraryApplet>, "CreateLibraryApplet"}, - {1, nullptr, "TerminateAllLibraryApplets"}, - {2, nullptr, "AreAnyLibraryAppletsLeft"}, + {1, &ILibraryAppletCreator::StubReplyZeroed, "TerminateAllLibraryApplets"}, + {2, &ILibraryAppletCreator::StubReplyZeroed, "AreAnyLibraryAppletsLeft"}, {3, D<&ILibraryAppletCreator::CreateLibraryAppletEx>, "CreateLibraryAppletEx"}, {10, D<&ILibraryAppletCreator::CreateStorage>, "CreateStorage"}, {11, D<&ILibraryAppletCreator::CreateTransferMemoryStorage>, "CreateTransferMemoryStorage"}, diff --git a/src/core/hle/service/am/service/self_controller.cpp b/src/core/hle/service/am/service/self_controller.cpp index a8078e6048..d67e1b32cd 100644 --- a/src/core/hle/service/am/service/self_controller.cpp +++ b/src/core/hle/service/am/service/self_controller.cpp @@ -37,44 +37,59 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet> {14, D<&ISelfController::SetRestartMessageEnabled>, "SetRestartMessageEnabled"}, {15, D<&ISelfController::SetScreenShotAppletIdentityInfo>, "SetScreenShotAppletIdentityInfo"}, {16, D<&ISelfController::SetOutOfFocusSuspendingEnabled>, "SetOutOfFocusSuspendingEnabled"}, - {17, nullptr, "SetControllerFirmwareUpdateSection"}, - {18, nullptr, "SetRequiresCaptureButtonShortPressedMessage"}, + {17, &ISelfController::StubReplyZeroed, "SetControllerFirmwareUpdateSection"}, + {18, &ISelfController::StubReplyZeroed, "SetRequiresCaptureButtonShortPressedMessage"}, {19, D<&ISelfController::SetAlbumImageOrientation>, "SetAlbumImageOrientation"}, - {20, nullptr, "SetDesirableKeyboardLayout"}, - {21, nullptr, "GetScreenShotProgramId"}, + {20, &ISelfController::StubReplyZeroed, "SetDesirableKeyboardLayout"}, + {21, &ISelfController::StubReplyZeroed, "GetScreenShotProgramId"}, + {22, &ISelfController::StubReplyZeroed, "Unknown22"}, + {23, &ISelfController::StubReplyZeroed, "Unknown23"}, + {24, &ISelfController::StubReplyZeroed, "Unknown24"}, {40, D<&ISelfController::CreateManagedDisplayLayer>, "CreateManagedDisplayLayer"}, {41, D<&ISelfController::IsSystemBufferSharingEnabled>, "IsSystemBufferSharingEnabled"}, {42, D<&ISelfController::GetSystemSharedLayerHandle>, "GetSystemSharedLayerHandle"}, {43, D<&ISelfController::GetSystemSharedBufferHandle>, "GetSystemSharedBufferHandle"}, {44, D<&ISelfController::CreateManagedDisplaySeparableLayer>, "CreateManagedDisplaySeparableLayer"}, - {45, nullptr, "SetManagedDisplayLayerSeparationMode"}, - {46, nullptr, "SetRecordingLayerCompositionEnabled"}, + {45, &ISelfController::StubReplyZeroed, "SetManagedDisplayLayerSeparationMode"}, + {46, &ISelfController::StubReplyZeroed, "SetRecordingLayerCompositionEnabled"}, {50, D<&ISelfController::SetHandlesRequestToDisplay>, "SetHandlesRequestToDisplay"}, {51, D<&ISelfController::ApproveToDisplay>, "ApproveToDisplay"}, {60, D<&ISelfController::OverrideAutoSleepTimeAndDimmingTime>, "OverrideAutoSleepTimeAndDimmingTime"}, {61, D<&ISelfController::SetMediaPlaybackState>, "SetMediaPlaybackState"}, {62, D<&ISelfController::SetIdleTimeDetectionExtension>, "SetIdleTimeDetectionExtension"}, {63, D<&ISelfController::GetIdleTimeDetectionExtension>, "GetIdleTimeDetectionExtension"}, - {64, nullptr, "SetInputDetectionSourceSet"}, + {64, &ISelfController::StubReplyZeroed, "SetInputDetectionSourceSet"}, {65, D<&ISelfController::ReportUserIsActive>, "ReportUserIsActive"}, - {66, nullptr, "GetCurrentIlluminance"}, + {66, &ISelfController::StubReplyZeroed, "GetCurrentIlluminance"}, {67, D<&ISelfController::IsIlluminanceAvailable>, "IsIlluminanceAvailable"}, {68, D<&ISelfController::SetAutoSleepDisabled>, "SetAutoSleepDisabled"}, {69, D<&ISelfController::IsAutoSleepDisabled>, "IsAutoSleepDisabled"}, - {70, nullptr, "ReportMultimediaError"}, - {71, nullptr, "GetCurrentIlluminanceEx"}, + {70, &ISelfController::StubReplyZeroed, "ReportMultimediaError"}, + {71, &ISelfController::StubReplyZeroed, "GetCurrentIlluminanceEx"}, {72, D<&ISelfController::SetInputDetectionPolicy>, "SetInputDetectionPolicy"}, - {80, nullptr, "SetWirelessPriorityMode"}, + {73, &ISelfController::StubReplyZeroed, "Unknown73"}, + {80, &ISelfController::StubReplyZeroed, "SetWirelessPriorityMode"}, {90, D<&ISelfController::GetAccumulatedSuspendedTickValue>, "GetAccumulatedSuspendedTickValue"}, {91, D<&ISelfController::GetAccumulatedSuspendedTickChangedEvent>, "GetAccumulatedSuspendedTickChangedEvent"}, {100, D<&ISelfController::SetAlbumImageTakenNotificationEnabled>, "SetAlbumImageTakenNotificationEnabled"}, - {110, nullptr, "SetApplicationAlbumUserData"}, + {110, &ISelfController::StubReplyZeroed, "SetApplicationAlbumUserData"}, {120, D<&ISelfController::SaveCurrentScreenshot>, "SaveCurrentScreenshot"}, {130, D<&ISelfController::SetRecordVolumeMuted>, "SetRecordVolumeMuted"}, + {200, &ISelfController::StubReplyZeroed, "Unknown200"}, + {210, &ISelfController::StubReplyZeroed, "Unknown210"}, + {211, &ISelfController::StubReplyZeroed, "Unknown211"}, + {220, &ISelfController::StubReplyZeroed, "Unknown220"}, + {221, &ISelfController::StubReplyZeroed, "Unknown221"}, {230, D<&ISelfController::Unknown230>, "Unknown230"}, {240, D<&ISelfController::Unknown240>, "Unknown240"}, - {1000, nullptr, "GetDebugStorageChannel"}, + {1000, &ISelfController::StubReplyInterface, "GetDebugStorageChannel"}, {2000, &ISelfController::GetNx2GraphicsSystemSession, "GetGraphicsSystemSession (Switch 2, guessed)"}, + {2100, &ISelfController::StubReplyZeroed, "Unknown2100"}, + {2200, &ISelfController::StubReplyZeroed, "Unknown2200"}, + {2300, &ISelfController::StubReplyZeroed, "Unknown2300"}, + {2400, &ISelfController::StubReplyZeroed, "Unknown2400"}, + {2401, &ISelfController::StubReplyZeroed, "Unknown2401"}, + {2402, &ISelfController::StubReplyZeroed, "Unknown2402"}, }; // clang-format on diff --git a/src/core/hle/service/am/service/window_controller.cpp b/src/core/hle/service/am/service/window_controller.cpp index f2b9dccb27..20bf71083e 100644 --- a/src/core/hle/service/am/service/window_controller.cpp +++ b/src/core/hle/service/am/service/window_controller.cpp @@ -18,7 +18,7 @@ IWindowController::IWindowController(Core::System& system_, std::shared_ptr<Appl m_window_system{window_system}, m_applet{std::move(applet)} { // clang-format off static const FunctionInfo functions[] = { - {0, nullptr, "CreateWindow"}, + {0, &IWindowController::StubReplyInterface, "CreateWindow"}, {1, D<&IWindowController::GetAppletResourceUserId>, "GetAppletResourceUserId"}, {2, D<&IWindowController::GetAppletResourceUserIdOfCallerApplet>, "GetAppletResourceUserIdOfCallerApplet"}, {10, D<&IWindowController::AcquireForegroundRights>, "AcquireForegroundRights"}, diff --git a/src/core/hle/service/aoc/addon_content_manager.cpp b/src/core/hle/service/aoc/addon_content_manager.cpp index be2aa34baa..c63b08e40d 100644 --- a/src/core/hle/service/aoc/addon_content_manager.cpp +++ b/src/core/hle/service/aoc/addon_content_manager.cpp @@ -68,19 +68,19 @@ IAddOnContentManager::IAddOnContentManager(Core::System& system_) {6, nullptr, "PrepareAddOnContentByApplicationId"}, {7, D<&IAddOnContentManager::PrepareAddOnContent>, "PrepareAddOnContent"}, {8, D<&IAddOnContentManager::GetAddOnContentListChangedEvent>, "GetAddOnContentListChangedEvent"}, - {9, nullptr, "GetAddOnContentLostErrorCode"}, + {9, &IAddOnContentManager::StubReplyZeroed, "GetAddOnContentLostErrorCode"}, {10, D<&IAddOnContentManager::GetAddOnContentListChangedEventWithProcessId>, "GetAddOnContentListChangedEventWithProcessId"}, {11, D<&IAddOnContentManager::NotifyMountAddOnContent>, "NotifyMountAddOnContent"}, {12, D<&IAddOnContentManager::NotifyUnmountAddOnContent>, "NotifyUnmountAddOnContent"}, - {13, nullptr, "IsAddOnContentMountedForDebug"}, + {13, &IAddOnContentManager::StubReplyZeroed, "IsAddOnContentMountedForDebug"}, {50, D<&IAddOnContentManager::CheckAddOnContentMountStatus>, "CheckAddOnContentMountStatus"}, {100, D<&IAddOnContentManager::CreateEcPurchasedEventManager>, "CreateEcPurchasedEventManager"}, {101, D<&IAddOnContentManager::CreatePermanentEcPurchasedEventManager>, "CreatePermanentEcPurchasedEventManager"}, - {110, nullptr, "CreateContentsServiceManager"}, - {200, nullptr, "SetRequiredAddOnContentsOnContentsAvailabilityTransition"}, - {300, nullptr, "SetupHostAddOnContent"}, - {301, nullptr, "GetRegisteredAddOnContentPath"}, - {302, nullptr, "UpdateCachedList"}, + {110, &IAddOnContentManager::StubReplyInterface, "CreateContentsServiceManager"}, + {200, &IAddOnContentManager::StubReplyZeroed, "SetRequiredAddOnContentsOnContentsAvailabilityTransition"}, + {300, &IAddOnContentManager::StubReplyZeroed, "SetupHostAddOnContent"}, + {301, &IAddOnContentManager::StubReplyZeroed, "GetRegisteredAddOnContentPath"}, + {302, &IAddOnContentManager::StubReplyZeroed, "UpdateCachedList"}, }; // clang-format on diff --git a/src/core/hle/service/audio/audio.cpp b/src/core/hle/service/audio/audio.cpp index 211e1cb6bc..eb1511857b 100644 --- a/src/core/hle/service/audio/audio.cpp +++ b/src/core/hle/service/audio/audio.cpp @@ -6,6 +6,7 @@ #include "core/core.h" #include "core/hle/service/audio/audio.h" +#include "core/hle/service/audio/nx2_audio_device_manager.h" #include "core/hle/service/audio/nx2_audio_user_client.h" #include "core/hle/service/audio/audio_controller.h" #include "core/hle/service/audio/audio_in_manager.h" @@ -206,6 +207,8 @@ void LoopProcess(Core::System& system) { server_manager->RegisterNamedService("audren:u", std::make_shared<IAudioRendererManager>(system), 30); // Switch 2 audio entry point (nn::audio::CreateUserClientSessionByHipc). server_manager->RegisterNamedService("aud:u", std::make_shared<INx2AudioUserClient>(system), 30); + server_manager->RegisterNamedService("auddmg:u", std::make_shared<INx2AudioDeviceManagerService>(system), 30); + server_manager->RegisterNamedService("audsmx:u", std::make_shared<INx2AudioSoundMixerService>(system), 30); server_manager->RegisterNamedService("audren:a", std::make_shared<IAudioRendererManagerForApplet>(system), 30); server_manager->RegisterNamedService("hwopus", std::make_shared<IHardwareOpusDecoderManager>(system), 25); ServerManager::RunServer(std::move(server_manager)); diff --git a/src/core/hle/service/audio/audio_device.cpp b/src/core/hle/service/audio/audio_device.cpp index e9f7e074a1..8f5856b76c 100644 --- a/src/core/hle/service/audio/audio_device.cpp +++ b/src/core/hle/service/audio/audio_device.cpp @@ -16,7 +16,9 @@ IAudioDevice::IAudioDevice(Core::System& system_, u64 applet_resource_user_id, u u32 device_num) : ServiceFramework{system_, "IAudioDevice"}, service_context{system_, "IAudioDevice"}, impl{std::make_unique<AudioDevice>(system_, applet_resource_user_id, revision)}, - event{service_context.CreateEvent(fmt::format("IAudioDeviceEvent-{}", device_num))} { + event{service_context.CreateEvent(fmt::format("IAudioDeviceEvent-{}", device_num))}, + notification_event{service_context.CreateEvent( + fmt::format("IAudioDeviceNotification-{}", device_num))} { static const FunctionInfo functions[] = { {0, D<&IAudioDevice::ListAudioDeviceName>, "ListAudioDeviceName"}, {1, D<&IAudioDevice::SetAudioDeviceOutputVolume>, "SetAudioDeviceOutputVolume"}, @@ -27,18 +29,19 @@ IAudioDevice::IAudioDevice(Core::System& system_, u64 applet_resource_user_id, u {6, D<&IAudioDevice::ListAudioDeviceNameAuto>, "ListAudioDeviceNameAuto"}, {7, D<&IAudioDevice::SetAudioDeviceOutputVolumeAuto>, "SetAudioDeviceOutputVolumeAuto"}, {8, D<&IAudioDevice::GetAudioDeviceOutputVolumeAuto>, "GetAudioDeviceOutputVolumeAuto"}, + {9, &IAudioDevice::StubReplyZeroed, "Unknown9"}, {10, D<&IAudioDevice::GetActiveAudioDeviceNameAuto>, "GetActiveAudioDeviceNameAuto"}, {11, D<&IAudioDevice::QueryAudioDeviceInputEvent>, "QueryAudioDeviceInputEvent"}, {12, D<&IAudioDevice::QueryAudioDeviceOutputEvent>, "QueryAudioDeviceOutputEvent"}, {13, D<&IAudioDevice::GetActiveAudioDeviceName>, "GetActiveAudioOutputDeviceName"}, {14, D<&IAudioDevice::ListAudioOutputDeviceName>, "ListAudioOutputDeviceName"}, - {15, nullptr, "AcquireAudioInputDeviceNotification"}, //17.0.0+ - {16, nullptr, "ReleaseAudioInputDeviceNotification"}, //17.0.0+ - {17, nullptr, "AcquireAudioOutputDeviceNotification"}, //17.0.0+ - {18, nullptr, "ReleaseAudioOutputDeviceNotification"}, //17.0.0+ + {15, D<&IAudioDevice::AcquireAudioDeviceNotification>, "AcquireAudioInputDeviceNotification"}, //17.0.0+ + {16, D<&IAudioDevice::ReleaseAudioDeviceNotification>, "ReleaseAudioInputDeviceNotification"}, //17.0.0+ + {17, D<&IAudioDevice::AcquireAudioDeviceNotification>, "AcquireAudioOutputDeviceNotification"}, //17.0.0+ + {18, D<&IAudioDevice::ReleaseAudioDeviceNotification>, "ReleaseAudioOutputDeviceNotification"}, //17.0.0+ {19, D<&IAudioDevice::SetAudioDeviceOutputVolumeAutoTuneEnabled>, "SetAudioDeviceOutputVolumeAutoTuneEnabled"}, //18.0.0+ {20, D<&IAudioDevice::IsAudioDeviceOutputVolumeAutoTuneEnabled>, "IsAudioDeviceOutputVolumeAutoTuneEnabled"}, //18.0.0+ - {21, nullptr, "IsActiveOutputDeviceEstimatedLowLatency"} //21.0.0+ + {21, &IAudioDevice::StubReplyZeroed, "IsActiveOutputDeviceEstimatedLowLatency"} //21.0.0+ }; RegisterHandlers(functions); @@ -46,6 +49,7 @@ IAudioDevice::IAudioDevice(Core::System& system_, u64 applet_resource_user_id, u } IAudioDevice::~IAudioDevice() { + service_context.CloseEvent(notification_event); service_context.CloseEvent(event); } @@ -145,6 +149,21 @@ Result IAudioDevice::QueryAudioDeviceOutputEvent(OutCopyHandle<Kernel::KReadable R_SUCCEED(); } +// 15/17(u64): an event that fires when the input/output device set changes, which it never +// does here. Switch 2 titles (Kirby Air Riders, Switch Sports) ask for it; the auto stub +// replied without a handle. +Result IAudioDevice::AcquireAudioDeviceNotification(OutCopyHandle<Kernel::KReadableEvent> out_event, + u64 unknown) { + LOG_DEBUG(Service_Audio, "called, unknown={:#x}", unknown); + *out_event = ¬ification_event->GetReadableEvent(); + R_SUCCEED(); +} + +Result IAudioDevice::ReleaseAudioDeviceNotification(u64 unknown) { + LOG_DEBUG(Service_Audio, "called, unknown={:#x}", unknown); + R_SUCCEED(); +} + Result IAudioDevice::GetActiveChannelCount(Out<u32> out_active_channel_count) { *out_active_channel_count = system.AudioCore().GetOutputSink().GetSystemChannels(); LOG_DEBUG(Service_Audio, "(STUBBED) called. Channels={}", *out_active_channel_count); diff --git a/src/core/hle/service/audio/audio_device.h b/src/core/hle/service/audio/audio_device.h index f6c12fbfeb..8a7b9a70b0 100644 --- a/src/core/hle/service/audio/audio_device.h +++ b/src/core/hle/service/audio/audio_device.h @@ -54,10 +54,14 @@ private: Out<s32> out_count); Result SetAudioDeviceOutputVolumeAutoTuneEnabled(bool enabled); Result IsAudioDeviceOutputVolumeAutoTuneEnabled(Out<bool> out_enabled); + Result AcquireAudioDeviceNotification(OutCopyHandle<Kernel::KReadableEvent> out_event, + u64 unknown); + Result ReleaseAudioDeviceNotification(u64 unknown); KernelHelpers::ServiceContext service_context; std::unique_ptr<AudioCore::Renderer::AudioDevice> impl; Kernel::KEvent* event; + Kernel::KEvent* notification_event; }; } // namespace Service::Audio diff --git a/src/core/hle/service/audio/audio_renderer_manager.cpp b/src/core/hle/service/audio/audio_renderer_manager.cpp index 97d9f44ddd..d669d64cec 100644 --- a/src/core/hle/service/audio/audio_renderer_manager.cpp +++ b/src/core/hle/service/audio/audio_renderer_manager.cpp @@ -27,7 +27,7 @@ IAudioRendererManager::IAudioRendererManager(Core::System& system_) {0, D<&IAudioRendererManager::OpenAudioRenderer>, "OpenAudioRenderer"}, {1, D<&IAudioRendererManager::GetWorkBufferSize>, "GetWorkBufferSize"}, {2, D<&IAudioRendererManager::GetAudioDeviceService>, "GetAudioDeviceService"}, - {3, nullptr, "OpenAudioRendererForManualExecution"}, + {3, &IAudioRendererManager::StubReplyInterface, "OpenAudioRendererForManualExecution"}, {4, D<&IAudioRendererManager::GetAudioDeviceServiceWithRevisionInfo>, "GetAudioDeviceServiceWithRevisionInfo"}, }; // clang-format on diff --git a/src/core/hle/service/audio/nx2_audio_device_manager.h b/src/core/hle/service/audio/nx2_audio_device_manager.h new file mode 100644 index 0000000000..746454660e --- /dev/null +++ b/src/core/hle/service/audio/nx2_audio_device_manager.h @@ -0,0 +1,539 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <algorithm> +#include <array> +#include <atomic> +#include <chrono> +#include <cstring> +#include <map> +#include <memory> +#include <mutex> +#include <optional> +#include <string_view> +#include <vector> + +#include <fmt/ranges.h> + +#include "common/string_util.h" +#include "core/core.h" +#include "core/core_timing.h" +#include "core/hle/kernel/k_event.h" +#include "core/memory.h" +#include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/kernel_helpers.h" +#include "core/hle/service/service.h" + +namespace Service::Audio { + +// Switch 2 sound mixer streams (nn::audio::sysmix, behind audsmx:u). A stream is a ring buffer in +// the game's own memory: CreateStream hands over its address and size (4 KiB aligned), OpenStream +// its format and length in frames, and once the streams are engaged the system takes frames out of +// the ring at the stream's sample rate. Eden doesn't mix these into its output yet; it drains the +// rings in real time so the game sees its audio being played (Switch Sports' menu movie feeds one). +// The ring's shared indices, in frames modulo (frame count + 1): the system's read index at +// buffer + 0x44, the game's write index at buffer + 0x1004. +struct Nx2SysmixStream { + u32 id{}; + u32 owner{}; ///< the user session whose timer drains it + u64 buffer{}; + u64 size{}; + u32 sample_type{}; + u32 channels{}; + u32 sample_rate{}; + u32 frames{}; + bool open{}; + bool engaged{}; + double owed_frames{}; + u32 data_mode{}; ///< command 7's mode; 0 leaves the data event alone + s32 data_threshold{}; ///< free frames that signal the data event + Kernel::KEvent* state_event{}; + Kernel::KEvent* data_event{}; +}; + +struct Nx2SysmixStreams { + std::mutex mutex; + std::map<u32, std::shared_ptr<Nx2SysmixStream>> streams; + u32 next_id{1}; +}; + +// Streams are created through audsmx:u but started and stopped through auddmg:u (AttachStream and +// DetachStream), so both services share one list. +inline std::shared_ptr<Nx2SysmixStreams> Nx2SysmixStreamList() { + static const auto list = std::make_shared<Nx2SysmixStreams>(); + return list; +} + +// Switch 2 virtual output devices (nn::audio::ListVirtualOutputDevices and friends), reached through +// the object auddmg:u command 0 returns. Eden plays everything through one stereo 48 kHz output, so +// there is one virtual device. The nx2: name keeps the other commands auto-stubbed and logged. +// - 0(u32): u32 count, the device names in the out buffer (0x100 bytes each). +// - 100(u32): the default device's name in the out buffer. +// - 300(16-byte stream format, name buffer): whether the format is supported (result only). +// - 401(u32, name buffer): the device's 0x228-byte property block in the out buffer. The sdk hands +// the game the name, the channel count (0x108) and the u64 at 0x218; with the zeroed placeholder +// reply it saw 0 channels and Switch Sports failed to set up audio (2153-1604). +// - 500(16 bytes, name buffer): an event signalled when the device's state changes. +// - 700(u32 stream id, u32 device functionality, name buffer): attach a sysmix stream to the +// device, which starts it (nn::audio::devmgr::AttachStream, behind OutputStream Start). +// - 800(u32 stream id): detach it again (Stop). +class INx2AudioVirtualDeviceManager final : public ServiceFramework<INx2AudioVirtualDeviceManager> { +public: + explicit INx2AudioVirtualDeviceManager(Core::System& system_) + : ServiceFramework{system_, "nx2:auddmg:u:manager"}, + service_context{system_, "nx2:auddmg:u:manager"}, + state_event{service_context.CreateEvent("nx2:auddmg:u:StateEvent")} { + // clang-format off + static const FunctionInfo functions[] = { + {0, &INx2AudioVirtualDeviceManager::ListVirtualOutputDevices, "ListVirtualOutputDevices"}, + {100, &INx2AudioVirtualDeviceManager::GetDefaultVirtualOutputDevice, "GetDefaultVirtualOutputDevice"}, + {300, &INx2AudioVirtualDeviceManager::IsStreamFormatSupported, "IsVirtualOutputDeviceStreamFormatSupported"}, + {401, &INx2AudioVirtualDeviceManager::GetVirtualOutputDeviceProperties, "GetVirtualOutputDeviceProperties"}, + {500, &INx2AudioVirtualDeviceManager::AcquireStateChangeEvent, "AcquireVirtualOutputDeviceStateEvent (guessed)"}, + {700, &INx2AudioVirtualDeviceManager::AttachStream, "AttachStream"}, + {800, &INx2AudioVirtualDeviceManager::DetachStream, "DetachStream"}, + }; + // clang-format on + RegisterHandlers(functions); + } + + ~INx2AudioVirtualDeviceManager() override { + service_context.CloseEvent(state_event); + } + +private: + static constexpr std::size_t NameSize = 0x100; + static constexpr std::string_view DeviceName = "AudioVirtualOutput0"; + static constexpr std::string_view PhysicalDeviceName = "AudioTvOutput"; + + static void CopyName(u8* out, std::string_view name) { + std::memcpy(out, name.data(), std::min(name.size(), NameSize - 1)); + } + + void ListVirtualOutputDevices(HLERequestContext& ctx) { + std::array<u8, NameSize> names{}; + CopyName(names.data(), DeviceName); + const u32 count = ctx.GetWriteBufferSize() >= NameSize ? 1 : 0; + if (count) { + ctx.WriteBuffer(names); + } + LOG_DEBUG(Service_Audio, "called, {} device", count); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(ResultSuccess); + rb.Push(count); + } + + void GetDefaultVirtualOutputDevice(HLERequestContext& ctx) { + std::array<u8, NameSize> name{}; + CopyName(name.data(), DeviceName); + ctx.WriteBuffer(name); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void IsStreamFormatSupported(HLERequestContext& ctx) { + LOG_DEBUG(Service_Audio, "called"); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Property block: name (0x000), u32 type (0x100, 3), u32 (0x104, 2), u32 channel count + // (0x108), u32 sample rate (0x10c), physical output name (0x110), its id (0x210), u64 latency + // in ns (0x218), u32 device id (0x220). The sdk's output streams pass the device id to + // sysmix CreateStream, which refuses 0 (Switch Sports failed with 2153-1604 on a zero there). + void GetVirtualOutputDeviceProperties(HLERequestContext& ctx) { + std::array<u8, 0x228> block{}; + const auto name = ctx.ReadBuffer(0); + std::memcpy(block.data(), name.data(), std::min(name.size(), NameSize - 1)); + const auto put32 = [&](std::size_t offset, u32 value) { + std::memcpy(block.data() + offset, &value, sizeof(value)); + }; + put32(0x100, 3); + put32(0x104, 2); + put32(0x108, 2); // stereo + put32(0x10c, 48000); // Hz + CopyName(block.data() + 0x110, PhysicalDeviceName); + const u64 latency_ns = 10'000'000; + std::memcpy(block.data() + 0x218, &latency_ns, sizeof(latency_ns)); + put32(0x220, 1); + ctx.WriteBuffer(block); + LOG_DEBUG(Service_Audio, "called for {}", Common::StringFromBuffer(name)); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void AcquireStateChangeEvent(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 2, 1}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, state_event->GetReadableEvent()); + } + + void AttachStream(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto stream_id = rp.Pop<u32>(); + const auto functionality = rp.Pop<u32>(); + LOG_INFO(Service_Audio, "attach stream {} to {} (functionality {})", stream_id, + Common::StringFromBuffer(ctx.ReadBuffer(0)), functionality); + SetStreamEngaged(stream_id, true); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void DetachStream(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto stream_id = rp.Pop<u32>(); + LOG_INFO(Service_Audio, "detach stream {}", stream_id); + SetStreamEngaged(stream_id, false); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void SetStreamEngaged(u32 stream_id, bool engaged) { + const auto list = Nx2SysmixStreamList(); + std::scoped_lock lk{list->mutex}; + if (const auto it = list->streams.find(stream_id); it != list->streams.end()) { + it->second->engaged = engaged; + it->second->owed_frames = 0; + if (it->second->state_event) { + it->second->state_event->Signal(system.Kernel()); + } + } + } + + KernelHelpers::ServiceContext service_context; + Kernel::KEvent* state_event; +}; + +// Switch 2 "auddmg:u": command 0 (16 bytes and the pid) returns the virtual device manager. +class INx2AudioDeviceManagerService final : public ServiceFramework<INx2AudioDeviceManagerService> { +public: + explicit INx2AudioDeviceManagerService(Core::System& system_) + : ServiceFramework{system_, "nx2:auddmg:u"} { + // clang-format off + static const FunctionInfo functions[] = { + {0, &INx2AudioDeviceManagerService::OpenVirtualDeviceManager, "OpenVirtualDeviceManager (guessed)"}, + }; + // clang-format on + RegisterHandlers(functions); + } + +private: + void OpenVirtualDeviceManager(HLERequestContext& ctx) { + LOG_DEBUG(Service_Audio, "called"); + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface<INx2AudioVirtualDeviceManager>(ctx, system); + } +}; + +// The stream object IUserSession command 1 returns. Commands (ids, request -> reply): +// - 0(u32 sample type, u32 channels, u32 sample rate, u32 frames, u32 prefill, u32 flags): open. +// - 1: close. 2: u32 state. 3: f32 gain. 4(f32 gain, s64 ramp ns): set the gain. +// - 5 / 6: acquire (copy handle) / release the state-change event. +// - 7(u32 mode, s32 threshold) / 8: acquire (copy handle) / release the data event. +// - 9(s64 ns): clock the stream. 10(s32 frames) -> u32: prime. 11(s32 frames) -> u32: purge. +// - 12(name buffer): rename. +class INx2SysmixStreamSession final : public ServiceFramework<INx2SysmixStreamSession> { +public: + explicit INx2SysmixStreamSession(Core::System& system_, std::shared_ptr<Nx2SysmixStreams> streams_, + std::shared_ptr<Nx2SysmixStream> stream_) + : ServiceFramework{system_, "nx2:audsmx:u:stream"}, service_context{system_, "nx2:audsmx:u:stream"}, + streams{std::move(streams_)}, stream{std::move(stream_)} { + // clang-format off + static const FunctionInfo functions[] = { + {0, &INx2SysmixStreamSession::OpenStream, "OpenStream"}, + {1, &INx2SysmixStreamSession::CloseStream, "CloseStream"}, + {2, &INx2SysmixStreamSession::GetStreamState, "GetStreamState"}, + {3, &INx2SysmixStreamSession::GetStreamGain, "GetStreamGain"}, + {4, &INx2SysmixStreamSession::SetStreamGain, "SetStreamGain"}, + {5, &INx2SysmixStreamSession::AcquireStateChangeEvent, "AcquireStreamStateChangeEvent"}, + {6, &INx2SysmixStreamSession::Succeed, "ReleaseStreamStateChangeEvent"}, + {7, &INx2SysmixStreamSession::AcquireDataEvent, "AcquireStreamDataEvent"}, + {8, &INx2SysmixStreamSession::Succeed, "ReleaseStreamDataEvent"}, + {9, &INx2SysmixStreamSession::Succeed, "ClockStream"}, + {10, &INx2SysmixStreamSession::EchoFrames, "PrimeStream"}, + {11, &INx2SysmixStreamSession::EchoFrames, "PurgeStream"}, + {12, &INx2SysmixStreamSession::Succeed, "RenameStream"}, + }; + // clang-format on + RegisterHandlers(functions); + std::scoped_lock lk{streams->mutex}; + stream->state_event = service_context.CreateEvent("nx2:audsmx:u:StreamStateEvent"); + stream->data_event = service_context.CreateEvent("nx2:audsmx:u:StreamDataEvent"); + } + + ~INx2SysmixStreamSession() override { + // The drain timer signals these events under the lock: unlist the stream first. + { + std::scoped_lock lk{streams->mutex}; + streams->streams.erase(stream->id); + service_context.CloseEvent(stream->state_event); + service_context.CloseEvent(stream->data_event); + stream->state_event = nullptr; + stream->data_event = nullptr; + } + } + +private: + // States the session reports: 0 created, 1 open, 2 open and engaged. + void OpenStream(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto sample_type = rp.Pop<u32>(); + const auto channels = rp.Pop<u32>(); + const auto sample_rate = rp.Pop<u32>(); + const auto frames = rp.Pop<u32>(); + const auto prefill = rp.Pop<u32>(); + const auto flags = rp.Pop<u32>(); + LOG_INFO(Service_Audio, + "stream {} open: sample type {}, {} channels, {} Hz, {} frames, prefill {}, flags {:#x}", + stream->id, sample_type, channels, sample_rate, frames, prefill, flags); + { + std::scoped_lock lk{streams->mutex}; + stream->sample_type = sample_type; + stream->channels = channels; + stream->sample_rate = sample_rate; + stream->frames = frames; + stream->owed_frames = 0; + stream->open = true; + stream->state_event->Signal(system.Kernel()); + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void CloseStream(HLERequestContext& ctx) { + { + std::scoped_lock lk{streams->mutex}; + stream->open = false; + stream->state_event->Signal(system.Kernel()); + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void GetStreamState(HLERequestContext& ctx) { + u32 state{}; + { + std::scoped_lock lk{streams->mutex}; + state = stream->open ? (stream->engaged ? 2 : 1) : 0; + } + LOG_DEBUG(Service_Audio, "stream {} state {}", stream->id, state); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(ResultSuccess); + rb.Push(state); + } + + void GetStreamGain(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(ResultSuccess); + rb.Push(gain); + } + + void SetStreamGain(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + gain = rp.Pop<f32>(); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void AcquireStateChangeEvent(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 2, 1}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, stream->state_event->GetReadableEvent()); + } + + void AcquireDataEvent(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto mode = rp.Pop<u32>(); + const auto threshold = rp.Pop<s32>(); + LOG_INFO(Service_Audio, "stream {} data event: mode {}, threshold {}", stream->id, mode, threshold); + { + // The game waits on this before its first write, so signal it at once when the ring + // already has room (it is empty right after OpenStream); the drain timer keeps it up. + std::scoped_lock lk{streams->mutex}; + stream->data_mode = mode; + stream->data_threshold = threshold; + if (mode != 0 && stream->open && static_cast<s64>(stream->frames) >= threshold) { + stream->data_event->Signal(system.Kernel()); + } + } + IPC::ResponseBuilder rb{ctx, 2, 1}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, stream->data_event->GetReadableEvent()); + } + + void EchoFrames(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto frames = rp.Pop<s32>(); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(ResultSuccess); + rb.Push<u32>(static_cast<u32>((std::max)(frames, 0))); + } + + void Succeed(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + KernelHelpers::ServiceContext service_context; + std::shared_ptr<Nx2SysmixStreams> streams; + std::shared_ptr<Nx2SysmixStream> stream; + f32 gain{1.0f}; +}; + +// The session audsmx:u command 0 returns. Commands: +// - 1(u32, u32 creation flags, u64 pid, u64 buffer address, u64 buffer size, name buffer): +// u32 stream id and the stream object. +// - 2 / 3(buffer of u32 stream ids): engage / disengage those streams. +class INx2SysmixUserSession final : public ServiceFramework<INx2SysmixUserSession> { +public: + explicit INx2SysmixUserSession(Core::System& system_) : ServiceFramework{system_, "nx2:audsmx:u:session"} { + // clang-format off + static const FunctionInfo functions[] = { + {1, &INx2SysmixUserSession::CreateStream, "CreateStream"}, + {2, &INx2SysmixUserSession::EngageStreams, "EngageStreams"}, + {3, &INx2SysmixUserSession::DisengageStreams, "DisengageStreams"}, + }; + // clang-format on + RegisterHandlers(functions); + drain_timer = Core::Timing::CreateEvent( + "nx2:audsmx:u drain", + [streams = streams, &system = system, owner = session_id](s64, std::chrono::nanoseconds) + -> std::optional<std::chrono::nanoseconds> { + Drain(system, *streams, owner); + return std::nullopt; + }); + system.CoreTiming().ScheduleLoopingEvent(DrainInterval, DrainInterval, drain_timer); + } + + ~INx2SysmixUserSession() override { + system.CoreTiming().UnscheduleEvent(drain_timer); + } + +private: + static constexpr std::chrono::nanoseconds DrainInterval{std::chrono::milliseconds{5}}; + static constexpr u64 ReadIndexOffset = 0x44; + static constexpr u64 WriteIndexOffset = 0x1004; + + // Takes up to one interval's worth of frames out of each engaged ring (frames owed while a ring + // is empty are dropped, as an underrun plays silence), and signals the data event of every open + // stream with at least its threshold of free frames, engaged or not: the game fills the ring + // before it engages the stream. + static void Drain(Core::System& system, Nx2SysmixStreams& state, u32 owner) { + std::scoped_lock lk{state.mutex}; + auto& memory = system.ApplicationMemory(); + for (auto& [id, stream] : state.streams) { + if (stream->owner != owner || !stream->open || stream->frames == 0 || !stream->buffer) { + continue; + } + const u32 capacity = stream->frames + 1; + const u32 write = memory.Read32(stream->buffer + WriteIndexOffset); + const u32 read = memory.Read32(stream->buffer + ReadIndexOffset); + if (write >= capacity || read >= capacity) { + continue; + } + u32 filled = (write + capacity - read) % capacity; + if (stream->engaged) { + stream->owed_frames += stream->sample_rate * + std::chrono::duration<double>(DrainInterval).count(); + const auto owed = static_cast<u32>(stream->owed_frames); + stream->owed_frames -= owed; + const u32 taken = (std::min)(filled, owed); + if (taken != 0) { + memory.Write32(stream->buffer + ReadIndexOffset, (read + taken) % capacity); + filled -= taken; + } + } + const u32 free_frames = capacity - 1 - filled; + if (stream->data_event && stream->data_mode != 0 && + static_cast<s64>(free_frames) >= stream->data_threshold) { + stream->data_event->Signal(system.Kernel()); + } + } + } + + void CreateStream(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto kind = rp.Pop<u32>(); + const auto flags = rp.Pop<u32>(); + [[maybe_unused]] const auto pid = rp.Pop<u64>(); + const auto buffer = rp.Pop<u64>(); + const auto size = rp.Pop<u64>(); + const auto name = Common::StringFromBuffer(ctx.ReadBuffer(0)); + auto stream = std::make_shared<Nx2SysmixStream>(); + stream->buffer = buffer; + stream->size = size; + { + std::scoped_lock lk{streams->mutex}; + stream->id = streams->next_id++; + stream->owner = session_id; + streams->streams.emplace(stream->id, stream); + } + LOG_INFO(Service_Audio, "stream {} '{}': kind {}, flags {:#x}, buffer {:#x} size {:#x}", + stream->id, name, kind, flags, buffer, size); + IPC::ResponseBuilder rb{ctx, 3, 0, 1}; + rb.Push(ResultSuccess); + rb.Push(stream->id); + rb.PushIpcInterface<INx2SysmixStreamSession>(ctx, system, streams, stream); + } + + void SetEngaged(HLERequestContext& ctx, bool engaged) { + const auto buffer = ctx.ReadBuffer(0); + std::vector<u32> ids(buffer.size() / sizeof(u32)); + std::memcpy(ids.data(), buffer.data(), ids.size() * sizeof(u32)); + { + std::scoped_lock lk{streams->mutex}; + for (const u32 id : ids) { + if (const auto it = streams->streams.find(id); it != streams->streams.end()) { + it->second->engaged = engaged; + it->second->owed_frames = 0; + if (it->second->state_event) { + it->second->state_event->Signal(system.Kernel()); + } + } + } + } + LOG_INFO(Service_Audio, "{} streams {}", engaged ? "engage" : "disengage", fmt::join(ids, ",")); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void EngageStreams(HLERequestContext& ctx) { + SetEngaged(ctx, true); + } + + void DisengageStreams(HLERequestContext& ctx) { + SetEngaged(ctx, false); + } + + static inline std::atomic<u32> next_session_id{1}; + const u32 session_id = next_session_id++; + std::shared_ptr<Nx2SysmixStreams> streams = Nx2SysmixStreamList(); + std::shared_ptr<Core::Timing::EventType> drain_timer; +}; + +// Switch 2 "audsmx:u": command 0 (16 bytes, the pid and a copy handle) returns the user session. +class INx2AudioSoundMixerService final : public ServiceFramework<INx2AudioSoundMixerService> { +public: + explicit INx2AudioSoundMixerService(Core::System& system_) + : ServiceFramework{system_, "nx2:audsmx:u"} { + // clang-format off + static const FunctionInfo functions[] = { + {0, &INx2AudioSoundMixerService::OpenUserSession, "OpenUserSession"}, + }; + // clang-format on + RegisterHandlers(functions); + } + +private: + void OpenUserSession(HLERequestContext& ctx) { + LOG_DEBUG(Service_Audio, "called"); + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface<INx2SysmixUserSession>(ctx, system); + } +}; + +} // namespace Service::Audio diff --git a/src/core/hle/service/caps/caps_su.cpp b/src/core/hle/service/caps/caps_su.cpp index 4ecfc840e4..b19e6b7a27 100644 --- a/src/core/hle/service/caps/caps_su.cpp +++ b/src/core/hle/service/caps/caps_su.cpp @@ -21,10 +21,10 @@ IScreenShotApplicationService::IScreenShotApplicationService( // clang-format off static const FunctionInfo functions[] = { {32, C<&IScreenShotApplicationService::SetShimLibraryVersion>, "SetShimLibraryVersion"}, - {201, nullptr, "SaveScreenShot"}, + {201, &IScreenShotApplicationService::StubReplyZeroed, "SaveScreenShot"}, {203, C<&IScreenShotApplicationService::SaveScreenShotEx0>, "SaveScreenShotEx0"}, {205, C<&IScreenShotApplicationService::SaveScreenShotEx1>, "SaveScreenShotEx1"}, - {210, nullptr, "SaveScreenShotEx2"}, + {210, &IScreenShotApplicationService::StubReplyZeroed, "SaveScreenShotEx2"}, }; // clang-format on diff --git a/src/core/hle/service/caps/caps_u.cpp b/src/core/hle/service/caps/caps_u.cpp index 6472b2bac6..eb4a10193e 100644 --- a/src/core/hle/service/caps/caps_u.cpp +++ b/src/core/hle/service/caps/caps_u.cpp @@ -20,17 +20,21 @@ IAlbumApplicationService::IAlbumApplicationService(Core::System& system_, static const FunctionInfo functions[] = { {32, C<&IAlbumApplicationService::SetShimLibraryVersion>, "SetShimLibraryVersion"}, {102, C<&IAlbumApplicationService::GetAlbumFileList0AafeAruidDeprecated>, "GetAlbumFileList0AafeAruidDeprecated"}, - {103, nullptr, "DeleteAlbumFileByAruid"}, - {104, nullptr, "GetAlbumFileSizeByAruid"}, - {105, nullptr, "DeleteAlbumFileByAruidForDebug"}, - {110, nullptr, "LoadAlbumScreenShotImageByAruid"}, - {120, nullptr, "LoadAlbumScreenShotThumbnailImageByAruid"}, - {130, nullptr, "PrecheckToCreateContentsByAruid"}, - {140, nullptr, "GetAlbumFileList1AafeAruidDeprecated"}, - {141, nullptr, "GetAlbumFileList2AafeUidAruidDeprecated"}, + {103, &IAlbumApplicationService::StubReplyZeroed, "DeleteAlbumFileByAruid"}, + {104, &IAlbumApplicationService::StubReplyZeroed, "GetAlbumFileSizeByAruid"}, + {105, &IAlbumApplicationService::StubReplyZeroed, "DeleteAlbumFileByAruidForDebug"}, + {110, &IAlbumApplicationService::StubReplyZeroed, "LoadAlbumScreenShotImageByAruid"}, + {120, &IAlbumApplicationService::StubReplyZeroed, "LoadAlbumScreenShotThumbnailImageByAruid"}, + {130, &IAlbumApplicationService::StubReplyZeroed, "PrecheckToCreateContentsByAruid"}, + {140, &IAlbumApplicationService::StubReplyZeroed, "GetAlbumFileList1AafeAruidDeprecated"}, + {141, &IAlbumApplicationService::StubReplyZeroed, "GetAlbumFileList2AafeUidAruidDeprecated"}, {142, C<&IAlbumApplicationService::GetAlbumFileList3AaeAruid>, "GetAlbumFileList3AaeAruid"}, - {143, nullptr, "GetAlbumFileList4AaeUidAruid"}, - {144, nullptr, "GetAllAlbumFileList3AaeAruid"}, + {143, &IAlbumApplicationService::StubReplyZeroed, "GetAlbumFileList4AaeUidAruid"}, + {144, &IAlbumApplicationService::StubReplyZeroed, "GetAllAlbumFileList3AaeAruid"}, + {145, &IAlbumApplicationService::StubReplyZeroed, "Unknown145"}, + {146, &IAlbumApplicationService::StubReplyZeroed, "Unknown146"}, + {147, &IAlbumApplicationService::StubReplyZeroed, "Unknown147"}, + {148, &IAlbumApplicationService::StubReplyZeroed, "Unknown148"}, {60002, nullptr, "OpenAccessorSessionForApplication"}, }; // clang-format on diff --git a/src/core/hle/service/hid/hid_server.cpp b/src/core/hle/service/hid/hid_server.cpp index 04b4663bc2..20118345a9 100644 --- a/src/core/hle/service/hid/hid_server.cpp +++ b/src/core/hle/service/hid/hid_server.cpp @@ -74,12 +74,12 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {70, C<&IHidServer::SetSixAxisSensorFusionParameters>, "SetSixAxisSensorFusionParameters"}, {71, C<&IHidServer::GetSixAxisSensorFusionParameters>, "GetSixAxisSensorFusionParameters"}, {72, C<&IHidServer::ResetSixAxisSensorFusionParameters>, "ResetSixAxisSensorFusionParameters"}, - {73, nullptr, "SetAccelerometerParameters"}, - {74, nullptr, "GetAccelerometerParameters"}, - {75, nullptr, "ResetAccelerometerParameters"}, - {76, nullptr, "SetAccelerometerPlayMode"}, - {77, nullptr, "GetAccelerometerPlayMode"}, - {78, nullptr, "ResetAccelerometerPlayMode"}, + {73, &IHidServer::StubReplyZeroed, "SetAccelerometerParameters"}, + {74, &IHidServer::StubReplyZeroed, "GetAccelerometerParameters"}, + {75, &IHidServer::StubReplyZeroed, "ResetAccelerometerParameters"}, + {76, &IHidServer::StubReplyZeroed, "SetAccelerometerPlayMode"}, + {77, &IHidServer::StubReplyZeroed, "GetAccelerometerPlayMode"}, + {78, &IHidServer::StubReplyZeroed, "ResetAccelerometerPlayMode"}, {79, C<&IHidServer::SetGyroscopeZeroDriftMode>, "SetGyroscopeZeroDriftMode"}, {80, C<&IHidServer::GetGyroscopeZeroDriftMode>, "GetGyroscopeZeroDriftMode"}, {81, C<&IHidServer::ResetGyroscopeZeroDriftMode>, "ResetGyroscopeZeroDriftMode"}, @@ -87,7 +87,7 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {83, C<&IHidServer::IsFirmwareUpdateAvailableForSixAxisSensor>, "IsFirmwareUpdateAvailableForSixAxisSensor"}, {84, C<&IHidServer::EnableSixAxisSensorUnalteredPassthrough>, "EnableSixAxisSensorUnalteredPassthrough"}, {85, C<&IHidServer::IsSixAxisSensorUnalteredPassthroughEnabled>, "IsSixAxisSensorUnalteredPassthroughEnabled"}, - {86, nullptr, "StoreSixAxisSensorCalibrationParameter"}, + {86, &IHidServer::StubReplyZeroed, "StoreSixAxisSensorCalibrationParameter"}, {87, C<&IHidServer::LoadSixAxisSensorCalibrationParameter>, "LoadSixAxisSensorCalibrationParameter"}, {88, C<&IHidServer::GetSixAxisSensorIcInformation>, "GetSixAxisSensorIcInformation"}, {89, C<&IHidServer::ResetIsSixAxisSensorDeviceNewlyAssigned>, "ResetIsSixAxisSensorDeviceNewlyAssigned"}, @@ -119,6 +119,13 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {134, C<&IHidServer::SetNpadAnalogStickUseCenterClamp>, "SetNpadAnalogStickUseCenterClamp"}, {135, C<&IHidServer::SetNpadCaptureButtonAssignment>, "SetNpadCaptureButtonAssignment"}, {136, C<&IHidServer::ClearNpadCaptureButtonAssignment>, "ClearNpadCaptureButtonAssignment"}, + {137, &IHidServer::StubReplyZeroed, "Unknown137"}, + {138, &IHidServer::StubReplyZeroed, "Unknown138"}, + {139, &IHidServer::StubReplyZeroed, "Unknown139"}, + {140, &IHidServer::StubReplyZeroed, "Unknown140"}, + {150, &IHidServer::StubReplyZeroed, "Unknown150"}, + {151, &IHidServer::StubReplyZeroed, "Unknown151"}, + {160, &IHidServer::StubReplyZeroed, "Unknown160"}, {200, C<&IHidServer::GetVibrationDeviceInfo>, "GetVibrationDeviceInfo"}, {201, C<&IHidServer::SendVibrationValue>, "SendVibrationValue"}, {202, C<&IHidServer::GetActualVibrationValue>, "GetActualVibrationValue"}, @@ -132,7 +139,9 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {210, C<&IHidServer::EndPermitVibrationSession>, "EndPermitVibrationSession"}, {211, C<&IHidServer::IsVibrationDeviceMounted>, "IsVibrationDeviceMounted"}, {212, C<&IHidServer::SendVibrationValueInBool>, "SendVibrationValueInBool"}, + {213, C<&IHidServer::Unknown213>, "Unknown213"}, {214, C<&IHidServer::SendNx2VibrationValues>, "SendNx2VibrationValues (guessed)"}, + {220, C<&IHidServer::Unknown220>, "Unknown220"}, {300, C<&IHidServer::ActivateConsoleSixAxisSensor>, "ActivateConsoleSixAxisSensor"}, {301, C<&IHidServer::StartConsoleSixAxisSensor>, "StartConsoleSixAxisSensor"}, {302, C<&IHidServer::StopConsoleSixAxisSensor>, "StopConsoleSixAxisSensor"}, @@ -141,19 +150,24 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {305, C<&IHidServer::StopSevenSixAxisSensor>, "StopSevenSixAxisSensor"}, {306, C<&IHidServer::InitializeSevenSixAxisSensor>, "InitializeSevenSixAxisSensor"}, {307, C<&IHidServer::FinalizeSevenSixAxisSensor>, "FinalizeSevenSixAxisSensor"}, - {308, nullptr, "SetSevenSixAxisSensorFusionStrength"}, - {309, nullptr, "GetSevenSixAxisSensorFusionStrength"}, + {308, &IHidServer::StubReplyZeroed, "SetSevenSixAxisSensorFusionStrength"}, + {309, &IHidServer::StubReplyZeroed, "GetSevenSixAxisSensorFusionStrength"}, {310, C<&IHidServer::ResetSevenSixAxisSensorTimestamp>, "ResetSevenSixAxisSensorTimestamp"}, - {320, nullptr, "EnableNpadImu"}, //21.0.0+ - {321, nullptr, "DisableNpadImu"}, //21.0.0+ + {311, &IHidServer::StubReplyZeroed, "Unknown311"}, + {312, &IHidServer::StubReplyZeroed, "Unknown312"}, + {320, &IHidServer::StubReplyZeroed, "EnableNpadImu"}, //21.0.0+ + {321, &IHidServer::StubReplyZeroed, "DisableNpadImu"}, //21.0.0+ + {322, &IHidServer::StubReplyZeroed, "Unknown322"}, + {323, &IHidServer::StubReplyZeroed, "Unknown323"}, + {324, &IHidServer::StubReplyZeroed, "Unknown324"}, {400, C<&IHidServer::IsUsbFullKeyControllerEnabled>, "IsUsbFullKeyControllerEnabled"}, - {401, nullptr, "EnableUsbFullKeyController"}, - {402, nullptr, "IsUsbFullKeyControllerConnected"}, - {403, nullptr, "HasBattery"}, - {404, nullptr, "HasLeftRightBattery"}, - {405, nullptr, "GetNpadInterfaceType"}, - {406, nullptr, "GetNpadLeftRightInterfaceType"}, - {407, nullptr, "GetNpadOfHighestBatteryLevel"}, + {401, &IHidServer::StubReplyZeroed, "EnableUsbFullKeyController"}, + {402, &IHidServer::StubReplyZeroed, "IsUsbFullKeyControllerConnected"}, + {403, &IHidServer::StubReplyZeroed, "HasBattery"}, + {404, &IHidServer::StubReplyZeroed, "HasLeftRightBattery"}, + {405, &IHidServer::StubReplyZeroed, "GetNpadInterfaceType"}, + {406, &IHidServer::StubReplyZeroed, "GetNpadLeftRightInterfaceType"}, + {407, &IHidServer::StubReplyZeroed, "GetNpadOfHighestBatteryLevel"}, {408, nullptr, "GetNpadOfHighestBatteryLevelForJoyRight"}, {500, C<&IHidServer::GetPalmaConnectionHandle>, "GetPalmaConnectionHandle"}, {501, C<&IHidServer::InitializePalma>, "InitializePalma"}, @@ -185,11 +199,13 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {527, C<&IHidServer::EnablePalmaBoostMode>, "EnablePalmaBoostMode"}, {528, C<&IHidServer::GetPalmaBluetoothAddress>, "GetPalmaBluetoothAddress"}, {529, C<&IHidServer::SetDisallowedPalmaConnection>, "SetDisallowedPalmaConnection"}, + {550, &IHidServer::StubReplyZeroed, "Unknown550"}, {1000, C<&IHidServer::SetNpadCommunicationMode>, "SetNpadCommunicationMode"}, {1001, C<&IHidServer::GetNpadCommunicationMode>, "GetNpadCommunicationMode"}, {1002, C<&IHidServer::SetTouchScreenConfiguration>, "SetTouchScreenConfiguration"}, {1003, C<&IHidServer::IsFirmwareUpdateNeededForNotification>, "IsFirmwareUpdateNeededForNotification"}, {1004, C<&IHidServer::SetTouchScreenResolution>, "SetTouchScreenResolution"}, + {1005, &IHidServer::StubReplyZeroed, "Unknown1005"}, {1270, nullptr, "DeleteButtonConfigStorageRight"}, {1271, nullptr, "IsUsingCustomButtonConfig"}, {1272, nullptr, "IsAnyCustomButtonConfigEnabled"}, @@ -222,23 +238,49 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {1325, nullptr, "IsTouchScreenAutoTuneRequiredForRepairProviderReplacement"}, {1420, nullptr, "GetAppletResourceProperty"}, {2000, nullptr, "ActivateDigitizer"}, - {3000, nullptr, "GetDebugPadGenericPadMap"}, - {3001, nullptr, "SetDebugPadGenericPadMap"}, - {3002, nullptr, "ResetDebugPadGenericPadMap"}, - {3003, nullptr, "GetDebugPadKeyboardMap"}, - {3004, nullptr, "SetDebugPadKeyboardMap"}, - {3005, nullptr, "ResetDebugPadKeyboardMap"}, - {3006, nullptr, "GetFullKeyGenericPadMap"}, - {3007, nullptr, "SetFullKeyGenericPadMap"}, - {3008, nullptr, "ResetFullKeyGenericPadMap"}, - {3009, nullptr, "GetFullKeyKeyboardMap"}, - {3010, nullptr, "SetFullKeyKeyboardMap"}, - {3011, nullptr, "ResetFullKeyKeyboardMap"}, - {3012, nullptr, "GetDebugPadGenericPadMap"}, //21.0.0+ - {3013, nullptr, "SetDebugPadGenericPadMap"}, //21.0.0+ - {3014, nullptr, "GetDebugPadKeyboardMap"}, //21.0.0+ - {3015, nullptr, "SetDebugPadKeyboardMap"}, //21.0.0+ + {3000, &IHidServer::StubReplyZeroed, "GetDebugPadGenericPadMap"}, + {3001, &IHidServer::StubReplyZeroed, "SetDebugPadGenericPadMap"}, + {3002, &IHidServer::StubReplyZeroed, "ResetDebugPadGenericPadMap"}, + {3003, &IHidServer::StubReplyZeroed, "GetDebugPadKeyboardMap"}, + {3004, &IHidServer::StubReplyZeroed, "SetDebugPadKeyboardMap"}, + {3005, &IHidServer::StubReplyZeroed, "ResetDebugPadKeyboardMap"}, + {3006, &IHidServer::StubReplyZeroed, "GetFullKeyGenericPadMap"}, + {3007, &IHidServer::StubReplyZeroed, "SetFullKeyGenericPadMap"}, + {3008, &IHidServer::StubReplyZeroed, "ResetFullKeyGenericPadMap"}, + {3009, &IHidServer::StubReplyZeroed, "GetFullKeyKeyboardMap"}, + {3010, &IHidServer::StubReplyZeroed, "SetFullKeyKeyboardMap"}, + {3011, &IHidServer::StubReplyZeroed, "ResetFullKeyKeyboardMap"}, + {3012, &IHidServer::StubReplyZeroed, "GetDebugPadGenericPadMap"}, //21.0.0+ + {3013, &IHidServer::StubReplyZeroed, "SetDebugPadGenericPadMap"}, //21.0.0+ + {3014, &IHidServer::StubReplyZeroed, "GetDebugPadKeyboardMap"}, //21.0.0+ + {3015, &IHidServer::StubReplyZeroed, "SetDebugPadKeyboardMap"}, //21.0.0+ + {3110, &IHidServer::StubReplyZeroed, "Unknown3110"}, + {3111, &IHidServer::StubReplyZeroed, "Unknown3111"}, + {3112, &IHidServer::StubReplyZeroed, "Unknown3112"}, + {3113, &IHidServer::StubReplyZeroed, "Unknown3113"}, + {3114, &IHidServer::StubReplyZeroed, "Unknown3114"}, + {3130, &IHidServer::StubReplyZeroed, "Unknown3130"}, + {3131, &IHidServer::StubReplyZeroed, "Unknown3131"}, + {3132, &IHidServer::StubReplyZeroed, "Unknown3132"}, + {3133, &IHidServer::StubReplyZeroed, "Unknown3133"}, + {3134, &IHidServer::StubReplyZeroed, "Unknown3134"}, + {3135, &IHidServer::StubReplyZeroed, "Unknown3135"}, + {3136, &IHidServer::StubReplyZeroed, "Unknown3136"}, + {3137, &IHidServer::StubReplyZeroed, "Unknown3137"}, + {3138, &IHidServer::StubReplyZeroed, "Unknown3138"}, + {3139, &IHidServer::StubReplyZeroed, "Unknown3139"}, + {3140, &IHidServer::StubReplyZeroed, "Unknown3140"}, + {3141, &IHidServer::StubReplyZeroed, "Unknown3141"}, + {3142, &IHidServer::StubReplyZeroed, "Unknown3142"}, + {3143, &IHidServer::StubReplyZeroed, "Unknown3143"}, + {3144, &IHidServer::StubReplyZeroed, "Unknown3144"}, + {3145, &IHidServer::StubReplyZeroed, "Unknown3145"}, + {3146, &IHidServer::StubReplyZeroed, "Unknown3146"}, + {3147, &IHidServer::StubReplyZeroed, "Unknown3147"}, + {3148, &IHidServer::StubReplyZeroed, "Unknown3148"}, + {3149, &IHidServer::StubReplyZeroed, "Unknown3149"}, {3150, C<&IHidServer::SetMouseLibraryVersion>, "SetMouseLibraryVersion"}, //21.0.0+ + {5000, &IHidServer::StubReplyZeroed, "Unknown5000"}, // What? -- {12010, nullptr, "SetButtonConfigLeft"}, }; // clang-format on @@ -1075,6 +1117,17 @@ Result IHidServer::SendNx2VibrationValues() { R_SUCCEED(); } +Result IHidServer::Unknown213() { + LOG_DEBUG(Service_HID, "(STUBBED) called"); + R_SUCCEED(); +} + +Result IHidServer::Unknown220(Out<u8> out) { + LOG_DEBUG(Service_HID, "(STUBBED) called"); + *out = 0; + R_SUCCEED(); +} + Result IHidServer::SendVibrationValueInBool( bool is_vibrating, Core::HID::VibrationDeviceHandle vibration_device_handle, ClientAppletResourceUserId aruid) { diff --git a/src/core/hle/service/hid/hid_server.h b/src/core/hle/service/hid/hid_server.h index 574d54c415..0767bb91b7 100644 --- a/src/core/hle/service/hid/hid_server.h +++ b/src/core/hle/service/hid/hid_server.h @@ -192,6 +192,12 @@ private: // Switch 2 sends vibration every frame with 214 (pointer buffers of device handles and // values, like 206); Eden doesn't emulate vibration, so accept and drop it. Result SendNx2VibrationValues(); + // 213 (32 bytes, aruid): sent per controller every frame by Kirby Air Riders (tens of + // thousands of auto stub log lines a run); no reply data. Accepted and dropped like 214. + Result Unknown213(); + // 220 (16 bytes, aruid): replies one byte (meaning unknown); answer 0 instead of leaving it to + // the message buffer's leftovers. + Result Unknown220(Out<u8> out); Result ActivateConsoleSixAxisSensor(ClientAppletResourceUserId aruid); Result StartConsoleSixAxisSensor(Core::HID::ConsoleSixAxisSensorHandle console_sixaxis_handle, ClientAppletResourceUserId aruid); diff --git a/src/core/hle/service/kernel_helpers.cpp b/src/core/hle/service/kernel_helpers.cpp index e4d6925f19..db55c6f81a 100644 --- a/src/core/hle/service/kernel_helpers.cpp +++ b/src/core/hle/service/kernel_helpers.cpp @@ -48,6 +48,17 @@ ServiceContext::~ServiceContext() { } } +namespace { +std::mutex debug_names_mutex; +std::unordered_map<const Kernel::KReadableEvent*, std::string> debug_names; +} // Anonymous namespace + +std::string DebugEventName(const Kernel::KReadableEvent* event) { + std::scoped_lock lk{debug_names_mutex}; + const auto it = debug_names.find(event); + return it != debug_names.end() ? it->second : std::string{"?"}; +} + Kernel::KEvent* ServiceContext::CreateEvent(std::string&& name) { // Reserve a new event from the process resource limit Kernel::KScopedResourceReservation event_reservation(kernel, process, @@ -74,6 +85,10 @@ Kernel::KEvent* ServiceContext::CreateEvent(std::string&& name) { Kernel::KEvent::Register(kernel, event); { + std::scoped_lock lk{debug_names_mutex}; + debug_names[&event->GetReadableEvent()] = name; + } + { std::scoped_lock lk{event_names_mutex}; event_names[event] = std::move(name); } @@ -86,6 +101,10 @@ void ServiceContext::CloseEvent(Kernel::KEvent* event) { std::scoped_lock lk{event_names_mutex}; event_names.erase(event); } + { + std::scoped_lock lk{debug_names_mutex}; + debug_names.erase(&event->GetReadableEvent()); + } event->GetReadableEvent().Close(kernel); event->Close(kernel); } diff --git a/src/core/hle/service/kernel_helpers.h b/src/core/hle/service/kernel_helpers.h index a62dd838aa..b03e49300f 100644 --- a/src/core/hle/service/kernel_helpers.h +++ b/src/core/hle/service/kernel_helpers.h @@ -16,6 +16,7 @@ namespace Kernel { class KernelCore; class KEvent; class KProcess; +class KReadableEvent; } // namespace Kernel namespace Service::KernelHelpers { @@ -37,4 +38,7 @@ public: bool process_created{false}; }; +/// The name a service gave an event (EDEN_NX2_SVC_TRACE shows the handles a thread waits on). +std::string DebugEventName(const Kernel::KReadableEvent* event); + } // namespace Service::KernelHelpers diff --git a/src/core/hle/service/lm/lm.cpp b/src/core/hle/service/lm/lm.cpp index d631a6dd80..a7d6d6e707 100644 --- a/src/core/hle/service/lm/lm.cpp +++ b/src/core/hle/service/lm/lm.cpp @@ -91,8 +91,9 @@ public: static const FunctionInfo functions[] = { {0, &ILogger::Log, "Log"}, {1, &ILogger::SetDestination, "SetDestination"}, - {2, nullptr, "TransmitHashedLog"}, //20.0.0+ - {3, nullptr, "DevNotify"}, //20.0.0+ + {2, &ILogger::StubReplyZeroed, "TransmitHashedLog"}, //20.0.0+ + {3, &ILogger::StubReplyZeroed, "DevNotify"}, //20.0.0+ + {11, &ILogger::StubReplyInterface, "Unknown11"}, }; RegisterHandlers(functions); } diff --git a/src/core/hle/service/mii/mii.cpp b/src/core/hle/service/mii/mii.cpp index 513bb671f8..ffdb17fd14 100644 --- a/src/core/hle/service/mii/mii.cpp +++ b/src/core/hle/service/mii/mii.cpp @@ -48,8 +48,8 @@ public: {15, D<&IDatabaseService::DestroyFile>, "DestroyFile"}, {16, D<&IDatabaseService::DeleteFile>, "DeleteFile"}, {17, D<&IDatabaseService::Format>, "Format"}, - {18, nullptr, "Import"}, - {19, nullptr, "Export"}, + {18, &IDatabaseService::StubReplyZeroed, "Import"}, + {19, &IDatabaseService::StubReplyZeroed, "Export"}, {20, D<&IDatabaseService::IsBrokenDatabaseWithClearFlag>, "IsBrokenDatabaseWithClearFlag"}, {21, D<&IDatabaseService::GetIndex>, "GetIndex"}, {22, D<&IDatabaseService::SetInterfaceVersion>, "SetInterfaceVersion"}, @@ -57,6 +57,8 @@ public: {24, D<&IDatabaseService::ConvertCoreDataToCharInfo>, "ConvertCoreDataToCharInfo"}, {25, D<&IDatabaseService::ConvertCharInfoToCoreData>, "ConvertCharInfoToCoreData"}, {26, D<&IDatabaseService::Append>, "Append"}, + {27, &IDatabaseService::StubReplyZeroed, "Unknown27"}, + {28, &IDatabaseService::StubReplyZeroed, "Unknown28"}, }; // clang-format on diff --git a/src/core/hle/service/ns/platform_service_manager.cpp b/src/core/hle/service/ns/platform_service_manager.cpp index 88aac56b1c..7bd8980a08 100644 --- a/src/core/hle/service/ns/platform_service_manager.cpp +++ b/src/core/hle/service/ns/platform_service_manager.cpp @@ -85,8 +85,12 @@ struct IPlatformServiceManager::Impl { // Automatically populated based on shared_fonts dump or system archives. // 6 builtin fonts + extra 2 for whatever may come after boost::container::static_vector<FontRegion, 8> shared_font_regions; - /// Plain TTF copies of the same fonts (Switch 2 GetBuiltInFontData), after the BFTTF data. + /// Where Switch 2 GetBuiltInFontData finds the same fonts as plain TTF data: page-aligned + /// copies in builtin_font_memory (the Switch 1 font memory is full with the system fonts). boost::container::static_vector<FontRegion, 8> builtin_font_regions; + u32 builtin_font_memory_size{}; + /// Made by the first GetBuiltInFontSharedMemory (Switch 2 titles only). + Kernel::KSharedMemory* builtin_font_memory{}; /// Backing memory for the shared font data std::array<u8, SHARED_FONT_MEM_SIZE> shared_font; }; @@ -122,7 +126,6 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch // Attempt to load shared font data from disk const auto* nand = fsc.GetSystemNANDContents(); std::size_t offset = 0; - std::vector<std::vector<u8>> plain_fonts; // Rebuild shared fonts from data ncas or synthesize for (auto& font : SHARED_FONTS) { FileSys::VirtualFile romfs; @@ -158,34 +161,52 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch Common::swap32); // Font offset and size do not account for the header const FontRegion region{u32(offset + 8), u32((font_data_u32.size() * sizeof(u32)) - 8)}; - std::vector<u8> plain(region.size); - DecryptSharedFontToTTF(font_data_u32, plain); - plain_fonts.push_back(std::move(plain)); DecryptSharedFont(font_data_u32, impl->shared_font, offset); impl->shared_font_regions.push_back(region); } - for (const auto& plain : plain_fonts) { - offset = Common::AlignUp(offset, 0x1000); - if (offset + plain.size() > SHARED_FONT_MEM_SIZE) { - LOG_WARNING(Service_NS, "No shared memory left for plain built-in font copies"); - break; - } - std::memcpy(impl->shared_font.data() + offset, plain.data(), plain.size()); - impl->builtin_font_regions.push_back({u32(offset), u32(plain.size())}); - offset += plain.size(); + // The data after each BFTTF header is the plain font: GetBuiltInFontSharedMemory copies it. + u32 builtin_offset = 0; + for (const auto& region : impl->shared_font_regions) { + impl->builtin_font_regions.push_back({builtin_offset, region.size}); + builtin_offset = Common::AlignUp(builtin_offset + region.size, 0x1000); } + impl->builtin_font_memory_size = std::max<u32>(builtin_offset, 0x1000); } // Switch 2: the shared memory GetBuiltInFontData offsets point into (copy handle, u32 size); the -// SDK maps it read-only. Eden serves its font shared memory, which holds plain copies too. +// SDK maps it read-only. It holds plain copies of the fonts, apart from the Switch 1 font memory. void IPlatformServiceManager::GetBuiltInFontSharedMemory(HLERequestContext& ctx) { - LOG_WARNING(Service_NS, "(STUBBED) Switch 2 GetBuiltInFontSharedMemory"); - std::memcpy(kernel.GetFontSharedMem().GetPointer(), impl->shared_font.data(), impl->shared_font.size()); + LOG_WARNING(Service_NS, "(STUBBED) Switch 2 GetBuiltInFontSharedMemory, {:#x} bytes", + impl->builtin_font_memory_size); + if (impl->builtin_font_memory == nullptr) { + auto* const memory = Kernel::KSharedMemory::Create(kernel); + if (const Result result = memory->Initialize(kernel, system.DeviceMemory(), nullptr, + Kernel::Svc::MemoryPermission::None, + Kernel::Svc::MemoryPermission::Read, + impl->builtin_font_memory_size); + result.IsError()) { + LOG_ERROR(Service_NS, "Failed to make {:#x} bytes of built-in font memory: {:#x}", + impl->builtin_font_memory_size, result.raw); + memory->Close(kernel); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result); + return; + } + Kernel::KSharedMemory::Register(kernel, memory); + u8* const data = memory->GetPointer(); + std::memset(data, 0, impl->builtin_font_memory_size); + for (size_t i = 0; i < impl->builtin_font_regions.size(); ++i) { + std::memcpy(data + impl->builtin_font_regions[i].offset, + impl->shared_font.data() + impl->shared_font_regions[i].offset, + impl->builtin_font_regions[i].size); + } + impl->builtin_font_memory = memory; + } IPC::ResponseBuilder rb{ctx, 3, 1}; rb.Push(ResultSuccess); - rb.Push<u32>(static_cast<u32>(SHARED_FONT_MEM_SIZE)); - rb.PushCopyObjects(ctx, kernel.GetFontSharedMem()); + rb.Push<u32>(impl->builtin_font_memory_size); + rb.PushCopyObjects(ctx, *impl->builtin_font_memory); } // nn::pl::GetBuiltInFontData(BuiltInFontDataOunce*, BuiltInFontTypeOunce, bool): returns a 16-byte @@ -212,7 +233,11 @@ void IPlatformServiceManager::GetBuiltInFontData(HLERequestContext& ctx) { rb.Push<u32>(0); } -IPlatformServiceManager::~IPlatformServiceManager() = default; +IPlatformServiceManager::~IPlatformServiceManager() { + if (impl->builtin_font_memory != nullptr) { + impl->builtin_font_memory->Close(kernel); + } +} Result IPlatformServiceManager::RequestLoad(SharedFontType type) { // Games don't call this so all fonts should be loaded diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index 9d696c027e..40313a10fb 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -174,7 +174,11 @@ void nvhost_as_gpu::Nx2Map(u64 gpu_address, u64 cpu_address, u64 size) { const DAddr device_address = range.address; nx2_device_ranges[{cpu_address, size}] = range; Nx2GpuMemoryRegistry::Get().RecordDeviceRange(cpu_address, size, device_address); - gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false); + // On the GPU thread, like unmaps: a GMMU map takes the texture cache lock, and the callers hold + // nx2:gpu locks that the GPU thread takes under that lock (sparse blocks backed on write). + system.GPU().RunInOrder([gmmu = gmmu.get(), gpu_address, device_address, size] { + gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false); + }); LOG_DEBUG(Service_NVDRV, "Switch 2 GPU mapping {:#x}+{:#x} -> CPU {:#x}", gpu_address, size, cpu_address); } @@ -248,7 +252,10 @@ void nvhost_as_gpu::Nx2Sparse(u64 gpu_address, u64 size, bool create) { return; } nx2_sparse_ranges[gpu_address] = {size, device_address}; - gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false); + // On the GPU thread, as in Nx2Map. + system.GPU().RunInOrder([gmmu = gmmu.get(), gpu_address, device_address, size] { + gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false); + }); LOG_DEBUG(Service_NVDRV, "Switch 2 sparse region {:#x}+{:#x} -> device {:#x}", gpu_address, size, device_address); } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp index cf4a473496..93d51cd538 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp @@ -228,9 +228,11 @@ void ApplyNx2Characteristics(GC& gc) { if (remap) { gc.flags |= 1ULL << 35; } - // Bits 40..43: the 2D, 3D, compute and inline-to-memory class families. The SDK filters its - // class enumeration by them; without them Fire Emblem: Fortune's Weave aborts at boot. - gc.flags |= 0xfULL << 40; + // Engine support bits 40-44 (2D, 3D, compute, inline-to-memory, ZBC). The SDK filters its class + // enumeration by them, and NvRm's newer device path lists a class family in the device info + // only when its bit is set; without them Fire Emblem: Fortune's Weave aborts at boot and a + // device opened that way has no compute class (Fire Emblem's CUDA driver reports no device). + gc.flags |= 0x1FULL << 40; // The Switch 2 layout keeps nine s16 interface versions at 0x70..0x80 (like nvgpu's // *_ioctl_nr_last); its NvRm requires e.g. [0x78] >= 6 and enables features above 0xa, 0x1f diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp index 0cd72faed3..848c15d6f8 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp @@ -24,6 +24,7 @@ #include "video_core/engines/puller.h" #include "video_core/gpu.h" #include "video_core/host1x/host1x.h" +#include "video_core/memory_manager.h" namespace Service::Nvidia::Devices { namespace { @@ -435,6 +436,30 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL entries.submit_hashes.push_back(Tegra::HashPushbufferWords(words)); } } + // The present returns before the GPU thread reaches this frame's work (nx2_gsv.h, + // frames ahead), and Mario Kart World then refills command memory it already + // submitted: the end of a frame ran with the next frame's present image as its render + // target, leaving an image three frames old to be shown. Keep the words as submitted. + // EDEN_NX2_PB_SNAPSHOT=0: execute from memory. + static const bool snapshot = [] { + const char* value = std::getenv("EDEN_NX2_PB_SNAPSHOT"); + return value == nullptr || value[0] != '0'; + }(); + if (snapshot && channel_state->memory_manager) { + size_t words = 0; + for (const auto& entry : entries.command_lists) { + words += entry.size; + } + entries.submitted_words.resize(words); + entries.submitted_offsets.reserve(entries.command_lists.size()); + size_t offset = 0; + for (const auto& entry : entries.command_lists) { + entries.submitted_offsets.push_back(static_cast<u32>(offset)); + channel_state->memory_manager->ReadBlockUnsafe( + entry.addr, entries.submitted_words.data() + offset, entry.size * sizeof(u32)); + offset += entry.size; + } + } } auto& flags = params.flags; diff --git a/src/core/hle/service/nvdrv/nvdrv.cpp b/src/core/hle/service/nvdrv/nvdrv.cpp index 12923496fb..7853d17235 100644 --- a/src/core/hle/service/nvdrv/nvdrv.cpp +++ b/src/core/hle/service/nvdrv/nvdrv.cpp @@ -133,6 +133,7 @@ Module::Module(Core::System& system) Devices::Nx2GpuMemoryRegistry::Get().Reset(); Tegra::Nx2ClearSemaphoreRegions(); Tegra::Nx2ClearSparseRegions(); + Tegra::Nx2ClearGpuWrittenPages(); Devices::g_nx2_driver = false; Tegra::g_nx2_ampere = false; Core::g_nx2_present_count = 0; diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.cpp b/src/core/hle/service/nvdrv/nvdrv_interface.cpp index 020036d74b..2ee0d726a2 100644 --- a/src/core/hle/service/nvdrv/nvdrv_interface.cpp +++ b/src/core/hle/service/nvdrv/nvdrv_interface.cpp @@ -13,6 +13,7 @@ #include "core/hle/kernel/k_process.h" #include "core/hle/kernel/k_readable_event.h" #include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/hle/service/nvdrv/nvdata.h" #include "core/hle/service/nvdrv/nvdrv.h" #include "core/hle/service/nvdrv/nvdrv_interface.h" @@ -36,7 +37,10 @@ void NVDRV::Open(HLERequestContext& ctx) { const std::string device_name(Common::StringFromBuffer(buffer)); LOG_DEBUG(Service_NVDRV, "device_name={}", device_name); - if (device_name == "/dev/nvhost-prof-gpu") { + // Switch 2 lists nvhost-prof-gpu with the other GPU nodes and the CUDA driver (Fire + // Emblem's subsdk1) opens it, so only refuse it to Switch 1 callers. Switch 2 opens get + // the logged stub device. + if (device_name == "/dev/nvhost-prof-gpu" && !Devices::g_nx2_driver) { rb.Push<DeviceFD>(0); rb.PushEnum(NvResult::NotSupported); diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h index 5a2963f179..37c797fa3f 100644 --- a/src/core/hle/service/nx2_gsv.h +++ b/src/core/hle/service/nx2_gsv.h @@ -107,6 +107,14 @@ inline void TipcReplyWords(HLERequestContext& ctx, std::initializer_list<u32> wo TipcReplyWords(ctx, std::span<const u32>{words.begin(), words.size()}); } +// TIPC 0(u64 token) on the Switch 2 sessions: a handshake newer SDKs send first; the reply echoes +// the token. (The auto stub's reply only echoed it by leaving the request's words in place.) +inline void TipcHandshake(HLERequestContext& ctx, std::string_view session) { + const auto* w = ctx.CommandBuffer(); + LOG_INFO(Service, "{} TIPC 0 handshake {:#x}", session, u64{w[2]} | (u64{w[3]} << 32)); + TipcReplyWords(ctx, {w[2], w[3]}); +} + // Replies with success and a copied handle to a new session served by `iface`. template <class T> inline void ReturnRawSession(Core::System& system, HLERequestContext& ctx, std::shared_ptr<T> iface) { @@ -144,6 +152,8 @@ public: {103, &INx2SyncptSession::QueryCounter, "QueryCounter"}, {201, &INx2SyncptSession::RequestSlotEvent, "RequestSlotEvent (guessed)"}, {202, &INx2SyncptSession::CancelSlotEvent, "CancelSlotEvent (guessed)"}, + // Firmware commands without a known meaning: zeroed replies. + {21, &INx2SyncptSession::StubReplyZeroed, "Unknown21"}, }); } @@ -165,9 +175,7 @@ public: private: // Newer SDKs (title 1) open with TIPC 0(u64 token) and abort unless the reply echoes it. void Handshake(HLERequestContext& ctx) { - const auto* w = ctx.CommandBuffer(); - LOG_INFO(Service, "nx2:syncpt TIPC 0 handshake {:#x}", u64{w[2]} | (u64{w[3]} << 32)); - TipcReplyWords(ctx, {w[2], w[3]}); + TipcHandshake(ctx, "nx2:syncpt"); } void GetCount(HLERequestContext& ctx) { @@ -304,6 +312,7 @@ public: explicit INx2GpuSession(Core::System& system_) : ServiceFramework{system_, "nx2:gpu"}, m_context{system_, "nx2:gpu"} { RegisterHandlersTipc({ + {0, &INx2GpuSession::Handshake, "Handshake"}, {11, &INx2GpuSession::GetIdRange, "GetIdRange (guessed)"}, {12, &INx2GpuSession::GetIdEvent, "GetIdEvent (guessed)"}, {21, &INx2GpuSession::Start, "Start (guessed)"}, @@ -323,10 +332,22 @@ public: {214, &INx2GpuSession::SubCreateGpuOnlyRegion, "CreateSparseRegion"}, {215, &INx2GpuSession::SubDestroySparseRegion, "DestroySparseRegion"}, {216, &INx2GpuSession::SubSparseMap, "ExecuteSparseMapRequests"}, + // Firmware commands without a known meaning: zeroed replies. + {23, &INx2GpuSession::StubReplyZeroed, "Unknown23"}, + {24, &INx2GpuSession::StubReplyZeroed, "Unknown24"}, + {25, &INx2GpuSession::StubReplyZeroed, "Unknown25"}, + {34, &INx2GpuSession::StubReplyZeroed, "Unknown34"}, + {51, &INx2GpuSession::StubReplyZeroed, "Unknown51"}, + {8001, &INx2GpuSession::StubReplyZeroed, "Unknown8001"}, + {8002, &INx2GpuSession::StubReplyZeroed, "Unknown8002"}, + {8003, &INx2GpuSession::StubReplyZeroed, "Unknown8003"}, + {8004, &INx2GpuSession::StubReplyZeroed, "Unknown8004"}, }); if (LazySparseRegions()) { Tegra::Nx2SetSparseBacking( - [this](u64 gpu_address, u64 size) { BackSparseWrite(gpu_address, size); }); + [this](u64 gpu_address, u64 size, const u8* data) { + BackSparseWrite(gpu_address, size, data); + }); } } @@ -341,6 +362,10 @@ public: } private: + void Handshake(HLERequestContext& ctx) { + TipcHandshake(ctx, "nx2:gpu"); + } + static constexpr u32 NumIds = 32; Kernel::KEvent* EventFor(u32 id) { @@ -588,12 +613,29 @@ private: // backed whole, they ran the pool dry and the later ones got no memory at all. // The region gets a device range of its own, unbacked pages read zero, and // BackSparseWrite gives a block memory before the GPU writes into it. - m_sparse_regions[start] = {size, std::vector<u8>(Common::DivCeil(size, SparseBlockSize))}; + m_sparse_regions[start] = { + .size = size, + .blocks = std::vector<u8>(Common::DivCeil(size, SparseBlockSize)), + }; Nvidia::Devices::Nx2GpuMemoryRegistry::Get().CreateSparse(start, size); Tegra::Nx2SparseRegionCreated(start, size, SparseBlockSize); Tegra::Nx2AddSparseSpan(start, size); LOG_INFO(Service, "nx2:gpu 214 sparse region [{:#x}, {:#x}] backed as the GPU writes it", start, last); + } else if (LazyGpuOnlyRegions()) { + // The rest is written by the host GPU, not through guest memory (title 1's and + // title 4's render targets, 285-300 MB regions): like a sparse region it gets a + // device range of its own and memory per block only when something writes guest + // memory there (downloads, copies). The texture cache sees ordinary memory. + m_sparse_regions[start] = { + .gpu_only = true, + .size = size, + .blocks = std::vector<u8>(Common::DivCeil(size, SparseBlockSize)), + }; + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().CreateSparse(start, size); + Tegra::Nx2AddSparseSpan(start, size); + LOG_INFO(Service, "nx2:gpu 214 GPU-only region [{:#x}, {:#x}] backed as it is written", + start, last); } else if (const auto address = BackGpuOnlyRegion(start, size)) { Nvidia::Devices::Nx2GpuMemoryRegistry::Get().MapAt(start, *address, size); LOG_INFO(Service, "nx2:gpu 214 GPU-only region [{:#x}, {:#x}] backed at {:#x} " @@ -702,6 +744,23 @@ private: TipcReplyWords(ctx, {static_cast<u32>(count), static_cast<u32>(u64{count} >> 32)}); } + static bool IsAllZero(const u8* data, size_t size) { + size_t i = 0; + for (; i + sizeof(u64) <= size; i += sizeof(u64)) { + u64 word; + std::memcpy(&word, data + i, sizeof(word)); + if (word != 0) { + return false; + } + } + for (; i < size; ++i) { + if (data[i] != 0) { + return false; + } + } + return true; + } + // Returns the pages of destroyed regions the GPU thread is done with to the kernel. void FreeReleasedRegions() { auto* const process = system.ApplicationProcess(); @@ -809,7 +868,7 @@ private: // usually on the GPU thread): the blocks the write touches get memory, cleared, behind the // region's device pages. The GPU mapping stays as it is, so the images over the region keep // their storage and only see the write. - void BackSparseWrite(u64 gpu_address, u64 size) { + void BackSparseWrite(u64 gpu_address, u64 size, const u8* data) { struct Pending { u64 region; u64 block; @@ -832,6 +891,7 @@ private: } }; bool waits_for_other = false; + u64 zero_bytes = 0; { std::unique_lock lk{m_region_mutex}; for_regions([&](u64 start, SparseRegion& region, u64 first, u64 last) { @@ -842,6 +902,18 @@ private: region.blocks[block] = BlockWritten; Tegra::Nx2SparseBlockWritten(start, block); } else if (region.blocks[block] == BlockEmpty) { + // Zeros over a block nobody wrote change nothing: it reads zero without + // memory. Title 1 clears ~1 GB of them at boot. + const u64 block_begin = start + block * SparseBlockSize; + const u64 write_begin = (std::max)(gpu_address, block_begin); + const u64 write_end = (std::min)( + {end, block_begin + SparseBlockSize, start + region.size}); + if (data != nullptr && + IsAllZero(data + (write_begin - gpu_address), write_end - write_begin) && + !Tegra::Nx2SparseBlockIsWritten(start, block)) { + zero_bytes += write_end - write_begin; + continue; + } region.blocks[block] = BlockPending; todo.push_back({start, block}); } else if (region.blocks[block] == BlockPending) { @@ -850,6 +922,16 @@ private: } }); } + if (zero_bytes != 0) { + std::scoped_lock lk{m_region_mutex}; + const u64 before = m_sparse_zero_bytes; + m_sparse_zero_bytes += zero_bytes; + if (before == 0 || (before >> 28) != (m_sparse_zero_bytes >> 28)) { + LOG_INFO(Service, "nx2:gpu {} MiB of zeros over unwritten sparse blocks left out " + "(they read zero without memory)", + m_sparse_zero_bytes >> 20); + } + } // Allocating and mapping happens outside the lock (see AllocateGpuOnlyBacking). for (const auto& [start, block] : todo) { u64 block_size = SparseBlockSize; @@ -933,6 +1015,15 @@ private: return lazy; } + // EDEN_NX2_GPU_ONLY_LAZY=0 backs GPU-only regions whole, as before. + static bool LazyGpuOnlyRegions() { + static const bool lazy = [] { + const char* value = std::getenv("EDEN_NX2_GPU_ONLY_LAZY"); + return value == nullptr || value[0] != '0'; + }(); + return lazy; + } + static constexpr u32 SparseRegionFlag = 0x200; ///< Word 7 of 214 (0x60c arrays, 0xa0c pools) static constexpr u64 SparseBlockSize = 4ULL << 20; static constexpr u8 BlockEmpty = 0; @@ -941,6 +1032,7 @@ private: static constexpr u8 BlockPending = 3; ///< Being allocated and mapped by BackSparseWrite static constexpr u8 BlockWritten = 4; ///< Written in a region the game maps tiles into struct SparseRegion { + bool gpu_only{}; ///< A GPU-only region (no sparse flag), backed lazily as well u64 size; std::vector<u8> blocks; ///< BlockEmpty / Backed / Failed / Pending / Written per block bool warned{}; ///< Logged that the pool ran out for its blocks @@ -950,6 +1042,7 @@ private: std::map<u64, SparseRegion> m_sparse_regions; ///< By GPU address u64 m_sparse_blocks{}; ///< Sparse blocks backed so far u64 m_sparse_backed_bytes{}; ///< Sparse region memory given out so far + u64 m_sparse_zero_bytes{}; ///< Zeros written over unwritten sparse blocks, left out u64 m_sparse_released_bytes{}; ///< Blocks given back once the game mapped tiles there // nn::gpu::sdk::SparseMapRequest (216), filled from NvRm's nvgpu_as_remap_op. @@ -1284,6 +1377,46 @@ private: std::map<u32, GpuMemory> m_memory; u32 m_next_memory_handle{1}; std::array<bool, 64> m_devices{}; // gpu 41 graphics device ids (0 unused) + + // Every 30 s: the backing of sparse blocks and GPU-only regions is marked cold. The game + // writes it once (texture uploads by 2D blits: title 1 fills 2-4 GB of texture pools) and it + // is read again only when the texture cache builds an image over it, so the OS may compress or + // page it out before anything else. Last member: stops before the regions are destroyed. + void HintColdBacking() { + struct Range { + u64 physical_offset; + u64 virtual_address; + u64 size; + }; + std::vector<Range> ranges; + { + std::scoped_lock lk{m_region_mutex}; + for (const auto& [gpu_address, region] : m_gpu_only_regions) { + u64 virtual_address = region.cpu_address; + for (const auto& block : *region.pages) { + ranges.push_back({GetInteger(block.GetAddress()) - Core::DramMemoryMap::Base, + virtual_address, block.GetSize()}); + virtual_address += block.GetSize(); + } + } + } + auto& buffer = system.DeviceMemory().buffer; + for (const Range& range : ranges) { + buffer.HintCold(range.physical_offset, range.size, range.virtual_address); + } + } + std::jthread m_cold_hint_thread{[this](std::stop_token stop) { + Common::SetCurrentThreadName("nx2:ColdHint"); + Common::SetCurrentThreadPriority(Common::ThreadPriority::Low); + while (!stop.stop_requested()) { + for (int i = 0; i < 30 && !stop.stop_requested(); ++i) { + std::this_thread::sleep_for(std::chrono::seconds{1}); + } + if (!stop.stop_requested()) { + HintColdBacking(); + } + } + }}; }; // Native memory and images handed out by the Switch 2 graphics server (gsv 111/114/423/429/315), @@ -1389,6 +1522,34 @@ private: std::map<u32, Kernel::KEvent*> m_events; }; +// nn::camera::sdk camera context (gsv 602). GameChat builds its local camera video storage from it +// and polls it every frame. Nothing about the commands is known beyond the arguments seen (3 and +// 21 take the camera name "Default", 29 a u32 0x180 = 384, the width of the 384x216 camera images), +// and the game runs on an empty success, so they answer that without a log line per call. +class INx2CameraSession final : public ServiceFramework<INx2CameraSession> { +public: + explicit INx2CameraSession(Core::System& system_) : ServiceFramework{system_, "nx2:camera"} { + RegisterHandlersTipc({ + {3, &INx2CameraSession::Stub, "Unknown3 (camera name \"Default\")"}, + {21, &INx2CameraSession::Stub, "Unknown21 (camera name \"Default\")"}, + {29, &INx2CameraSession::Stub, "Unknown29 (u32 0x180, polled every frame)"}, + }); + } + +private: + void Stub(HLERequestContext& ctx) { + const u32 command = ctx.GetCommand(); + if (m_logged.insert(command).second) { + LOG_WARNING(Service, "(STUBBED) nx2:camera TIPC {} [2]={:#x} [3]={:#x}, returning success", + command, ctx.CommandBuffer()[2], ctx.CommandBuffer()[3]); + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + std::set<u32> m_logged; +}; + // Video decoder output images (nn::codec image pools, gsv 991/911/961). The preference // (MakeDecodeImagePreference) holds an output layout code at word 1 and (width - 1) | // (height - 1) << 16 at word 2. Codes come in pairs, one per CodecMode: 1/4 NV12 (format 3, what @@ -1443,20 +1604,73 @@ public: : ServiceFramework{system_, "nx2:vi2"}, m_context{system_, "nx2:vi2"}, m_objects{std::move(objects_)} { RegisterHandlersTipc({ + {0, &INx2Vi2Session::Handshake, "Handshake"}, {1, &INx2Vi2Session::ReturnOne, "Unknown1"}, {3, &INx2Vi2Session::ReturnOne, "GetDisplayCount (guessed)"}, {5, &INx2Vi2Session::GetTimestampRecorderCount, "GetTimestampRecorderCount (guessed)"}, {11, &INx2Vi2Session::ReturnEvent, "Unknown11"}, - {31, &INx2Vi2Session::GetVsyncEvent, "GetVsyncEvent (guessed)"}, - {32, &INx2Vi2Session::GetVsyncEvent, "GetDisplayVsyncEvent (guessed)"}, - {104, &INx2Vi2Session::GetHdrStatus, "GetHdrStatus"}, - // Title 1 calls this ~75 times a second with the display name "default"; the - // auto stub's empty success reply works, but logged two lines per call (a 50 MB log - // in an 18 minute session). - {105, &INx2Vi2Session::SucceedQuietly, "Unknown105 (display \"default\")"}, + {31, &INx2Vi2Session::AcquireNativeEvent, "AcquireNativeEvent"}, + {32, &INx2Vi2Session::DuplicateNativeEvent, "DuplicateNativeEvent"}, + {261, &INx2Vi2Session::BindVsync, "BindDisplayVsyncEvent"}, + {262, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayVsyncEvent"}, + {263, &INx2Vi2Session::BindChange, "BindDisplayRefreshRateEvent"}, + {264, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayRefreshRateEvent"}, + {265, &INx2Vi2Session::BindChange, "BindDisplayHdrStatusEvent"}, + {266, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayHdrStatusEvent"}, + {267, &INx2Vi2Session::BindChange, "BindDisplayHdrPaperWhiteEvent"}, + {268, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayHdrPaperWhiteEvent"}, + {269, &INx2Vi2Session::BindChange, "BindDisplayBufferMaxResolutionEvent"}, + {270, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayBufferMaxResolutionEvent"}, + {271, &INx2Vi2Session::BindChange, "BindDisplayModeCapabilityEvent"}, + {272, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayModeCapabilityEvent"}, + // Title 1 calls these ~75 times a second with the display name "default"; quiet + // handlers (the auto stub logged two lines per call: a 50 MB log in an 18 minute + // session). + {103, &INx2Vi2Session::QueryDisplayRefreshRate, "QueryDisplayRefreshRate"}, + {104, &INx2Vi2Session::QueryDisplayHdrStatus, "QueryDisplayHdrStatus"}, + {105, &INx2Vi2Session::QueryDisplayHdrPaperWhite, "QueryDisplayHdrPaperWhite"}, + {212, &INx2Vi2Session::GetLayerBufferBits, "GetLayerBufferBits"}, {106, &INx2Vi2Session::GetBufferMaxResolution, "GetBufferMaxResolution"}, {228, &INx2Vi2Session::CancelClientBuffer, "CancelClientBuffer"}, {229, &INx2Vi2Session::PresentClientBuffer, "PresentClientBuffer"}, + // Firmware commands without a known meaning: zeroed replies. + {13, &INx2Vi2Session::StubReplyZeroed, "Unknown13"}, + {51, &INx2Vi2Session::StubReplyZeroed, "Unknown51"}, + {101, &INx2Vi2Session::StubReplyZeroed, "Unknown101"}, + {102, &INx2Vi2Session::StubReplyZeroed, "Unknown102"}, + {107, &INx2Vi2Session::StubReplyZeroed, "Unknown107"}, + {108, &INx2Vi2Session::StubReplyZeroed, "Unknown108"}, + {111, &INx2Vi2Session::StubReplyZeroed, "Unknown111"}, + {112, &INx2Vi2Session::StubReplyZeroed, "Unknown112"}, + {113, &INx2Vi2Session::StubReplyZeroed, "Unknown113"}, + {202, &INx2Vi2Session::StubReplyZeroed, "Unknown202"}, + {230, &INx2Vi2Session::StubReplyZeroed, "Unknown230"}, + {231, &INx2Vi2Session::StubReplyZeroed, "Unknown231"}, + {232, &INx2Vi2Session::StubReplyZeroed, "Unknown232"}, + {251, &INx2Vi2Session::StubReplyZeroed, "Unknown251"}, + {252, &INx2Vi2Session::StubReplyZeroed, "Unknown252"}, + {261, &INx2Vi2Session::StubReplyZeroed, "Unknown261"}, + {262, &INx2Vi2Session::StubReplyZeroed, "Unknown262"}, + {263, &INx2Vi2Session::StubReplyZeroed, "Unknown263"}, + {264, &INx2Vi2Session::StubReplyZeroed, "Unknown264"}, + {265, &INx2Vi2Session::StubReplyZeroed, "Unknown265"}, + {266, &INx2Vi2Session::StubReplyZeroed, "Unknown266"}, + {267, &INx2Vi2Session::StubReplyZeroed, "Unknown267"}, + {268, &INx2Vi2Session::StubReplyZeroed, "Unknown268"}, + {269, &INx2Vi2Session::StubReplyZeroed, "Unknown269"}, + {270, &INx2Vi2Session::StubReplyZeroed, "Unknown270"}, + {271, &INx2Vi2Session::StubReplyZeroed, "Unknown271"}, + {272, &INx2Vi2Session::StubReplyZeroed, "Unknown272"}, + {301, &INx2Vi2Session::StubReplyZeroed, "Unknown301"}, + {302, &INx2Vi2Session::StubReplyZeroed, "Unknown302"}, + {351, &INx2Vi2Session::StubReplyZeroed, "Unknown351"}, + {352, &INx2Vi2Session::StubReplyZeroed, "Unknown352"}, + {353, &INx2Vi2Session::StubReplyZeroed, "Unknown353"}, + {354, &INx2Vi2Session::StubReplyZeroed, "Unknown354"}, + {355, &INx2Vi2Session::StubReplyZeroed, "Unknown355"}, + {356, &INx2Vi2Session::StubReplyZeroed, "Unknown356"}, + {358, &INx2Vi2Session::StubReplyZeroed, "Unknown358"}, + {359, &INx2Vi2Session::StubReplyZeroed, "Unknown359"}, }); } @@ -1476,6 +1690,10 @@ public: } private: + void Handshake(HLERequestContext& ctx) { + TipcHandshake(ctx, "nx2:vi2"); + } + // 229(u64 client token, u32 slot, u32 image, ...fences): result, u64 mask of buffer slots the // client may acquire next, four zero words. Nothing is shown yet; every slot is handed back // at once (changes.patch's default mask is 0x7fff). @@ -1500,7 +1718,19 @@ private: LOG_DEBUG(Service, "nx2:vi2 present #{} fence ({}, {:#x}), waiting for ({}, {:#x}); now {:#x}", m_presents, ~w[22], w[24], m_previous_fence_id, m_previous_fence_value, ~w[22] < 192 ? system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(~w[22]) : 0U); + // Only run a frame ahead while the GPU keeps up. With the previous frame still on the + // GPU here, the GPU is the limit and running ahead gains nothing; and Splatoon Raiders, + // which reuses its command memory as if the GPU were at most a frame behind, rewrote + // command lists Eden had not read yet (misread commands, then crashes, with a slow + // GPU). Wait for this frame too then. + const bool gpu_behind = + m_previous_fence_id < 192 && + !system.Host1x().GetSyncpointManager().IsReadyGuest(m_previous_fence_id, + m_previous_fence_value); WaitForFrame(m_previous_fence_id, m_previous_fence_value); + if (gpu_behind) { + WaitForFrame(~w[22], w[24]); + } m_previous_fence_id = ~w[22]; m_previous_fence_value = w[24]; } else { @@ -1650,9 +1880,9 @@ private: // Shows a presented native image: word 5 is the image id, ~word 22 / word 24 the GPU fence // (syncpoint, value) to wait for (layout from changes.patch). The image is a block-linear // RGBA8 surface in the console's native GOB layout at GPU address == CPU address. - // present_size: word 7 of the present, (height - 1) << 16 | (width - 1) of the frame drawn - // into the image's top-left corner (Fire Emblem: Fortune's Weave: 1280x720 in 3840x2160). - void Composite(u32 image_id, u32 fence_id, u32 fence_value, u32 present_size = 0) { + // shown_size: the present's word 7, ((height - 1) << 16) | (width - 1) of the area the game drew + // into from the image's top-left corner (0 when it gives none). + void Composite(u32 image_id, u32 fence_id, u32 fence_value, u32 shown_size) { u64 address{}; Nx2ImageLayout layout{}; u32 image_format{}; @@ -1678,10 +1908,15 @@ private: } u32 crop_width = layout.width; u32 crop_height = layout.height; - // Present word 7 holds the size of the frame drawn into the image's top-left corner. - if (present_size != 0) { - crop_width = (std::min)(crop_width, (present_size & 0xffffU) + 1); - crop_height = (std::min)(crop_height, (present_size >> 16) + 1); + // The present says how much of the image the game drew: Kirby Air Riders draws 1920x1080 + // into 2560x1440 images, Mario Kart World 2560x1440 into 3840x2160 ones. Showing the whole + // image put the frame in the top-left corner. + const u32 shown_width = (shown_size & 0xFFFF) + 1, shown_height = (shown_size >> 16) + 1; + const bool has_shown_size = shown_size != 0 && shown_width <= layout.width && + shown_height <= layout.height; + if (has_shown_size) { + crop_width = shown_width; + crop_height = shown_height; } // Handheld titles keep their docked present images (up to 3840x2160) and draw the frame // into the top-left 1920x1080, the handheld panel's resolution; the console's display @@ -1690,7 +1925,8 @@ private: if (!Settings::IsDockedMode()) { crop_width = (std::min)(crop_width, 1920U); crop_height = (std::min)(crop_height, 1080U); - } else if (system.GetApplicationProcessProgramID() == 0x0400C3F00006E000ULL && + } else if (!has_shown_size && + system.GetApplicationProcessProgramID() == 0x0400C3F00006E000ULL && layout.width == 3840 && layout.height == 2160) { // Mario Kart World docked draws 2560x1440 into the top-left of its 4K images. crop_width = 2560; @@ -1712,8 +1948,9 @@ private: (u64{crop_width} << 16) | crop_height; if (shown != m_shown_layout) { m_shown_layout = shown; - LOG_INFO(Service, "nx2:vi2 composite image {} layout {}x{}, crop {}x{}", - image_id, layout.width, layout.height, crop_width, crop_height); + LOG_INFO(Service, + "nx2:vi2 composite image {} layout {}x{}, drawn area {:#x}, crop {}x{}", + image_id, layout.width, layout.height, shown_size, crop_width, crop_height); } std::vector<Tegra::FramebufferConfig> layers{Tegra::FramebufferConfig{ .address = *device, @@ -1822,32 +2059,45 @@ private: static_cast<u32>(AvailableSlots >> 32), 0, 0, 0, 0}); } + // nn::vi2::QueryDisplayRefreshRate(display name): u32 refresh rate in Hz, then a bool (a u64 + // in all). nn::vi::detail::DefaultDisplay::QueryRefreshRate takes only -1, 60 or 120 and fails + // on anything else; Minecraft polls it and never draws while it fails. A one-word reply left + // the bool to whatever the message buffer held. + void QueryDisplayRefreshRate(HLERequestContext& ctx) { + TipcReplyWords(ctx, {60, 0}); + } + + // nn::vi2::QueryDisplayHdrPaperWhite(display name): u16 paper white in nits. The empty reply + // left it to the message buffer's leftovers; answer the BT.2408 reference white. + void QueryDisplayHdrPaperWhite(HLERequestContext& ctx) { + TipcReplyWords(ctx, {203}); + } + + // nn::vi2::QueryDisplayHdrStatus(display name): u64 HdrOutputFormat, u16. The sdk's + // DefaultDisplay::QueryHdrStatus takes format 1 as HDR output and fails with 0x272 on + // anything above 1. With the empty auto stub reply it read the request's leftover words as + // the format; Kirby Air Riders re-queries whenever its HDR status event fires and rebuilt its + // window images every few frames, which showed black. Report SDR output. + // nn::vi2::QueryDisplayHdrStatus(display name): u64 output format, u16 peak luminance in + // nits. The sdk takes format 1 as SDR and 2 as HDR and fails the query on any other value. + // Report SDR output with a typical SDR panel's peak. + void QueryDisplayHdrStatus(HLERequestContext& ctx) { + TipcReplyWords(ctx, {1, 0, 400}); + } + + // 212(u64 layer): u64 bits of the layer's buffer slots that are ready. The client polls it once + // a second until some are set; it only got going because the auto stub's reply happened to + // hold non-zero leftovers. Report the same slots 228/229 hand back. + void GetLayerBufferBits(HLERequestContext& ctx) { + constexpr u64 AvailableSlots = 0x7fff; + TipcReplyWords(ctx, {static_cast<u32>(AvailableSlots), static_cast<u32>(AvailableSlots >> 32)}); + } + void ReturnOne(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC {}, returning 1", ctx.GetCommand()); TipcReplyWords(ctx, {1}); } - // 104(u64 display name): result, u64 output format, u32 0. Format 1 is SDR. A bare success - // left the format to stale reply words and a game could set up HDR output from them. - void GetHdrStatus(HLERequestContext& ctx) { - const auto* packet = ctx.CommandBuffer(); - const u64 name = u64{packet[2]} | (u64{packet[3]} << 32); - constexpr u64 DefaultDisplay = 0x00746c7561666564ULL; // "default" - Result result = Kernel::ResultInvalidCombination; - u64 format{}; - if (packet[0] == 0x78 && packet[1] == 2) { - result = Result{ErrorModule::VI, 7}; - if (name == DefaultDisplay) { - format = 1; - result = ResultSuccess; - } - } - IPC::ResponseBuilder rb{ctx, 5}; - rb.Push(result.raw); - rb.Push(format); - rb.Push<u32>(0); - } - // The highest buffer resolution mode of the "default" display. Fire Emblem: Fortune's Weave picks // its output from it: 1 -> 1280x720, 2 -> 1920x1080, 4 and 8 -> 2560x1440; 0 (the auto stub's // reply), 3 and 5 fall back to 720p. Docked reports 4, handheld 2 (the screen is 1080p); @@ -1871,12 +2121,6 @@ private: rb.Push(mode); } - // The same reply as the auto stub (success, no data), without its log lines. - void SucceedQuietly(HLERequestContext& ctx) { - IPC::ResponseBuilder rb{ctx, 2}; - rb.Push(ResultSuccess); - } - // Must be < 8; nn::vi::detail::LayerHistory creates 7 timestamp recorders per layer. void GetTimestampRecorderCount(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC 5, returning 7"); @@ -1893,17 +2137,105 @@ private: rb.PushCopyObjects(ctx, event->GetReadableEvent()); } - // 31(u32 kind) and 32(u32 display): events signalled on every vblank. Title 1's main thread - // waits on the 32 event until it has fired twice before it submits any GPU work, and hung - // there while it never fired. - void GetVsyncEvent(HLERequestContext& ctx) { - LOG_INFO(Service, "nx2:vi2 {} vsync event, [2]={:#x}", ctx.GetCommand(), ctx.CommandBuffer()[2]); - auto* event = m_context.CreateEvent(fmt::format("nx2:vi2:Vsync{}", m_events.size())); - m_events.push_back(event); - { - std::scoped_lock lk{m_vsync->mutex}; - m_vsync->events.push_back(event); + // 31(u32 index < 8): the session's event for that slot (the sdk picks a free index). The + // 261..272 commands bind a slot to a display event source and unbind it again. Until a slot + // is bound it fires on every vblank, as all of them used to. + void AcquireNativeEvent(HLERequestContext& ctx) { + const u32 index = ctx.CommandBuffer()[2]; + if (index >= m_native_events.size()) { + LOG_ERROR(Service, "nx2:vi2 31 index {} out of range", index); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultUnknown); + return; + } + LOG_INFO(Service, "nx2:vi2 31 native event {}", index); + ReplyNativeEvent(ctx, index); + } + + // 32(u32 index, copy handle): the client hands back its handle for an acquired slot and gets + // the slot's event again; the handle descriptor and handle come before the index. Games call + // it for slot 0 right after binding it to vsync. Title 1's main thread waits on this event + // until it has fired twice before it submits any GPU work, and hung there while it never + // fired. + void DuplicateNativeEvent(HLERequestContext& ctx) { + const u32* words = ctx.CommandBuffer(); + u32 data = 2; + if (words[1] >> 31) { + const u32 descriptor = words[2]; + data = 3 + ((descriptor >> 1) & 0xf) + ((descriptor >> 5) & 0xf) + (descriptor & 1 ? 2 : 0); + } + const u32 index = words[data]; + if (index >= m_native_events.size()) { + LOG_ERROR(Service, "nx2:vi2 32 index {} out of range", index); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultUnknown); + return; + } + LOG_INFO(Service, "nx2:vi2 32 native event {}", index); + ReplyNativeEvent(ctx, index); + } + + // 261(u64 display name, u32 index): fire the slot on every vblank. + void BindVsync(HLERequestContext& ctx) { + const u32 index = ctx.CommandBuffer()[4]; + LOG_INFO(Service, "nx2:vi2 261 native event {} -> vsync", index); + if (index < m_native_events.size() && m_native_events[index]) { + SetVsyncSlot(index, true); + } + TipcReplyWords(ctx, {}); + } + + // 263/265/267/269/271(u64 display name, u32 index): fire the slot when the display's refresh + // rate, HDR status, HDR paper white, buffer max resolution or mode capability changes, so the + // game re-queries it. None of them change here: fire once so the game reads the current + // state, then leave the slot quiet. Kirby Air Riders re-queried its HDR status on every + // present while this slot fired with vsync. + void BindChange(HLERequestContext& ctx) { + const u32 index = ctx.CommandBuffer()[4]; + LOG_INFO(Service, "nx2:vi2 {} native event {} -> change notification", ctx.GetCommand(), index); + if (index < m_native_events.size() && m_native_events[index]) { + SetVsyncSlot(index, false); + m_native_events[index]->Signal(system.Kernel()); } + TipcReplyWords(ctx, {}); + } + + // 262..272 even (u64 display name, u32 index): detach the slot from its source. + void UnbindNativeEvent(HLERequestContext& ctx) { + const u32 index = ctx.CommandBuffer()[4]; + LOG_INFO(Service, "nx2:vi2 {} native event {} unbound", ctx.GetCommand(), index); + if (index < m_native_events.size() && m_native_events[index]) { + SetVsyncSlot(index, false); + } + TipcReplyWords(ctx, {}); + } + + void ReplyNativeEvent(HLERequestContext& ctx, u32 index) { + auto*& event = m_native_events[index]; + if (!event) { + event = m_context.CreateEvent(fmt::format("nx2:vi2:Native{}", index)); + m_events.push_back(event); + SetVsyncSlot(index, true); + StartVsyncTimer(); + } + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, event->GetReadableEvent()); + } + + void SetVsyncSlot(u32 index, bool on) { + auto* event = m_native_events[index]; + std::scoped_lock lk{m_vsync->mutex}; + auto& list = m_vsync->events; + const auto it = std::find(list.begin(), list.end(), event); + if (on && it == list.end()) { + list.push_back(event); + } else if (!on && it != list.end()) { + list.erase(it); + } + } + + void StartVsyncTimer() { if (!m_vsync_timer) { m_vsync_timer = Core::Timing::CreateEvent( "nx2:vi2 vsync", @@ -1924,14 +2256,12 @@ private: }()}; system.CoreTiming().ScheduleLoopingEvent(VblankInterval, VblankInterval, m_vsync_timer); } - IPC::ResponseBuilder rb{ctx, 2, 1, 0}; - rb.Push(ResultSuccess); - rb.PushCopyObjects(ctx, event->GetReadableEvent()); } KernelHelpers::ServiceContext m_context; std::shared_ptr<Nx2NativeObjects> m_objects; std::vector<Kernel::KEvent*> m_events; + std::array<Kernel::KEvent*, 8> m_native_events{}; ///< 31's slots, also listed in m_events // Shared with the vsync timer callback, which can outlive the session by one call. struct VsyncState { std::mutex mutex; @@ -1956,7 +2286,9 @@ inline bool Nx2FirstVideoCommand(std::string_view service, u32 command) { // The Switch 2 video decoder (nn::codec::video::detail::ipc). gsv 902/906/1002 hand out a session // whose TIPC 1 (CreateVideoDecoderService: handle at word 3, result at word 4) creates a decoder // service, another session of this class. Layouts read from the SDK's nn::codec VP9 decoder: -// - 200(copy handle: work buffer transfer memory, u64 size, u32, u32, u32): initialize. +// - 200(copy handle: work buffer transfer memory, u64 size, u32, u32, u32): initialize. 201 adds a +// fourth u32 (a flag that 200 leaves at 1) and 202 that and a u64: the same initialization. +// 210(copy handle, u32 x4) replies only a result. // - 101(A: codec name, "video/vp9\0"; u32 width, u32 height, u32 rate, u32): configure. // - 3(): the submission-ready event (copy handle at word 3, result at word 4). // - 6 / 207 / 8: start / stop / flush; 115: finalize. @@ -1967,6 +2299,8 @@ inline bool Nx2FirstVideoCommand(std::string_view service, u32 command) { // rewritten once that handle (the client's reads) has passed. Reply the image's next handle. // - 212(u32 image id): unregister it. // - 109(A: one compressed frame, u64 pts in us, u8 end of stream): submit input; reply a u64. +// 400 is the same with a second A buffer after the frame, a 0x838-byte block of frame +// information; the decoder here only needs the frame. // - 213(): the next finished frame (16 words): u32 image id, u32 width, u32 height, u32 width, // u32 height, u32 colour space, TimeSpan pts (ns), u8 end of stream. // A NativeSyncHandle is {u32 SyncCounterId (~syncpoint), u32 mode, u64 value}; clients wait on it @@ -1988,11 +2322,15 @@ public: {109, &INx2VideoSession::SubmitInput, "SubmitInputBuffer"}, {115, &INx2VideoSession::Succeed, "Finalize"}, {200, &INx2VideoSession::Succeed, "Initialize"}, + {201, &INx2VideoSession::Succeed, "InitializeWithFlag"}, + {202, &INx2VideoSession::Succeed, "InitializeWithFlagAndValue"}, {207, &INx2VideoSession::Flush, "Stop"}, + {210, &INx2VideoSession::Succeed, "Unknown210"}, {212, &INx2VideoSession::UnregisterBuffer, "UnregisterBuffer"}, {213, &INx2VideoSession::GetOutputInfo, "GetOutputInfo"}, {311, &INx2VideoSession::RegisterBuffer, "RegisterBuffer"}, {314, &INx2VideoSession::ReleaseBuffer, "ReleaseBuffer"}, + {400, &INx2VideoSession::SubmitInput, "SubmitInputBufferWithInfo"}, }); } @@ -2180,9 +2518,11 @@ private: void SubmitInput(HLERequestContext& ctx) { const auto* w = ctx.CommandBuffer(); - // Words 2-4 are the A descriptor of the compressed frame. - const u64 pts = u64{w[5]} | (u64{w[6]} << 32); - const bool end = (w[7] & 0xFF) != 0; + // Words 2-4 are the A descriptor of the compressed frame (400: 5-7 that of its information + // block), the data words follow. + const size_t words = 2 + 3 * ctx.BufferDescriptorA().size(); + const u64 pts = u64{w[words]} | (u64{w[words + 1]} << 32); + const bool end = (w[words + 2] & 0xFF) != 0; Log(ctx, "submit"); Input input{{}, pts, end}; if (!ctx.BufferDescriptorA().empty()) { @@ -2480,6 +2820,7 @@ public: "CalculateDecoderImagePoolBuffer"}, {901, &INx2GraphicsSystemSession::Stub, "VideoInitialize (guessed)"}, {902, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSession (guessed)"}, + {905, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSessionWithFlag"}, {906, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSessionWithVersion"}, {1001, &INx2GraphicsSystemSession::Stub, "VideoInitialize2 (guessed)"}, {1002, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSession2 (guessed)"}, @@ -2492,7 +2833,92 @@ public: {1291, &INx2GraphicsSystemSession::CalculateChatImagePoolSize, "CalculateChatImagePoolSize (guessed)"}, {911, &INx2GraphicsSystemSession::CreateDecoderImage, "CreateDecoderImage"}, + // 912(u32 image id): the video group's x12, freeing one decoder output image (Switch + // Sports frees its four when its movie ends). Unlisting it stops the decode thread + // writing frames into memory the game reuses. + {912, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyDecoderImage"}, {961, &INx2GraphicsSystemSession::GetCpuImageDescriptor, "GetCpuUserImageDescriptor"}, + // Firmware commands without a known meaning: zeroed replies. + {11, &INx2GraphicsSystemSession::RegisterClientHandle, "RegisterClientHandle"}, + {113, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown113"}, + {115, &INx2GraphicsSystemSession::QueryGrantedFeatures, "QueryGrantedFeatures"}, + {120, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown120"}, + {121, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown121"}, + {122, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown122"}, + {123, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown123"}, + {124, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown124"}, + {125, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown125"}, + {126, &INx2GraphicsSystemSession::GetSessionFlag, "GetSessionFlag"}, + {131, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown131"}, + {133, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown133"}, + {187, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown187"}, + {201, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission201"}, + {211, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown211"}, + {212, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown212"}, + {213, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown213"}, + {214, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown214"}, + {215, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown215"}, + {301, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission301"}, + {303, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown303"}, + {311, &INx2GraphicsSystemSession::CreateNativeImage, "CreateFeatureImagePool311"}, + {312, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyFeatureImagePool312"}, + {321, &INx2GraphicsSystemSession::CreateNativeImage, "CreateFeatureImagePool321"}, + {361, &INx2GraphicsSystemSession::GetCpuImageDescriptor, "GetFeatureCpuImageDescriptor361"}, + {391, &INx2GraphicsSystemSession::CalculateChatImagePoolSize, "CalculateFeatureImagePoolSize391"}, + {401, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission401"}, + {402, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission402"}, + {403, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission403"}, + {424, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown424"}, + {425, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown425"}, + {427, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown427"}, + {434, &INx2GraphicsSystemSession::AttachGpuAddressSpace, "AttachGpuAddressSpace"}, + {436, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown436"}, + {461, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown461"}, + {462, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown462"}, + {463, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown463"}, + {464, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown464"}, + {465, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown465"}, + {466, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown466"}, + {467, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown467"}, + {468, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown468"}, + {493, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown493"}, + {494, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission494"}, + {603, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown603"}, + {612, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyFeatureImagePool612"}, + {613, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown613"}, + {614, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown614"}, + {615, &INx2GraphicsSystemSession::CreateNativeImage, "CreateFeatureImagePool615"}, + {691, &INx2GraphicsSystemSession::CalculateChatImagePoolSize, "CalculateFeatureImagePoolSize691"}, + {801, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission801"}, + {802, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown802"}, + {803, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown803"}, + {811, &INx2GraphicsSystemSession::CreateNativeImage, "CreateFeatureImagePool811"}, + {812, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyFeatureImagePool812"}, + {813, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown813"}, + {814, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown814"}, + {861, &INx2GraphicsSystemSession::GetCpuImageDescriptor, "GetFeatureCpuImageDescriptor861"}, + {891, &INx2GraphicsSystemSession::CalculateChatImagePoolSize, "CalculateFeatureImagePoolSize891"}, + {1003, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1003"}, + {1011, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1011"}, + {1012, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1012"}, + {1091, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1091"}, + {1101, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1101"}, + {1102, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1102"}, + {1111, &INx2GraphicsSystemSession::SetFeatureVersion, "SetFeatureVersion"}, + {1112, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1112"}, + {1113, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1113"}, + {1121, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1121"}, + {1203, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1203"}, + {1212, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyFeatureImagePool1212"}, + {1501, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1501"}, + {1502, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1502"}, + {1503, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1503"}, + {1601, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1601"}, + {1602, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1602"}, + {1603, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1603"}, + {1611, &INx2GraphicsSystemSession::CreateNativeImage, "CreateFeatureImagePool1611"}, + {1612, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyFeatureImagePool1612"}, + {1691, &INx2GraphicsSystemSession::CalculateChatImagePoolSize, "CalculateFeatureImagePoolSize1691"}, }); } @@ -2508,6 +2934,56 @@ public: } private: + // Feature permission checks (201, 301, 401-403, 1101, 1102, 1501-1503): each tests one bit of + // the permissions the session was opened with and fails when it is clear. Games only ask for + // features they are granted. + void CheckPermission(HLERequestContext& ctx) { + LOG_DEBUG(Service, "nx2:gsv TIPC {} permission check", ctx.GetCommand()); + TipcReplyWords(ctx, {}); + } + + // 115(u64 requested feature flags): the subset that is granted, which is all of them here. + // A zeroed reply would deny every feature. + void QueryGrantedFeatures(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + LOG_INFO(Service, "nx2:gsv TIPC 115 features {:#x}", u64{w[2]} | (u64{w[3]} << 32)); + TipcReplyWords(ctx, {w[2], w[3]}); + } + + // 126: one bit of the session's state (meaning unknown); clear. + void GetSessionFlag(HLERequestContext& ctx) { + TipcReplyWords(ctx, {0}); + } + + // 434(u32 id from nx2:gpu 33): the session takes a reference on that GPU address space. + void AttachGpuAddressSpace(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + LOG_INFO(Service, "nx2:gsv TIPC 434 address space {:#x}", id); + m_address_spaces.insert(id); + TipcReplyWords(ctx, {}); + } + + // 11(copy handle): handed over once per session and kept; a second call fails. + void RegisterClientHandle(HLERequestContext& ctx) { + if (m_client_handle_registered) { + LOG_WARNING(Service, "nx2:gsv TIPC 11 called twice"); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultUnknown); + return; + } + m_client_handle_registered = true; + TipcReplyWords(ctx, {}); + } + + // 1111(u32): a version or mode number the server checks and records. + void SetFeatureVersion(HLERequestContext& ctx) { + LOG_INFO(Service, "nx2:gsv TIPC 1111 value {:#x}", ctx.CommandBuffer()[2]); + TipcReplyWords(ctx, {}); + } + + std::set<u32> m_address_spaces; ///< Ids attached with 434 + bool m_client_handle_registered = false; ///< Set by 11 + // Index of the first raw data word of a TIPC request: header (2 words), then an optional // special header with pid and copy/move handle words. static size_t TipcDataOffset(const u32* cmd_buf) { @@ -2531,7 +3007,9 @@ private: // and InitializeVp9Decoder failed with 2001-0114 (title 1 aborts when a movie starts). // Newer SDKs (nn::codec::video::detail::InitializeServerContext) send 906(u8, u64 decoder // library version) instead of 902; title 4 sends version 0x3'00030000 and aborted with - // InvalidHandle at its first movie while 906 was auto-stubbed. + // InvalidHandle at its first movie while 906 was auto-stubbed. The SDK of later titles + // (ipc::Initialize(bool)) sends 905(u8) instead of 902 when its flag is set; the reply is the + // same. 1001/1002 open the encoder's creator session (CreateVideoEncoderService uses it). void OpenVideoSession(HLERequestContext& ctx) { const auto* w = ctx.CommandBuffer(); const bool second = ctx.GetCommand() == 1002; @@ -2544,13 +3022,11 @@ private: // nn::camera::sdk::CreateCameraContext sends 601() then 602(u64 1), which returns the camera // context session (handle at word 3, result at word 4). Auto-stubbed, 602 had no handle, the - // SDK sent 603 (destroy) and title 1 (1.8.0) aborted with 2034-0000. Its commands fall - // through to the nx2 auto stub. + // SDK sent 603 (destroy) and title 1 (1.8.0) aborted with 2034-0000. Its commands are + // answered by INx2CameraSession. void OpenCameraSession(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 602, returning a nx2:camera session"); - ReturnRawSession(system, ctx, - std::make_shared<INx2StubInterface>( - system, INx2StubInterface::InternName("nx2:camera"))); + ReturnRawSession(system, ctx, std::make_shared<INx2CameraSession>(system)); } // nn::chat::gsv::Initialize sends 1201 then 1202, the same pair as 901/902 and 1001/1002. @@ -2798,6 +3274,8 @@ private: TipcReplyWords(ctx, {id}); } + // Pool size of a feature's images (1291 here; 391, 691, 891 and 1691 for the other feature + // groups, which share the server template). // nn::chat::gsv::CalculateRequiredBufferMemory: 1291(preference[16], u32 count), the request // of 1510, replies result, u64 size, u64 alignment. Auto-stubbed, the size became 0x1000 and // the camera pool (1211) got memory for none of its images. diff --git a/src/core/hle/service/ro/ro.cpp b/src/core/hle/service/ro/ro.cpp index 335efa3ad3..2af4f1cf58 100644 --- a/src/core/hle/service/ro/ro.cpp +++ b/src/core/hle/service/ro/ro.cpp @@ -356,6 +356,22 @@ public: // Read NRR. NrrHeader header{}; process->GetMemory().ReadBlock(nrr_address, std::addressof(header), sizeof(header)); + size_t hashes_offset = header.GetHashesOffset(); + u32 num_hashes = header.GetNumHashes(); + if (process->IsSwitch2()) { + // Switch 2 NRRs carry 3072-bit keys and signatures: the fields after them (program + // id, size, kind, hash list offset and count) sit 0x180 bytes later, the hash list + // offset and count at 0x4c0. + std::array<u32, 2> fields{}; + process->GetMemory().ReadBlock(nrr_address + 0x4c0, fields.data(), sizeof(fields)); + hashes_offset = fields[0]; + num_hashes = fields[1]; + } + // The hash list lies inside the NRR (read as a Switch 1 header, a Switch 2 one asked for + // 51 GB of hashes). + R_UNLESS(hashes_offset <= nrr_size && + num_hashes <= (nrr_size - hashes_offset) / sizeof(Sha256Hash), + RO::ResultInvalidNrr); // Set NRR info. context->SetNrrInfoInUse(nrr_info, true); @@ -363,10 +379,9 @@ public: nrr_info->nrr_heap_size = nrr_size; // Read NRR hash list. - nrr_info->hashes.resize(header.GetNumHashes()); - process->GetMemory().ReadBlock(nrr_address + header.GetHashesOffset(), - nrr_info->hashes.data(), - sizeof(Sha256Hash) * header.GetNumHashes()); + nrr_info->hashes.resize(num_hashes); + process->GetMemory().ReadBlock(nrr_address + hashes_offset, nrr_info->hashes.data(), + sizeof(Sha256Hash) * num_hashes); R_SUCCEED(); } diff --git a/src/core/hle/service/service.cpp b/src/core/hle/service/service.cpp index 1490af0328..b47d67468b 100644 --- a/src/core/hle/service/service.cpp +++ b/src/core/hle/service/service.cpp @@ -9,6 +9,10 @@ #include <tuple> #include <mutex> #include <map> +#include <memory> +#include <set> +#include <string> +#include <string_view> #include <chrono> #include <algorithm> #include <cstdlib> @@ -18,8 +22,10 @@ #include "common/settings.h" #include "core/core.h" #include "core/hle/ipc.h" +#include "core/hle/kernel/k_event.h" #include "core/hle/kernel/kernel.h" #include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/kernel_helpers.h" #include "core/hle/service/service.h" #include "core/hle/service/nx2_stub_interface.h" #include "core/hle/service/sm/sm.h" @@ -49,6 +55,75 @@ ServiceFrameworkBase::ServiceFrameworkBase(Core::System& system_, const char* se ServiceFrameworkBase::~ServiceFrameworkBase() { // Wait for other threads to release access before destroying const auto guard = ServiceFrameworkBase::LockService(); + if (stub_event) { + stub_context->CloseEvent(stub_event); + } +} + +namespace { + +// Callers read the reply words they expect without checking how many were written, so an empty +// reply hands them the request's leftover words as data (an HDR output format, a refresh rate, a +// device id). Stub replies carry this many zeroed words. +constexpr u32 StubZeroedWords = 32; + +void PushStubZeroes(IPC::ResponseBuilder& rb) { + for (u32 i = 0; i < StubZeroedWords; ++i) { + rb.Push<u32>(0); + } +} + +} // namespace + +bool ServiceFrameworkBase::StubAllowed(HLERequestContext& ctx) { + static const bool env_auto_stub = std::getenv("EDEN_NX2_AUTO_STUB") != nullptr; + const auto* const caller = ctx.GetThread().GetOwnerProcess(); + if (Settings::values.use_auto_stub || env_auto_stub || + std::string_view{service_name}.starts_with(Nx2StubPrefix) || + (caller != nullptr && caller->IsSwitch2())) { + // Logged once per command: these are still unimplemented. + static std::mutex mutex; + static std::set<std::pair<std::string_view, u32>> logged; + std::scoped_lock lk{mutex}; + if (logged.emplace(service_name, ctx.GetCommand()).second) { + LOG_WARNING(Service, "(STUBBED) {}::{} (placeholder reply)", service_name, ctx.GetCommand()); + } + return true; + } + const auto& table = ctx.IsTipc() ? handlers_tipc : handlers; + const auto it = table.find(ctx.GetCommand()); + ReportUnimplementedFunction(ctx, it == table.end() ? nullptr : &it->second); + return false; +} + +void ServiceFrameworkBase::StubReplyZeroed(HLERequestContext& ctx) { + if (!StubAllowed(ctx)) { + return; + } + IPC::ResponseBuilder rb{ctx, 2 + StubZeroedWords}; + rb.Push(ResultSuccess); + PushStubZeroes(rb); +} + +void ServiceFrameworkBase::StubReplyEvent(HLERequestContext& ctx) { + if (!StubAllowed(ctx)) { + return; + } + if (!stub_event) { + stub_context = std::make_unique<KernelHelpers::ServiceContext>(system, service_name); + stub_event = stub_context->CreateEvent(fmt::format("{}:StubEvent", service_name)); + } + IPC::ResponseBuilder rb{ctx, 2 + StubZeroedWords, 1, 0}; + rb.Push(ResultSuccess); + PushStubZeroes(rb); + rb.PushCopyObjects(ctx, stub_event->GetReadableEvent()); +} + +void ServiceFrameworkBase::StubReplyInterface(HLERequestContext& ctx) { + if (!StubAllowed(ctx)) { + return; + } + ReturnNx2StubInterface(system, ctx, service_name); } void ServiceFrameworkBase::RegisterHandlersBase(const FunctionInfoBase* functions, std::size_t n) { @@ -67,6 +142,27 @@ void ServiceFrameworkBase::RegisterHandlersBaseTipc(const FunctionInfoBase* func void ServiceFrameworkBase::ReportUnimplementedFunction(HLERequestContext& ctx, const FunctionInfoBase* info) { + // EDEN_NX2_AUTO_STUB=1 auto-stubs every unknown command without touching the config. Switch 2 + // programs are always auto-stubbed: they still call many commands nobody has implemented. + static const bool env_auto_stub = std::getenv("EDEN_NX2_AUTO_STUB") != nullptr; + const auto* const caller = ctx.GetThread().GetOwnerProcess(); + const bool auto_stub = Settings::values.use_auto_stub || env_auto_stub || + std::string_view{service_name}.starts_with(Nx2StubPrefix) || + (caller != nullptr && caller->IsSwitch2()); + // Auto-stubbed commands are reported once each: title 1 polls nx2:camera 29 every frame + // (~47 calls a second), and every report wrote a Critical line, which flushes the log file. + if (auto_stub) { + static std::mutex reported_mutex; + static std::set<std::pair<std::string, u32>> reported; + std::scoped_lock lk{reported_mutex}; + if (!reported.emplace(service_name, ctx.GetCommand()).second) { + IPC::ResponseBuilder rb{ctx, 2 + StubZeroedWords}; + rb.Push(ResultSuccess); + PushStubZeroes(rb); + return; + } + } + auto cmd_buf = ctx.CommandBuffer(); std::string function_name = info == nullptr ? "<unknown>" : info->name; @@ -80,16 +176,11 @@ void ServiceFrameworkBase::ReportUnimplementedFunction(HLERequestContext& ctx, system.GetReporter().SaveUnimplementedFunctionReport(ctx, ctx.GetCommand(), function_name, service_name); UNIMPLEMENTED_MSG("Unknown / unimplemented {}", fmt::to_string(buf)); - // EDEN_NX2_AUTO_STUB=1 auto-stubs every unknown command without touching the config. Switch 2 - // programs are always auto-stubbed: they still call many commands nobody has implemented. - static const bool env_auto_stub = std::getenv("EDEN_NX2_AUTO_STUB") != nullptr; - const auto* const caller = ctx.GetThread().GetOwnerProcess(); - if (Settings::values.use_auto_stub || env_auto_stub || - std::string_view{service_name}.starts_with(Nx2StubPrefix) || - (caller != nullptr && caller->IsSwitch2())) { - LOG_WARNING(Service, "Using auto stub fallback!"); - IPC::ResponseBuilder rb{ctx, 2}; + if (auto_stub) { + LOG_WARNING(Service, "Using auto stub fallback (repeats of this command are not logged)"); + IPC::ResponseBuilder rb{ctx, 2 + StubZeroedWords}; rb.Push(ResultSuccess); + PushStubZeroes(rb); } } diff --git a/src/core/hle/service/service.h b/src/core/hle/service/service.h index 68b0d636d8..d1be62c8c7 100644 --- a/src/core/hle/service/service.h +++ b/src/core/hle/service/service.h @@ -7,6 +7,7 @@ #pragma once #include <cstddef> +#include <memory> #include <mutex> #include "common/container/unordered_map.h" #include "common/common_types.h" @@ -20,12 +21,17 @@ class System; } namespace Kernel { +class KEvent; class KServerSession; class ServiceThread; } // namespace Kernel namespace Service { +namespace KernelHelpers { +class ServiceContext; +} + namespace FileSystem { class FileSystemController; } @@ -77,6 +83,24 @@ protected: [[nodiscard]] virtual std::unique_lock<std::mutex> LockService() noexcept { return std::unique_lock{lock_service}; } + + /// Placeholder handlers for commands whose reply shape is known but whose meaning is not, for + /// Switch 2 callers (others get the unimplemented-function path, as with nullptr): + /// StubReplyZeroed replies with zeroed data words, StubReplyEvent adds a copy handle to an + /// event that never fires, and StubReplyInterface returns an nx2: stub interface whose + /// commands are auto-stubbed. + void StubReplyZeroed(HLERequestContext& ctx); + void StubReplyEvent(HLERequestContext& ctx); + void StubReplyInterface(HLERequestContext& ctx); + +private: + /// False (after reporting the command as unimplemented) for callers the stubs don't serve. + bool StubAllowed(HLERequestContext& ctx); + + /// Event handed out by StubReplyEvent, created on first use. + Kernel::KEvent* stub_event{}; + std::unique_ptr<KernelHelpers::ServiceContext> stub_context; + private: template <typename T> friend class ServiceFramework; diff --git a/src/core/hle/service/set/settings_server.cpp b/src/core/hle/service/set/settings_server.cpp index 942dddc9bc..78847ba082 100644 --- a/src/core/hle/service/set/settings_server.cpp +++ b/src/core/hle/service/set/settings_server.cpp @@ -102,9 +102,14 @@ ISettingsServer::ISettingsServer(Core::System& system_) : ServiceFramework{syste {7, C<&ISettingsServer::GetKeyCodeMap>, "GetKeyCodeMap"}, {8, C<&ISettingsServer::GetQuestFlag>, "GetQuestFlag"}, {9, C<&ISettingsServer::GetKeyCodeMap2>, "GetKeyCodeMap2"}, - {10, nullptr, "GetFirmwareVersionForDebug"}, + {10, &ISettingsServer::StubReplyZeroed, "GetFirmwareVersionForDebug"}, {11, C<&ISettingsServer::GetDeviceNickName>, "GetDeviceNickName"}, {12, C<&ISettingsServer::GetKeyCodeMapByPort>, "GetKeyCodeMapByPort"}, + {13, &ISettingsServer::StubReplyZeroed, "Unknown13"}, + {14, &ISettingsServer::StubReplyZeroed, "Unknown14"}, + {15, &ISettingsServer::StubReplyZeroed, "Unknown15"}, + {16, &ISettingsServer::StubReplyZeroed, "Unknown16"}, + {17, &ISettingsServer::StubReplyZeroed, "Unknown17"}, }; // clang-format on diff --git a/src/core/hle/service/sockets/bsd.cpp b/src/core/hle/service/sockets/bsd.cpp index 0137a3be66..02c6bb7762 100644 --- a/src/core/hle/service/sockets/bsd.cpp +++ b/src/core/hle/service/sockets/bsd.cpp @@ -1089,10 +1089,10 @@ BSD_USA::BSD_USA(Core::System& system_, const char* name, bool is_user_) {1, &BSD_USA::StartMonitoring, "StartMonitoring"}, {2, &BSD_USA::Socket, "Socket"}, {3, &BSD_USA::SocketExempt, "SocketExempt"}, - {4, nullptr, "Open"}, + {4, &BSD_USA::StubReplyZeroed, "Open"}, {5, &BSD_USA::Select, "Select"}, {6, &BSD_USA::Poll, "Poll"}, - {7, nullptr, "Sysctl"}, + {7, &BSD_USA::StubReplyZeroed, "Sysctl"}, {8, &BSD_USA::Recv, "Recv"}, {9, &BSD_USA::RecvFrom, "RecvFrom"}, {10, &BSD_USA::Send, "Send"}, @@ -1104,33 +1104,37 @@ BSD_USA::BSD_USA(Core::System& system_, const char* name, bool is_user_) {16, &BSD_USA::GetSockName, "GetSockName"}, {17, &BSD_USA::GetSockOpt, "GetSockOpt"}, {18, &BSD_USA::Listen, "Listen"}, - {19, nullptr, "Ioctl"}, + {19, &BSD_USA::StubReplyZeroed, "Ioctl"}, {20, &BSD_USA::Fcntl, "Fcntl"}, {21, &BSD_USA::SetSockOpt, "SetSockOpt"}, {22, &BSD_USA::Shutdown, "Shutdown"}, - {23, nullptr, "ShutdownAllSockets"}, + {23, &BSD_USA::StubReplyZeroed, "ShutdownAllSockets"}, {24, &BSD_USA::Write, "Write"}, {25, &BSD_USA::Read, "Read"}, {26, &BSD_USA::Close, "Close"}, {27, &BSD_USA::DuplicateSocket, "DuplicateSocket"}, - {28, nullptr, "GetResourceStatistics"}, - {29, nullptr, "RecvMMsg"}, //3.0.0+ - {30, nullptr, "SendMMsg"}, //3.0.0+ + {28, &BSD_USA::StubReplyZeroed, "GetResourceStatistics"}, + {29, &BSD_USA::StubReplyZeroed, "RecvMMsg"}, //3.0.0+ + {30, &BSD_USA::StubReplyZeroed, "SendMMsg"}, //3.0.0+ {31, &BSD_USA::EventFd, "EventFd"}, //7.0.0+ - {32, nullptr, "RegisterResourceStatisticsName"}, //7.0.0+ - {33, nullptr, "RegisterClientShared"}, //10.0.0+ - {34, nullptr, "GetSocketStatistics"}, //15.0.0+ - {35, nullptr, "NifIoctl"}, //17.0.0+ - {36, nullptr, "Unknown36"}, //18.0.0+ - {37, nullptr, "Unknown37"}, //18.0.0+ - {38, nullptr, "Unknown38"}, //18.0.0+ - {39, nullptr, "Unknown39"}, //20.0.0+ - {40, nullptr, "Unknown40"}, //20.0.0+ - {41, nullptr, "Unknown41"}, //21.0.0+ - {42, nullptr, "Unknown42"}, //21.0.0+ - {43, nullptr, "Unknown43"}, //21.0.0+ - {200, nullptr, "SetThreadCoreMask"}, //15.0.0+ - {201, nullptr, "GetThreadCoreMask"}, //15.0.0+ + {32, &BSD_USA::StubReplyZeroed, "RegisterResourceStatisticsName"}, //7.0.0+ + {33, &BSD_USA::StubReplyZeroed, "RegisterClientShared"}, //10.0.0+ + {34, &BSD_USA::StubReplyZeroed, "GetSocketStatistics"}, //15.0.0+ + {35, &BSD_USA::StubReplyZeroed, "NifIoctl"}, //17.0.0+ + // Switch 2 titles register with 36 (16 bytes and the pid; a bsd errno back) instead of + // RegisterClient. The auto stub left the errno to the message buffer's leftovers, and + // Minecraft's socket layer failed later in EventFd. + {36, &BSD_USA::RegisterClient, "RegisterClientSwitch2"}, //18.0.0+ + {37, &BSD_USA::StubReplyZeroed, "Unknown37"}, //18.0.0+ + {38, &BSD_USA::StubReplyZeroed, "Unknown38"}, //18.0.0+ + {39, &BSD_USA::StubReplyZeroed, "Unknown39"}, //20.0.0+ + {40, &BSD_USA::StubReplyZeroed, "Unknown40"}, //20.0.0+ + {41, &BSD_USA::StubReplyEvent, "Unknown41"}, //21.0.0+ + {42, &BSD_USA::StubReplyZeroed, "Unknown42"}, //21.0.0+ + {43, &BSD_USA::StubReplyZeroed, "Unknown43"}, //21.0.0+ + {200, &BSD_USA::StubReplyZeroed, "SetThreadCoreMask"}, //15.0.0+ + {201, &BSD_USA::StubReplyZeroed, "GetThreadCoreMask"}, //15.0.0+ + {44, &BSD_USA::StubReplyZeroed, "Unknown44"}, }; // clang-format on @@ -1158,22 +1162,28 @@ BSDCFG::BSDCFG(Core::System& system_, const char *name) : ServiceFramework{system_, name} { // clang-format off static const FunctionInfo functions[] = { - {0, nullptr, "SetIfUp"}, - {1, nullptr, "SetIfUpWithEvent"}, - {2, nullptr, "CancelIf"}, - {3, nullptr, "SetIfDown"}, - {4, nullptr, "GetIfState"}, - {5, nullptr, "DhcpRenew"}, - {6, nullptr, "AddStaticArpEntry"}, - {7, nullptr, "RemoveArpEntry"}, - {8, nullptr, "LookupArpEntry"}, - {9, nullptr, "LookupArpEntry2"}, - {10, nullptr, "ClearArpEntries"}, - {11, nullptr, "ClearArpEntries2"}, - {12, nullptr, "PrintArpEntries"}, - {13, nullptr, "Unknown13"}, - {14, nullptr, "Unknown14"}, - {15, nullptr, "Unknown15"}, + {0, &BSDCFG::StubReplyZeroed, "SetIfUp"}, + {1, &BSDCFG::StubReplyZeroed, "SetIfUpWithEvent"}, + {2, &BSDCFG::StubReplyZeroed, "CancelIf"}, + {3, &BSDCFG::StubReplyZeroed, "SetIfDown"}, + {4, &BSDCFG::StubReplyZeroed, "GetIfState"}, + {5, &BSDCFG::StubReplyZeroed, "DhcpRenew"}, + {6, &BSDCFG::StubReplyZeroed, "AddStaticArpEntry"}, + {7, &BSDCFG::StubReplyZeroed, "RemoveArpEntry"}, + {8, &BSDCFG::StubReplyZeroed, "LookupArpEntry"}, + {9, &BSDCFG::StubReplyZeroed, "LookupArpEntry2"}, + {10, &BSDCFG::StubReplyZeroed, "ClearArpEntries"}, + {11, &BSDCFG::StubReplyZeroed, "ClearArpEntries2"}, + {12, &BSDCFG::StubReplyZeroed, "PrintArpEntries"}, + {13, &BSDCFG::StubReplyZeroed, "Unknown13"}, + {14, &BSDCFG::StubReplyZeroed, "Unknown14"}, + {15, &BSDCFG::StubReplyZeroed, "Unknown15"}, + {16, &BSDCFG::StubReplyZeroed, "Unknown16"}, + {100, &BSDCFG::StubReplyZeroed, "Unknown100"}, + {101, &BSDCFG::StubReplyZeroed, "Unknown101"}, + {102, &BSDCFG::StubReplyZeroed, "Unknown102"}, + {103, &BSDCFG::StubReplyZeroed, "Unknown103"}, + {104, &BSDCFG::StubReplyZeroed, "Unknown104"}, }; // clang-format on diff --git a/src/core/hle/service/sockets/nsd.cpp b/src/core/hle/service/sockets/nsd.cpp index 241f2971ca..eb867cf08a 100644 --- a/src/core/hle/service/sockets/nsd.cpp +++ b/src/core/hle/service/sockets/nsd.cpp @@ -86,7 +86,7 @@ void NSD::SetChangeEnvironmentIdentifierDisabled(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) called, disabled={}", disabled); - IPC::ResponseBuilder rb{ctx, 1}; + IPC::ResponseBuilder rb{ctx, 2}; rb.Push(ResultSuccess); } @@ -114,7 +114,7 @@ void NSD::ResolveEx(HLERequestContext& ctx) { } ctx.WriteBuffer(fqdn_out); - IPC::ResponseBuilder rb{ctx, 4}; + IPC::ResponseBuilder rb{ctx, 3}; rb.Push(ResultSuccess); rb.Push(ResultSuccess); } diff --git a/src/core/hle/service/ssl/ssl.cpp b/src/core/hle/service/ssl/ssl.cpp index 999efd4295..15aa478b75 100644 --- a/src/core/hle/service/ssl/ssl.cpp +++ b/src/core/hle/service/ssl/ssl.cpp @@ -87,7 +87,7 @@ public: {3, D<&ISslConnection::SetIoMode>, "SetIoMode"}, {4, D<&ISslConnection::GetSocketDescriptor>, "GetSocketDescriptor"}, {5, D<&ISslConnection::GetHostName>, "GetHostName"}, - {6, nullptr, "GetVerifyOption"}, + {6, &ISslConnection::StubReplyZeroed, "GetVerifyOption"}, {7, D<&ISslConnection::GetIoMode>, "GetIoMode"}, {8, D<&ISslConnection::DoHandshake>, "DoHandshake"}, {9, &ISslConnection::DoHandshakeGetServerCert, "DoHandshakeGetServerCert"}, @@ -105,18 +105,20 @@ public: {21, D<&ISslConnection::GetRenegotiationMode>, "GetRenegotiationMode"}, {22, D<&ISslConnection::SetOption>, "SetOption"}, {23, D<&ISslConnection::GetOption>, "GetOption"}, - {24, nullptr, "GetVerifyCertErrors"}, - {25, nullptr, "GetCipherInfo"}, + {24, &ISslConnection::StubReplyZeroed, "GetVerifyCertErrors"}, + {25, &ISslConnection::StubReplyZeroed, "GetCipherInfo"}, {26, D<&ISslConnection::SetNextAlpnProto>, "SetNextAlpnProto"}, {27, D<&ISslConnection::GetNextAlpnProto>, "GetNextAlpnProto"}, - {28, nullptr, "SetDtlsSocketDescriptor"}, - {29, nullptr, "GetDtlsHandshakeTimeout"}, - {30, nullptr, "SetPrivateOption"}, - {31, nullptr, "SetSrtpCiphers"}, - {32, nullptr, "GetSrtpCipher"}, - {33, nullptr, "ExportKeyingMaterial"}, - {34, nullptr, "SetIoTimeout"}, - {35, nullptr, "GetIoTimeout"}, + {28, &ISslConnection::StubReplyZeroed, "SetDtlsSocketDescriptor"}, + {29, &ISslConnection::StubReplyZeroed, "GetDtlsHandshakeTimeout"}, + {30, &ISslConnection::StubReplyZeroed, "SetPrivateOption"}, + {31, &ISslConnection::StubReplyZeroed, "SetSrtpCiphers"}, + {32, &ISslConnection::StubReplyZeroed, "GetSrtpCipher"}, + {33, &ISslConnection::StubReplyZeroed, "ExportKeyingMaterial"}, + {34, &ISslConnection::StubReplyZeroed, "SetIoTimeout"}, + {35, &ISslConnection::StubReplyZeroed, "GetIoTimeout"}, + {36, &ISslConnection::StubReplyZeroed, "Unknown36"}, + {37, &ISslConnection::StubReplyZeroed, "Unknown37"}, }; // clang-format on @@ -420,10 +422,14 @@ private: R_SUCCEED(); } - Result GetNextAlpnProto(OutBuffer<BufferAttr_HipcMapAlias> data, Out<u32> to_write) { - *to_write = u32((std::min)(next_alpn_proto.size(), data.size())); - next_alpn_proto.assign(data.begin(), data.begin() + *to_write); - LOG_DEBUG(Service_SSL, "GetNextAlpnProto called, size={}", *to_write); + // Replies with two words, the ALPN state and the size written to the buffer. It used to reply + // with one and copy the caller's (output) buffer over the protocols SetNextAlpnProto stored. + // No TLS negotiation happens here, so report that nothing was negotiated. + Result GetNextAlpnProto(OutBuffer<BufferAttr_HipcMapAlias> data, Out<u32> out_state, + Out<u32> out_size) { + LOG_DEBUG(Service_SSL, "GetNextAlpnProto called, buffer size={}", data.size()); + *out_state = 0; + *out_size = 0; R_SUCCEED(); } @@ -557,8 +563,10 @@ private: rb.Push(client_id); } - Result RegisterInternalPki() { + // Replies with a u64 (the registered certificate's id); the reply used to carry no data. + Result RegisterInternalPki(Out<u64> out_id) { LOG_WARNING(Service_SSL, "(STUBBED) called"); + *out_id = 0; R_SUCCEED(); } }; @@ -636,7 +644,7 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> explicit ISslServiceForSystem(Core::System& system_) : ServiceFramework{system_, "ssl:s"} { // clang-format off static const FunctionInfo functions[] = { - {0, D<&ISslServiceForSystem::CreateContext>, "CreateContext"}, + {0, &ISslServiceForSystem::CreateContext, "CreateContext"}, {1, D<&ISslServiceForSystem::GetContextCount>, "GetContextCount"}, {2, D<&ISslServiceForSystem::GetCertificates>, "GetCertificates"}, {3, D<&ISslServiceForSystem::GetCertificateBufSize>, "GetCertificateBufSize"}, @@ -646,7 +654,7 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> {7, D<&ISslServiceForSystem::SetDebugOption>, "SetDebugOption"}, {8, D<&ISslServiceForSystem::GetDebugOption>, "GetDebugOption"}, {9, D<&ISslServiceForSystem::ClearTls12FallbackFlag>, "ClearTls12FallbackFlag"}, - {100, D<&ISslServiceForSystem::CreateContextForSystem>, "CreateContextForSystem"}, + {100, &ISslServiceForSystem::CreateContext, "CreateContextForSystem"}, {101, D<&ISslServiceForSystem::SetThreadCoreMask>, "SetThreadCoreMask"}, {102, D<&ISslServiceForSystem::GetThreadCoreMask>, "GetThreadCoreMask"}, {103, D<&ISslServiceForSystem::VerifySignature>, "VerifySignature"} @@ -656,19 +664,30 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> RegisterHandlers(functions); }; - Result CreateContext() { - LOG_DEBUG(Service_SSL, "(STUBBED) called."); + // 0 and 100(SslVersion, padding, u64 pid placeholder; pid): an ISslContext, like ssl's + // CreateContext. The empty reply returned no object, so the caller got handle 0 + // (Minecraft calls 100). + void CreateContext(HLERequestContext& ctx) { + struct Parameters { + SslVersion ssl_version; + INSERT_PADDING_BYTES(0x4); + u64 pid_placeholder; + }; + static_assert(sizeof(Parameters) == 0x10, "Parameters is an invalid size"); - // TODO (jarrodnorwell) + IPC::RequestParser rp{ctx}; + const auto parameters = rp.PopRaw<Parameters>(); + LOG_WARNING(Service_SSL, "(STUBBED) called, command={}, api_version={}", ctx.GetCommand(), + parameters.ssl_version.api_version); - return ResultSuccess; - }; + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface<ISslContext>(ctx, system, parameters.ssl_version); + } - Result GetContextCount() { + Result GetContextCount(Out<u32> out_count) { LOG_DEBUG(Service_SSL, "(STUBBED) called."); - - // TODO (jarrodnorwell) - + *out_count = 0; return ResultSuccess; }; @@ -704,11 +723,9 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> return ResultSuccess; }; - Result FlushSessionCache() { + Result FlushSessionCache(Out<u32> out_count) { LOG_DEBUG(Service_SSL, "(STUBBED) called."); - - // TODO (jarrodnorwell) - + *out_count = 0; return ResultSuccess; }; @@ -736,14 +753,6 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> return ResultSuccess; }; - Result CreateContextForSystem() { - LOG_DEBUG(Service_SSL, "(STUBBED) called."); - - // TODO (jarrodnorwell) - - return ResultSuccess; - }; - Result SetThreadCoreMask() { LOG_DEBUG(Service_SSL, "(STUBBED) called."); @@ -752,11 +761,9 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> return ResultSuccess; }; - Result GetThreadCoreMask() { + Result GetThreadCoreMask(Out<u64> out_mask) { LOG_DEBUG(Service_SSL, "(STUBBED) called."); - - // TODO (jarrodnorwell) - + *out_mask = 0; return ResultSuccess; }; diff --git a/src/core/loader/deconstructed_rom_directory.cpp b/src/core/loader/deconstructed_rom_directory.cpp index cda1645548..04b05d188f 100644 --- a/src/core/loader/deconstructed_rom_directory.cpp +++ b/src/core/loader/deconstructed_rom_directory.cpp @@ -7,6 +7,9 @@ #include <algorithm> #include <cctype> #include <cstring> +#include <map> +#include <memory> +#include <mutex> #include <string_view> #include "common/logging.h" #include "common/random.h" @@ -90,8 +93,19 @@ FileSys::VirtualFile FindLooseRomFS(const FileSys::VirtualDir& exefs_dir) { } if (const auto parent = exefs_dir->GetParentDirectory()) { if (const auto romfs_dir = parent->GetSubdirectory("romfs")) { + // Building it walks every file and keeps an object for each (title 1: 206k files, ~150 + // MiB), so loaders of the same directory share one build while any of them holds it. + static std::mutex cache_mutex; + static std::map<std::string, std::weak_ptr<FileSys::VfsFile>> cache; + std::scoped_lock lk{cache_mutex}; + auto& cached = cache[romfs_dir->GetFullPath()]; + if (auto built = cached.lock()) { + return built; + } LOG_INFO(Loader, "Using RomFS directory next to the ExeFS"); - return FileSys::CreateRomFS(romfs_dir); + auto built = FileSys::CreateRomFS(romfs_dir); + cached = built; + return built; } } return nullptr; @@ -233,7 +247,7 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(FileSys name = control->GetApplicationName(); } - romfs = FindLooseRomFS(file_dir); + romfs_exefs_dir = file_dir; if (const auto title_dir = file_dir->GetParentDirectory()) { romfs_version = RomFSProductVersion(title_dir->GetSubdirectory("romfs")); } @@ -245,7 +259,17 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory( , dir(std::move(directory)) , override_update(override_update_) { - romfs = FindLooseRomFS(dir); + romfs_exefs_dir = dir; +} + +// Loaders made only for metadata (the game list, title lookups) never need the RomFS, and several +// of them were alive with a whole copy each. +const FileSys::VirtualFile& AppLoader_DeconstructedRomDirectory::LooseRomFS() { + if (!romfs_resolved) { + romfs_resolved = true; + romfs = FindLooseRomFS(romfs_exefs_dir); + } + return romfs; } FileType AppLoader_DeconstructedRomDirectory::IdentifyType(const FileSys::VirtualFile& dir_file) { @@ -304,7 +328,7 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect // Loose-ExeFS loads bypass the NCA/NSP loaders, which are the ones that normally apply // LayeredFS to the RomFS; apply it here so mods in user/load/<titleid>/ work too. - if (romfs != nullptr) { + if (LooseRomFS() != nullptr) { const FileSys::PatchManager patch_manager(metadata.GetTitleID(), system.GetFileSystemController(), system.GetContentProvider()); @@ -414,7 +438,7 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect } ResultStatus AppLoader_DeconstructedRomDirectory::ReadRomFS(FileSys::VirtualFile& out_dir) { - if (romfs == nullptr) { + if (LooseRomFS() == nullptr) { return ResultStatus::ErrorNoRomFS; } diff --git a/src/core/loader/deconstructed_rom_directory.h b/src/core/loader/deconstructed_rom_directory.h index b976b71ecb..db0120f4ad 100644 --- a/src/core/loader/deconstructed_rom_directory.h +++ b/src/core/loader/deconstructed_rom_directory.h @@ -58,8 +58,13 @@ public: ResultStatus ReadNSOModules(Modules& out_modules) override; private: + /// The loose RomFS next to the ExeFS, found (and built) on first use. + const FileSys::VirtualFile& LooseRomFS(); + FileSys::ProgramMetadata metadata; FileSys::VirtualFile romfs; + FileSys::VirtualDir romfs_exefs_dir; ///< Where LooseRomFS looks + bool romfs_resolved{}; FileSys::VirtualDir dir; std::vector<u8> icon_data; diff --git a/src/dynarmic/src/dynarmic/backend/block_range_information.cpp b/src/dynarmic/src/dynarmic/backend/block_range_information.cpp index 1a56b7adb6..068e86c2bc 100644 --- a/src/dynarmic/src/dynarmic/backend/block_range_information.cpp +++ b/src/dynarmic/src/dynarmic/backend/block_range_information.cpp @@ -8,7 +8,8 @@ #include "dynarmic/backend/block_range_information.h" -#include <boost/icl/interval_map.hpp> +#include <algorithm> + #include <boost/icl/interval_set.hpp> #include "common/common_types.h" #include "common/container/unordered_set.h" @@ -17,24 +18,28 @@ namespace Dynarmic::Backend { template<typename P> void BlockRangeInformation<P>::AddRange(boost::icl::discrete_interval<P> range, IR::LocationDescriptor location) { - block_ranges.add(std::make_pair(range, ::Common::unordered_set<IR::LocationDescriptor>{location})); + if (!boost::icl::is_empty(range)) { + ranges.push_back({boost::icl::first(range), boost::icl::last(range), location}); + } } template<typename P> void BlockRangeInformation<P>::ClearCache() { - block_ranges.clear(); + ranges.clear(); } template<typename P> -::Common::unordered_set<IR::LocationDescriptor> BlockRangeInformation<P>::InvalidateRanges(const boost::icl::interval_set<P>& ranges) { +::Common::unordered_set<IR::LocationDescriptor> BlockRangeInformation<P>::InvalidateRanges(const boost::icl::interval_set<P>& invalidated) { ::Common::unordered_set<IR::LocationDescriptor> erase_locations; - for (auto invalidate_interval : ranges) { - auto pair = block_ranges.equal_range(invalidate_interval); - for (auto it = pair.first; it != pair.second; ++it) - for (const auto& descriptor : it->second) - erase_locations.insert(descriptor); - } - // TODO: EFFICIENCY: Remove ranges that are to be erased. + std::erase_if(ranges, [&](const Range& range) { + for (const auto& interval : invalidated) { + if (range.first <= boost::icl::last(interval) && boost::icl::first(interval) <= range.last) { + erase_locations.insert(range.location); + return true; + } + } + return false; + }); return erase_locations; } diff --git a/src/dynarmic/src/dynarmic/backend/block_range_information.h b/src/dynarmic/src/dynarmic/backend/block_range_information.h index 8f7daa9d9c..f894240603 100644 --- a/src/dynarmic/src/dynarmic/backend/block_range_information.h +++ b/src/dynarmic/src/dynarmic/backend/block_range_information.h @@ -8,11 +8,9 @@ #pragma once -#include <set> +#include <vector> -#include <boost/icl/interval_map.hpp> #include <boost/icl/interval_set.hpp> -#include "common/container/unordered_map.h" #include "common/container/unordered_set.h" #include "dynarmic/ir/location_descriptor.h" @@ -24,8 +22,18 @@ class BlockRangeInformation { public: void AddRange(boost::icl::discrete_interval<P> range, IR::LocationDescriptor location); void ClearCache(); - ::Common::unordered_set<IR::LocationDescriptor> InvalidateRanges(const boost::icl::interval_set<P>& ranges); - boost::icl::interval_map<P, ::Common::unordered_set<IR::LocationDescriptor>> block_ranges; + ::Common::unordered_set<IR::LocationDescriptor> InvalidateRanges(const boost::icl::interval_set<P>& invalidated); + +private: + /// The guest code [first, last] a block was translated from. Code is invalidated rarely (when + /// it is unmapped or reprotected), so a list scanned then costs least: an interval map of + /// block sets held ~300 bytes per block. + struct Range { + P first; + P last; + IR::LocationDescriptor location; + }; + std::vector<Range> ranges; }; } // namespace Dynarmic::Backend diff --git a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cpp b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cpp index 5c6a90e15a..e0e3c463d1 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cpp +++ b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cpp @@ -1168,7 +1168,7 @@ bool EmitTerminalImpl(A32EmitX64& e, IR::Term::LinkBlock terminal, IR::LocationD } else { if (e.conf.enable_cycle_counting) { e.code.cmp(qword[rsp + ABI_SHADOW_SPACE + offsetof(StackLayout, cycles_remaining)], 0); - e.patch_information[terminal.next].jg.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jg); if (const auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJg(terminal.next, next_bb->entrypoint); } else { @@ -1176,7 +1176,7 @@ bool EmitTerminalImpl(A32EmitX64& e, IR::Term::LinkBlock terminal, IR::LocationD } } else { e.code.cmp(dword[e.code.ABI_JIT_PTR + offsetof(A32JitState, halt_reason)], 0); - e.patch_information[terminal.next].jz.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jz); if (const auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJz(terminal.next, next_bb->entrypoint); } else { @@ -1196,7 +1196,7 @@ bool EmitTerminalImpl(A32EmitX64& e, IR::Term::LinkBlockFast terminal, IR::Locat e.code.mov(MJitStateReg(A32::Reg::PC), A32::LocationDescriptor{terminal.next}.PC()); e.code.ReturnFromRunCode(); } else { - e.patch_information[terminal.next].jmp.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jmp); if (const auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJmp(terminal.next, next_bb->entrypoint); } else { diff --git a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h index a106bb6482..eb36bffd88 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h +++ b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h @@ -87,13 +87,17 @@ public: // Fastmem information using DoNotFastmemMarker = std::tuple<IR::LocationDescriptor, unsigned>; + /// Where a fastmem access that faults goes: code addresses are offsets in the code space. struct FastmemPatchInfo { - u64 resume_rip; - u64 callback; - DoNotFastmemMarker marker; - bool recompile; + u32 rip; + u32 resume_rip; + u32 callback; + u32 inst_name : 31; + u32 recompile : 1; + IR::LocationDescriptor location; }; std::optional<DoNotFastmemMarker> ShouldFastmem(A32EmitContext& ctx, IR::Inst* inst) const; + void AddFastmemPatch(CodePtr location, CodePtr resume_location, void (*callback)(), const DoNotFastmemMarker& marker, bool recompile); FakeCall FastmemCallback(u64 rip); // Memory access helpers @@ -127,7 +131,7 @@ public: RegAlloc reg_alloc; //reusable reg alloc BlockRangeInformation<u32> block_ranges; std::array<FastDispatchEntry, fast_dispatch_table_size> fast_dispatch_table; - ::Common::unordered_map<u64, FastmemPatchInfo> fastmem_patch_info; + std::vector<FastmemPatchInfo> fastmem_patch_info; // sorted by rip ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> read_fallbacks; ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> write_fallbacks; ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> exclusive_write_fallbacks; diff --git a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cpp b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cpp index 71e5e62cd9..2307226744 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cpp +++ b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cpp @@ -628,7 +628,7 @@ bool EmitTerminalImpl(A64EmitX64& e, IR::Term::LinkBlock terminal, IR::LocationD if (e.conf.HasOptimization(OptimizationFlag::BlockLinking) && !is_single_step) { if (e.conf.enable_cycle_counting) { e.code.cmp(qword[rsp + ABI_SHADOW_SPACE + offsetof(StackLayout, cycles_remaining)], 0); - e.patch_information[terminal.next].jg.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jg); if (const auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJg(terminal.next, next_bb->entrypoint); } else { @@ -636,7 +636,7 @@ bool EmitTerminalImpl(A64EmitX64& e, IR::Term::LinkBlock terminal, IR::LocationD } } else { e.code.cmp(dword[e.code.ABI_JIT_PTR + offsetof(A64JitState, halt_reason)], 0); - e.patch_information[terminal.next].jz.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jz); if (const auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJz(terminal.next, next_bb->entrypoint); } else { @@ -656,7 +656,7 @@ bool EmitTerminalImpl(A64EmitX64& e, IR::Term::LinkBlock terminal, IR::LocationD bool EmitTerminalImpl(A64EmitX64& e, IR::Term::LinkBlockFast terminal, IR::LocationDescriptor, bool is_single_step) { if (e.conf.HasOptimization(OptimizationFlag::BlockLinking) && !is_single_step) { - e.patch_information[terminal.next].jmp.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jmp); if (auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJmp(terminal.next, next_bb->entrypoint); } else { diff --git a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h index 1310b4ad34..a9f2d9b650 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h +++ b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h @@ -83,13 +83,17 @@ public: // Fastmem information using DoNotFastmemMarker = std::tuple<IR::LocationDescriptor, unsigned>; + /// Where a fastmem access that faults goes: code addresses are offsets in the code space. struct FastmemPatchInfo { - u64 resume_rip; - u64 callback; - DoNotFastmemMarker marker; - bool recompile; + u32 rip; + u32 resume_rip; + u32 callback; + u32 inst_name : 31; + u32 recompile : 1; + IR::LocationDescriptor location; }; std::optional<DoNotFastmemMarker> ShouldFastmem(A64EmitContext& ctx, IR::Inst* inst) const; + void AddFastmemPatch(CodePtr location, CodePtr resume_location, void (*callback)(), const DoNotFastmemMarker& marker, bool recompile); FakeCall FastmemCallback(u64 rip); // Memory access helpers @@ -123,7 +127,7 @@ public: RegAlloc reg_alloc; //reusable reg alloc BlockRangeInformation<u64> block_ranges; std::array<FastDispatchEntry, fast_dispatch_table_size> fast_dispatch_table; - ::Common::unordered_map<u64, FastmemPatchInfo> fastmem_patch_info; + std::vector<FastmemPatchInfo> fastmem_patch_info; // sorted by rip ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> read_fallbacks; ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> write_fallbacks; ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> exclusive_write_fallbacks; diff --git a/src/dynarmic/src/dynarmic/backend/x64/block_of_code.cpp b/src/dynarmic/src/dynarmic/backend/x64/block_of_code.cpp index 384d6203a4..57001d4ef0 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/block_of_code.cpp +++ b/src/dynarmic/src/dynarmic/backend/x64/block_of_code.cpp @@ -60,6 +60,34 @@ namespace { constexpr size_t CONSTANT_POOL_SIZE = 2 * 1024 * 1024; constexpr size_t PRELUDE_COMMIT_SIZE = 16 * 1024 * 1024; +#ifdef _WIN32 +constexpr size_t COMMIT_STEP = 4 * 1024 * 1024; + +// Reserves the code space; EnsureMemoryCommitted commits it as code is emitted. Xbyak's default +// allocator commits all of it up front: 512 MiB per core, 3 GiB of commit charge for six cores +// of which a game fills a fraction. +struct ReservingAllocator final : Xbyak::Allocator { + uint8_t* alloc(size_t size) override { + return static_cast<uint8_t*>(VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_NOACCESS)); + } + void free(uint8_t* p) override { + VirtualFree(p, 0, MEM_RELEASE); + } + bool useProtect() const override { + return false; + } +}; + +Xbyak::Allocator* CodeAllocator() { + // Never destroyed: code blocks may outlive static destruction. + static auto* const allocator = new ReservingAllocator; + return allocator; +} +#else +Xbyak::Allocator* CodeAllocator() { + return nullptr; +} +#endif #ifdef DYNARMIC_ENABLE_NO_EXECUTE_SUPPORT void ProtectMemory(const void* base, size_t size, bool is_executable) { @@ -149,7 +177,7 @@ BlockOfCode::BlockOfCode(RunCodeCallbacks cb, JitStateInfo jsi, size_t total_cod #else , nullptr //Allow RWE #endif - , nullptr) + , CodeAllocator()) , constant_pool(*this, CONSTANT_POOL_SIZE) , jsi(jsi) , cb(std::move(cb)) @@ -206,7 +234,9 @@ size_t BlockOfCode::SpaceRemaining() const { void BlockOfCode::EnsureMemoryCommitted([[maybe_unused]] size_t codesize) { #ifdef _WIN32 if (committed_size < size_ + codesize) { - committed_size = std::min<size_t>(maxSize_, committed_size + codesize); + // The code space is only reserved (CodeAllocator), so everything up to the end of the next + // emission has to be committed; in steps, as each VirtualAlloc call is a kernel call. + committed_size = std::min<size_t>(maxSize_, (std::max)(committed_size + COMMIT_STEP, size_ + codesize)); # ifdef DYNARMIC_ENABLE_NO_EXECUTE_SUPPORT VirtualAlloc(top_, committed_size, MEM_COMMIT, PAGE_READWRITE); # else diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cpp b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cpp index b9df3987ca..84b9888bf5 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cpp +++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cpp @@ -102,7 +102,7 @@ void EmitX64::PushRSBHelper(Xbyak::Reg64 loc_desc_reg, Xbyak::Reg64 index_reg, I code.mov(index_reg.cvt32(), dword[code.ABI_JIT_PTR + code.GetJitStateInfo().offsetof_rsb_ptr]); code.mov(loc_desc_reg, target.Value()); - patch_information[target].mov_rcx.push_back(code.getCurr()); + AddPatchLocation(target, PatchKind::MovRcx); EmitPatchMovRcx(target_code_ptr); code.mov(qword[code.ABI_JIT_PTR + index_reg * 8 + code.GetJitStateInfo().offsetof_rsb_location_descriptors], loc_desc_reg); code.mov(qword[code.ABI_JIT_PTR + index_reg * 8 + code.GetJitStateInfo().offsetof_rsb_codeptrs], rcx); @@ -357,37 +357,61 @@ EmitX64::BlockDescriptor EmitX64::RegisterBlock(const IR::LocationDescriptor& de return block_desc; } -void EmitX64::Patch(const IR::LocationDescriptor& target_desc, CodePtr target_code_ptr) { - const CodePtr save_code_ptr = code.getCurr(); - const PatchInformation& patch_info = patch_information[target_desc]; - - for (CodePtr location : patch_info.jg) { - code.SetCodePtr(location); - EmitPatchJg(target_desc, target_code_ptr); +void EmitX64::AddPatchLocation(const IR::LocationDescriptor& target_desc, PatchKind kind) { + const size_t offset = code.getCurr<const u8*>() - static_cast<const u8*>(code.getCode()); + ASSERT(offset > 0 && offset < (size_t(1) << 30)); + const u32 entry = u32(offset << 2) | u32(kind); + PatchInformation& patch_info = patch_information[target_desc]; + for (u32& slot : patch_info.first) { + if (slot == 0) { + slot = entry; + return; + } } - - for (CodePtr location : patch_info.jz) { - code.SetCodePtr(location); - EmitPatchJz(target_desc, target_code_ptr); + if (!patch_info.rest) { + patch_info.rest = std::make_unique<std::vector<u32>>(); } + patch_info.rest->push_back(entry); +} - for (CodePtr location : patch_info.jmp) { - code.SetCodePtr(location); - EmitPatchJmp(target_desc, target_code_ptr); +void EmitX64::Patch(const IR::LocationDescriptor& target_desc, CodePtr target_code_ptr) { + const auto it = patch_information.find(target_desc); + if (it == patch_information.end()) { + return; } - - for (CodePtr location : patch_info.mov_rcx) { - code.SetCodePtr(location); - EmitPatchMovRcx(target_code_ptr); + const CodePtr save_code_ptr = code.getCurr(); + const auto patch = [&](u32 entry) { + code.SetCodePtr(static_cast<const u8*>(code.getCode()) + (entry >> 2)); + switch (PatchKind(entry & 3)) { + case PatchKind::Jg: + EmitPatchJg(target_desc, target_code_ptr); + break; + case PatchKind::Jz: + EmitPatchJz(target_desc, target_code_ptr); + break; + case PatchKind::Jmp: + EmitPatchJmp(target_desc, target_code_ptr); + break; + case PatchKind::MovRcx: + EmitPatchMovRcx(target_code_ptr); + break; + } + }; + for (const u32 entry : it->second.first) { + if (entry != 0) { + patch(entry); + } + } + if (it->second.rest) { + for (const u32 entry : *it->second.rest) { + patch(entry); + } } - code.SetCodePtr(save_code_ptr); } void EmitX64::Unpatch(const IR::LocationDescriptor& target_desc) { - if (patch_information.count(target_desc)) { - Patch(target_desc, nullptr); - } + Patch(target_desc, nullptr); } void EmitX64::ClearCache() { diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.h b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.h index 5e540e5f70..327cac7668 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.h +++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.h @@ -116,12 +116,16 @@ public: virtual bool EmitTerminal(IR::Term::Terminal const& terminal, IR::LocationDescriptor initial_location, bool is_single_step) noexcept = 0; // Patching + enum class PatchKind : u32 { Jg, Jz, Jmp, MovRcx }; + /// Places in emitted code that jump to a block (or load its entrypoint), patched when it is + /// compiled or invalidated: each the place's offset in the code space times four plus its + /// PatchKind, 0 when unused. Most blocks are linked from one or two places. struct PatchInformation { - boost::container::small_vector<CodePtr, 4> jg; //4*8=32 - boost::container::small_vector<CodePtr, 4> jz; //4*8=32 - boost::container::small_vector<CodePtr, 4> jmp; //4*8=32 - boost::container::small_vector<CodePtr, 4> mov_rcx; //4*8=32 + std::array<u32, 2> first{}; + std::unique_ptr<std::vector<u32>> rest; }; + /// Records that the code about to be emitted at the current position is a patch of `kind`. + void AddPatchLocation(const IR::LocationDescriptor& target_desc, PatchKind kind); void Patch(const IR::LocationDescriptor& target_desc, CodePtr target_code_ptr); virtual void Unpatch(const IR::LocationDescriptor& target_desc); virtual void EmitPatchJg(const IR::LocationDescriptor& target_desc, CodePtr target_code_ptr = nullptr) = 0; diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc index 8bb8be3d5b..689e507abe 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc +++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc @@ -30,15 +30,48 @@ std::optional<AxxEmitX64::DoNotFastmemMarker> AxxEmitX64::ShouldFastmem(AxxEmitC return marker; } +void AxxEmitX64::AddFastmemPatch(CodePtr location, CodePtr resume_location, void (*callback)(), const DoNotFastmemMarker& marker, bool recompile) { + const u64 base = std::bit_cast<u64>(code.getCode()); + const auto offset = [base](u64 address) { + ASSERT(address >= base && address - base <= UINT32_MAX); + return u32(address - base); + }; + FastmemPatchInfo info{ + .rip = offset(std::bit_cast<u64>(location)), + .resume_rip = offset(std::bit_cast<u64>(resume_location)), + .callback = offset(std::bit_cast<u64>(callback)), + .inst_name = 0, + .recompile = 0, + .location = std::get<0>(marker), + }; + info.inst_name = std::get<1>(marker); + info.recompile = recompile ? 1 : 0; + // Code is emitted in address order; only a block's deferred fallbacks can come after a later + // access of the same block. + auto it = fastmem_patch_info.end(); + if (!fastmem_patch_info.empty() && fastmem_patch_info.back().rip >= info.rip) { + it = std::lower_bound(fastmem_patch_info.begin(), fastmem_patch_info.end(), info.rip, + [](const FastmemPatchInfo& entry, u32 value) { return entry.rip < value; }); + if (it != fastmem_patch_info.end() && it->rip == info.rip) { + return; + } + } + fastmem_patch_info.insert(it, info); +} + FakeCall AxxEmitX64::FastmemCallback(u64 rip_) { - const auto iter = fastmem_patch_info.find(rip_); - if (iter != fastmem_patch_info.end()) { + const u64 base = std::bit_cast<u64>(code.getCode()); + const auto iter = rip_ >= base && rip_ - base <= UINT32_MAX + ? std::lower_bound(fastmem_patch_info.begin(), fastmem_patch_info.end(), u32(rip_ - base), + [](const FastmemPatchInfo& entry, u32 value) { return entry.rip < value; }) + : fastmem_patch_info.end(); + if (iter != fastmem_patch_info.end() && iter->rip == u32(rip_ - base)) { FakeCall result{ - .call_rip = iter->second.callback, - .ret_rip = iter->second.resume_rip, + .call_rip = base + iter->callback, + .ret_rip = base + iter->resume_rip, }; - if (iter->second.recompile) { - const auto marker = iter->second.marker; + if (iter->recompile) { + const DoNotFastmemMarker marker{iter->location, static_cast<unsigned>(iter->inst_name)}; do_not_fastmem.insert(marker); InvalidateBasicBlocks({std::get<0>(marker)}); } @@ -99,14 +132,7 @@ void AxxEmitX64::EmitMemoryRead(AxxEmitContext& ctx, IR::Inst* inst) { code.L(*abort); code.call(wrapped_fn); - fastmem_patch_info.emplace( - std::bit_cast<u64>(location), - FastmemPatchInfo{ - std::bit_cast<u64>(code.getCurr()), - std::bit_cast<u64>(wrapped_fn), - *fastmem_marker, - conf.recompile_on_fastmem_failure, - }); + AddFastmemPatch(location, code.getCurr(), wrapped_fn, *fastmem_marker, conf.recompile_on_fastmem_failure); EmitCheckMemoryAbort(ctx, inst, end); code.jmp(*end, code.T_NEAR); @@ -186,14 +212,7 @@ void AxxEmitX64::EmitMemoryWrite(AxxEmitContext& ctx, IR::Inst* inst) { code.L(*abort); code.call(wrapped_fn); - fastmem_patch_info.emplace( - std::bit_cast<u64>(location), - FastmemPatchInfo{ - std::bit_cast<u64>(code.getCurr()), - std::bit_cast<u64>(wrapped_fn), - *fastmem_marker, - conf.recompile_on_fastmem_failure, - }); + AddFastmemPatch(location, code.getCurr(), wrapped_fn, *fastmem_marker, conf.recompile_on_fastmem_failure); EmitCheckMemoryAbort(ctx, inst, end); code.jmp(*end, code.T_NEAR); @@ -357,14 +376,7 @@ void AxxEmitX64::EmitExclusiveReadMemoryInline(AxxEmitContext& ctx, IR::Inst* in const auto location = EmitReadMemoryMov<bitsize>(code, value_idx, src_ptr, ordered); - fastmem_patch_info.emplace( - std::bit_cast<u64>(location), - FastmemPatchInfo{ - std::bit_cast<u64>(code.getCurr()), - std::bit_cast<u64>(wrapped_fn), - *fastmem_marker, - conf.recompile_on_exclusive_fastmem_failure, - }); + AddFastmemPatch(location, code.getCurr(), wrapped_fn, *fastmem_marker, conf.recompile_on_exclusive_fastmem_failure); code.L(*end); @@ -495,14 +507,7 @@ void AxxEmitX64::EmitExclusiveWriteMemoryInline(AxxEmitContext& ctx, IR::Inst* i code.L(*abort); code.call(wrapped_fn); - fastmem_patch_info.emplace( - std::bit_cast<u64>(location), - FastmemPatchInfo{ - std::bit_cast<u64>(code.getCurr()), - std::bit_cast<u64>(wrapped_fn), - *fastmem_marker, - conf.recompile_on_exclusive_fastmem_failure, - }); + AddFastmemPatch(location, code.getCurr(), wrapped_fn, *fastmem_marker, conf.recompile_on_exclusive_fastmem_failure); code.xor_(status.cvt32(), status.cvt32()); //dep-break code.test(code.al, code.al); diff --git a/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.cpp b/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.cpp index 5a828839c1..4f3d253441 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.cpp +++ b/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.cpp @@ -465,22 +465,24 @@ std::optional<HostLoc> RegAlloc::ValueLocation(const IR::Inst* value) const noex return std::nullopt; } +// The checks that scan every location for def_inst are debug-only: per IR instruction they cost as +// much as the rest of register allocation. void RegAlloc::DefineValueImpl(BlockOfCode& code, IR::Inst* def_inst, HostLoc host_loc) noexcept { - ASSERT(!ValueLocation(def_inst) && "def_inst has already been defined"); + DEBUG_ASSERT(!ValueLocation(def_inst) && "def_inst has already been defined"); LocInfo(host_loc).AddValue(host_loc, def_inst); - ASSERT(*ValueLocation(def_inst) == host_loc); + DEBUG_ASSERT(*ValueLocation(def_inst) == host_loc); } void RegAlloc::DefineValueImpl(BlockOfCode& code, IR::Inst* def_inst, const IR::Value& use_inst) noexcept { - ASSERT(!ValueLocation(def_inst) && "def_inst has already been defined"); + DEBUG_ASSERT(!ValueLocation(def_inst) && "def_inst has already been defined"); if (use_inst.IsImmediate()) { const HostLoc location = ScratchImpl(code, gpr_order); DefineValueImpl(code, def_inst, location); LoadImmediate(code, use_inst, location); } else { - ASSERT(ValueLocation(use_inst.GetInst()) && "use_inst must already be defined"); - const HostLoc location = *ValueLocation(use_inst.GetInst()); - DefineValueImpl(code, def_inst, location); + const auto location = ValueLocation(use_inst.GetInst()); + ASSERT(location && "use_inst must already be defined"); + DefineValueImpl(code, def_inst, *location); } } diff --git a/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.h b/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.h index 455c6a970a..0cad07796c 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.h +++ b/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.h @@ -65,6 +65,11 @@ public: } void ReleaseOne() noexcept; void ReleaseAll() noexcept; + /// True when ReleaseAll would change nothing, as for most of the spill slots most of the time. + inline bool IsReleased() const noexcept { + return current_references == 0 && is_being_used_count == 0 && !is_scratch && !is_set_last_use && + (total_uses != accumulated_uses || (values.empty() && total_uses == 0 && max_bit_width == 0)); + } constexpr size_t GetMaxBitWidth() const noexcept { return 1 << max_bit_width; } void AddValue(HostLoc loc, IR::Inst* inst) noexcept; /// Checks if the given instruction is in our values set @@ -201,8 +206,10 @@ public: void ReleaseStackSpace(BlockOfCode& code, const size_t stack_space) noexcept; inline void EndOfAllocScope() noexcept { + // After every IR instruction: skip the locations it left alone. for (auto& iter : hostloc_info) - iter.ReleaseAll(); + if (!iter.IsReleased()) + iter.ReleaseAll(); } inline void AssertNoMoreUses() noexcept { ASSERT(std::all_of(hostloc_info.begin(), hostloc_info.end(), [](const auto& i) noexcept { return i.IsEmpty(); })); diff --git a/src/shader_recompiler/CMakeLists.txt b/src/shader_recompiler/CMakeLists.txt index e60f26d47f..e4df9c045d 100644 --- a/src/shader_recompiler/CMakeLists.txt +++ b/src/shader_recompiler/CMakeLists.txt @@ -24,6 +24,7 @@ add_library(shader_recompiler STATIC backend/spirv/emit_spirv_memory.cpp backend/spirv/emit_spirv_select.cpp backend/spirv/emit_spirv_shared_memory.cpp + backend/spirv/emit_spirv_mesh.cpp backend/spirv/emit_spirv_special.cpp backend/spirv/emit_spirv_undefined.cpp backend/spirv/emit_spirv_warp.cpp diff --git a/src/shader_recompiler/backend/glasm/emit_glasm.cpp b/src/shader_recompiler/backend/glasm/emit_glasm.cpp index b795c01797..c18eec63cd 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm.cpp +++ b/src/shader_recompiler/backend/glasm/emit_glasm.cpp @@ -341,6 +341,9 @@ std::string_view StageHeader(Stage stage) { return "!!NVfp5.0\n"; case Stage::Compute: return "!!NVcp5.0\n"; + case Stage::Task: + case Stage::Mesh: + break; } throw InvalidArgument("Invalid stage {}", stage); } diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h index 6f522ab5c2..d66fe2f3d6 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h +++ b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h @@ -58,6 +58,10 @@ void EmitGetAttributeIndexed(EmitContext& ctx, IR::Inst& inst, ScalarS32 offset, void EmitSetAttributeIndexed(EmitContext& ctx, ScalarU32 offset, ScalarF32 value, ScalarU32 vertex); void EmitGetPatch(EmitContext& ctx, IR::Inst& inst, IR::Patch patch); void EmitSetPatch(EmitContext& ctx, IR::Patch patch, ScalarF32 value); +void EmitIsbeRead(EmitContext& ctx); +void EmitIsbeWriteAttribute(EmitContext& ctx); +void EmitIsbeWriteMapU8(EmitContext& ctx); +void EmitIsbeWriteMapU32(EmitContext& ctx); void EmitSetFragColor(EmitContext& ctx, u32 index, u32 component, ScalarF32 value); void EmitSetSampleMask(EmitContext& ctx, ScalarS32 value); void EmitSetFragDepth(EmitContext& ctx, ScalarF32 value); diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp b/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp index 1f4ffdd624..41c829ca5d 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp +++ b/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp @@ -97,4 +97,20 @@ void EmitGetInBoundsFromOp(EmitContext& ctx) { NotImplemented(); } +void EmitIsbeRead(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteAttribute(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteMapU8(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteMapU32(EmitContext& ctx) { + NotImplemented(); +} + } // namespace Shader::Backend::GLASM diff --git a/src/shader_recompiler/backend/glasm/glasm_emit_context.cpp b/src/shader_recompiler/backend/glasm/glasm_emit_context.cpp index 333a91cc58..02b6992b52 100644 --- a/src/shader_recompiler/backend/glasm/glasm_emit_context.cpp +++ b/src/shader_recompiler/backend/glasm/glasm_emit_context.cpp @@ -80,6 +80,9 @@ EmitContext::EmitContext(IR::Program& program, Bindings& bindings, const Profile case Stage::Compute: stage_name = "invocation"; break; + case Stage::Task: + case Stage::Mesh: + throw NotImplementedException("GLASM task and mesh programs"); } const std::string_view attr_stage{stage == Stage::Fragment ? "fragment" : "vertex"}; const VaryingState loads{info.loads.mask | info.passthrough.mask}; diff --git a/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h b/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h index 96aac502d5..06069d4c56 100644 --- a/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h +++ b/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h @@ -72,6 +72,10 @@ void EmitSetAttributeIndexed(EmitContext& ctx, std::string_view offset, std::str std::string_view vertex); void EmitGetPatch(EmitContext& ctx, IR::Inst& inst, IR::Patch patch); void EmitSetPatch(EmitContext& ctx, IR::Patch patch, std::string_view value); +void EmitIsbeRead(EmitContext& ctx); +void EmitIsbeWriteAttribute(EmitContext& ctx); +void EmitIsbeWriteMapU8(EmitContext& ctx); +void EmitIsbeWriteMapU32(EmitContext& ctx); void EmitSetFragColor(EmitContext& ctx, u32 index, u32 component, std::string_view value); void EmitSetSampleMask(EmitContext& ctx, std::string_view value); void EmitSetFragDepth(EmitContext& ctx, std::string_view value); diff --git a/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp b/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp index 9f1ed95a47..18779e51b8 100644 --- a/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp +++ b/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp @@ -97,4 +97,20 @@ void EmitGetInBoundsFromOp(EmitContext& ctx) { NotImplemented(); } +void EmitIsbeRead(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteAttribute(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteMapU8(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteMapU32(EmitContext& ctx) { + NotImplemented(); +} + } // namespace Shader::Backend::GLSL diff --git a/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp b/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp index a2df6159fb..e4b11d6844 100644 --- a/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp +++ b/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp @@ -317,6 +317,9 @@ EmitContext::EmitContext(IR::Program& program, Bindings& bindings, const Profile header += "layout(early_fragment_tests)in;"; } break; + case Stage::Task: + case Stage::Mesh: + throw NotImplementedException("GLSL task and mesh programs"); case Stage::Compute: stage_name = "cs"; const u32 local_x{(std::max)(program.workgroup_size[0], 1u)}; diff --git a/src/shader_recompiler/backend/spirv/emit_spirv.cpp b/src/shader_recompiler/backend/spirv/emit_spirv.cpp index a903480c78..633ab25935 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv.cpp @@ -248,6 +248,20 @@ void DefineEntryPoint(const IR::Program& program, EmitContext& ctx, Id main) { case Stage::VertexB: execution_model = spv::ExecutionModel::Vertex; break; + case Stage::Task: + execution_model = spv::ExecutionModel::TaskEXT; + ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, program.workgroup_size[0], 1U, + 1U); + break; + case Stage::Mesh: + execution_model = spv::ExecutionModel::MeshEXT; + ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, program.workgroup_size[0], 1U, + 1U); + ctx.AddExecutionMode(main, spv::ExecutionMode::OutputVertices, program.mesh.max_vertices); + ctx.AddExecutionMode(main, spv::ExecutionMode::OutputPrimitivesEXT, + program.mesh.max_primitives); + ctx.AddExecutionMode(main, spv::ExecutionMode::OutputTrianglesEXT); + break; case Stage::TessellationControl: execution_model = spv::ExecutionModel::TessellationControl; ctx.AddCapability(spv::Capability::Tessellation); @@ -548,6 +562,51 @@ void RewriteOpcodes(std::vector<u32>& code, std::span<const std::pair<u32, spv:: offset += word_count; } } + +// Replaces the OpStore markers of the mesh instructions Sirit cannot emit: OpStore(vertices, +// primitives) becomes OpSetMeshOutputsEXT, and OpStore(x, y), OpStore(z, payload), OpReturn +// becomes the OpEmitMeshTasksEXT terminator. +void PatchMeshInstructions(std::vector<u32>& code, const EmitContext& ctx) { + const bool set_outputs{Sirit::ValidId(ctx.mesh_outputs_marker[0])}; + const bool emit_tasks{Sirit::ValidId(ctx.mesh_tasks_marker[0])}; + if (!set_outputs && !emit_tasks) { + return; + } + const auto store_at = [&](size_t offset, Id pointer, Id object) { + return offset + 2 < code.size() && + code[offset] == ((3U << 16) | static_cast<u32>(spv::Op::OpStore)) && + code[offset + 1] == pointer.value && code[offset + 2] == object.value; + }; + const auto& tasks{ctx.mesh_tasks_marker}; + size_t offset = 5; + bool patched{}; + while (offset < code.size()) { + const u32 word_count{code[offset] >> 16}; + if (word_count == 0) { + break; + } + if (set_outputs && + store_at(offset, ctx.mesh_outputs_marker[0], ctx.mesh_outputs_marker[1])) { + code[offset] = (3U << 16) | static_cast<u32>(spv::Op::OpSetMeshOutputsEXT); + patched = true; + } else if (emit_tasks && store_at(offset, tasks[0], tasks[1]) && + store_at(offset + 3, tasks[2], tasks[3]) && offset + 6 < code.size() && + code[offset + 6] == ((1U << 16) | static_cast<u32>(spv::Op::OpReturn))) { + const std::array<u32, 5> emit{(5U << 16) | static_cast<u32>(spv::Op::OpEmitMeshTasksEXT), + tasks[0].value, tasks[1].value, tasks[2].value, + tasks[3].value}; + code.erase(code.begin() + offset, code.begin() + offset + 7); + code.insert(code.begin() + offset, emit.begin(), emit.end()); + patched = true; + offset += emit.size(); + continue; + } + offset += word_count; + } + if (!patched) { + throw LogicError("Mesh instruction markers not found"); + } +} } // Anonymous namespace std::vector<u32> EmitSPIRV(const Profile& profile, const RuntimeInfo& runtime_info, IR::Program& program, Bindings& bindings) { @@ -564,6 +623,7 @@ std::vector<u32> EmitSPIRV(const Profile& profile, const RuntimeInfo& runtime_in PatchPhiNodes(program, ctx); std::vector<u32> code{ctx.Assemble()}; RewriteOpcodes(code, ctx.opcode_rewrites); + PatchMeshInstructions(code, ctx); return code; } diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h index 165ec3fe84..ec526ff354 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h +++ b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h @@ -61,6 +61,12 @@ Id EmitGetAttributeIndexed(EmitContext& ctx, Id offset, Id vertex); void EmitSetAttributeIndexed(EmitContext& ctx, Id offset, Id value, Id vertex); Id EmitGetPatch(EmitContext& ctx, IR::Patch patch); void EmitSetPatch(EmitContext& ctx, IR::Patch patch, Id value); +Id EmitIsbeRead(EmitContext& ctx, Id address); +void EmitIsbeWriteAttribute(EmitContext& ctx, Id address, Id value); +void EmitIsbeWriteMapU8(EmitContext& ctx, Id address, Id value); +void EmitIsbeWriteMapU32(EmitContext& ctx, const IR::Value& address, Id value); +void EmitMeshPrologue(EmitContext& ctx); +void EmitMeshEpilogue(EmitContext& ctx); void EmitSetFragColor(EmitContext& ctx, u32 index, u32 component, Id value); void EmitSetSampleMask(EmitContext& ctx, Id value); void EmitSetFragDepth(EmitContext& ctx, Id value); diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_mesh.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_mesh.cpp new file mode 100644 index 0000000000..6c599528cc --- /dev/null +++ b/src/shader_recompiler/backend/spirv/emit_spirv_mesh.cpp @@ -0,0 +1,227 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// Switch 2 (NVN) task and mesh programs. They talk to the hardware through the stage buffer +// (ISBE): a task program writes the number of mesh workgroups at byte 4 of its output region and +// its payload after it; each mesh workgroup reads that region (its own index at byte 0) and +// writes its vertices and primitives to an attribute region in groups of 32, one 32-word line per +// attribute, plus the primitive count and 8-bit vertex indices to a map region. Here the task's +// output region is the task payload, the mesh output regions live in workgroup memory, and the +// epilogue (reached by every invocation, see structured_control_flow.cpp) hands them to Vulkan. + +#include <algorithm> +#include <span> + +#include "shader_recompiler/backend/spirv/emit_spirv.h" +#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h" +#include "shader_recompiler/backend/spirv/spirv_emit_context.h" + +namespace Shader::Backend::SPIRV { +namespace { +void Barrier(EmitContext& ctx) { + ctx.OpControlBarrier(ctx.Const(static_cast<u32>(spv::Scope::Workgroup)), + ctx.Const(static_cast<u32>(spv::Scope::Workgroup)), + ctx.Const(static_cast<u32>(spv::MemorySemanticsMask::AcquireRelease | + spv::MemorySemanticsMask::WorkgroupMemory))); +} + +Id LocalInvocation(EmitContext& ctx) { + return ctx.OpCompositeExtract(ctx.U32[1], ctx.OpLoad(ctx.U32[3], ctx.local_invocation_id), 0U); +} + +Id Clamped(EmitContext& ctx, Id index, u32 count) { + return ctx.OpUMin(ctx.U32[1], index, ctx.Const(count - 1)); +} + +Id PayloadWord(EmitContext& ctx, Id index) { + return ctx.OpAccessChain(ctx.mesh_payload_u32, ctx.mesh_payload, + Clamped(ctx, index, ctx.mesh_info.payload_words)); +} + +Id SharedWord(EmitContext& ctx, Id array, Id index) { + return ctx.OpAccessChain(ctx.mesh_shared_u32, array, index); +} + +// Runs body under `if (condition)`. +template <typename Func> +void If(EmitContext& ctx, Id condition, Func&& body) { + const Id then_label{ctx.OpLabel()}; + const Id merge_label{ctx.OpLabel()}; + ctx.OpSelectionMerge(merge_label, spv::SelectionControlMask::MaskNone); + ctx.OpBranchConditional(condition, then_label, merge_label); + ctx.AddLabel(then_label); + body(); + ctx.OpBranch(merge_label); + ctx.AddLabel(merge_label); +} + +// Copies the lines of element `index` (a vertex or a primitive) from the attribute region. +void CopyLines(EmitContext& ctx, Id index, u32 base, u32 num_lines, + const std::vector<EmitContext::MeshLineTarget>& targets) { + // Word of line 0: base + (index / 32) * num_lines * 32 + index % 32. + const Id group{ctx.OpShiftRightLogical(ctx.U32[1], index, ctx.Const(5U))}; + const Id slot{ctx.OpBitwiseAnd(ctx.U32[1], index, ctx.Const(31U))}; + const Id first{ctx.OpIAdd(ctx.U32[1], + ctx.OpIMul(ctx.U32[1], group, ctx.Const(num_lines * 32)), + ctx.OpIAdd(ctx.U32[1], slot, ctx.Const(base)))}; + for (u32 line = 0; line < num_lines; ++line) { + const EmitContext::MeshLineTarget& target{targets[line]}; + if (!Sirit::ValidId(target.variable)) { + continue; + } + const Id word{ctx.OpLoad( + ctx.U32[1], + SharedWord(ctx, ctx.mesh_attributes, ctx.OpIAdd(ctx.U32[1], first, ctx.Const(line * 32))))}; + const Id value{target.is_float ? ctx.OpBitcast(ctx.F32[1], word) : word}; + const Id pointer_type{target.is_float ? ctx.output_f32 : ctx.output_u32}; + const Id pointer{target.element == ~0U + ? ctx.OpAccessChain(pointer_type, target.variable, index) + : ctx.OpAccessChain(pointer_type, target.variable, index, + ctx.Const(target.element))}; + ctx.OpStore(pointer, value); + } +} + +// OpenGL-style clip space: z = (z + w) / 2, from the stage buffer's position lines. +void ConvertDepth(EmitContext& ctx, Id vertex) { + const auto& lines{ctx.mesh_info.vertex_lines}; + const auto z_line{std::ranges::find(lines, static_cast<u32>(IR::Attribute::PositionZ))}; + const auto w_line{std::ranges::find(lines, static_cast<u32>(IR::Attribute::PositionW))}; + if (z_line == lines.end() || w_line == lines.end()) { + return; + } + const u32 num_lines{static_cast<u32>(lines.size())}; + const Id group{ctx.OpShiftRightLogical(ctx.U32[1], vertex, ctx.Const(5U))}; + const Id slot{ctx.OpBitwiseAnd(ctx.U32[1], vertex, ctx.Const(31U))}; + const Id first{ctx.OpIAdd(ctx.U32[1], ctx.OpIMul(ctx.U32[1], group, ctx.Const(num_lines * 32)), + slot)}; + const auto read = [&](auto line) { + const u32 offset{static_cast<u32>(line - lines.begin()) * 32}; + return ctx.OpBitcast(ctx.F32[1], + ctx.OpLoad(ctx.U32[1], SharedWord(ctx, ctx.mesh_attributes, + ctx.OpIAdd(ctx.U32[1], first, + ctx.Const(offset))))); + }; + const Id z{ctx.OpFMul(ctx.F32[1], ctx.OpFAdd(ctx.F32[1], read(z_line), read(w_line)), + ctx.Const(0.5f))}; + ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, ctx.output_position, vertex, ctx.Const(2U)), z); +} + +void TaskEpilogue(EmitContext& ctx) { + Barrier(ctx); + const Id count{ctx.OpLoad(ctx.U32[1], PayloadWord(ctx, ctx.Const(1U)))}; + const Id one{ctx.Const(1U)}; + // Marker for OpEmitMeshTasksEXT count, 1, 1, payload (PatchMeshInstructions). + ctx.OpStore(count, one); + ctx.OpStore(one, ctx.mesh_payload); + ctx.mesh_tasks_marker = {count, one, one, ctx.mesh_payload}; +} + +void MeshEpilogue(EmitContext& ctx) { + const MeshInfo& mesh{ctx.mesh_info}; + Barrier(ctx); + const Id count{ctx.OpUMin(ctx.U32[1], ctx.OpLoad(ctx.U32[1], ctx.mesh_primitive_count), + ctx.Const(mesh.max_primitives))}; + const Id max_vertices{ctx.Const(mesh.max_vertices)}; + // Marker for OpSetMeshOutputsEXT max_vertices, count (PatchMeshInstructions). + ctx.OpStore(max_vertices, count); + ctx.mesh_outputs_marker = {max_vertices, count}; + + const Id invocation{LocalInvocation(ctx)}; + const u32 threads{ctx.mesh_threads}; + const bool convert_depth{ctx.runtime_info.convert_depth_mode && !ctx.profile.support_native_ndc && + Sirit::ValidId(ctx.output_position)}; + const u32 vertex_lines{static_cast<u32>(mesh.vertex_lines.size())}; + for (u32 first = 0; first < mesh.max_vertices; first += threads) { + const Id vertex{ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first))}; + const auto copy = [&] { + CopyLines(ctx, vertex, 0, vertex_lines, ctx.mesh_vertex_targets); + if (convert_depth) { + ConvertDepth(ctx, vertex); + } + }; + if (first + threads > mesh.max_vertices) { + If(ctx, ctx.OpULessThan(ctx.U1, vertex, max_vertices), copy); + } else { + copy(); + } + } + const u32 primitive_lines{static_cast<u32>(mesh.primitive_lines.size())}; + for (u32 first = 0; first < mesh.max_primitives; first += threads) { + const Id primitive{ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first))}; + If(ctx, ctx.OpULessThan(ctx.U1, primitive, count), [&] { + // Vertex indices: map bytes 4 + 3 * primitive. + const Id byte{ctx.OpIAdd(ctx.U32[1], ctx.OpIMul(ctx.U32[1], primitive, ctx.Const(3U)), + ctx.Const(4U))}; + std::array<Id, 3> indices; + for (u32 element = 0; element < 3; ++element) { + const Id address{ctx.OpIAdd(ctx.U32[1], byte, ctx.Const(element))}; + indices[element] = ctx.OpLoad(ctx.U32[1], SharedWord(ctx, ctx.mesh_map, address)); + } + const Id output_indices{ctx.TypePointer(spv::StorageClass::Output, ctx.U32[3])}; + ctx.OpStore(ctx.OpAccessChain(output_indices, ctx.mesh_primitive_indices, primitive), + ctx.OpCompositeConstruct(ctx.U32[3], std::span<const Id>(indices))); + CopyLines(ctx, primitive, ctx.mesh_primitive_base, primitive_lines, + ctx.mesh_primitive_targets); + }); + } +} +} // Anonymous namespace + +Id EmitIsbeRead(EmitContext& ctx, Id address) { + const Id workgroup{ + ctx.OpCompositeExtract(ctx.U32[1], ctx.OpLoad(ctx.U32[3], ctx.workgroup_id), 0U)}; + if (ctx.stage == Stage::Task) { + return workgroup; // the input region holds the workgroup index (VERTEX_ID) + } + const Id index{ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.Const(2U))}; + const Id word{ctx.OpLoad(ctx.U32[1], PayloadWord(ctx, index))}; + return ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, address, ctx.u32_zero_value), workgroup, + word); +} + +void EmitIsbeWriteAttribute(EmitContext& ctx, Id address, Id value) { + const Id index{ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.Const(2U))}; + if (ctx.stage == Stage::Task) { + ctx.OpStore(PayloadWord(ctx, index), value); + return; + } + ctx.OpStore(SharedWord(ctx, ctx.mesh_attributes, Clamped(ctx, index, ctx.mesh_attribute_words)), + value); +} + +void EmitIsbeWriteMapU8(EmitContext& ctx, Id address, Id value) { + if (ctx.stage != Stage::Mesh) { + throw NotImplementedException("Stage buffer map write outside mesh programs"); + } + ctx.OpStore(SharedWord(ctx, ctx.mesh_map, Clamped(ctx, address, ctx.mesh_map_bytes)), + ctx.OpBitwiseAnd(ctx.U32[1], value, ctx.Const(0xffU))); +} + +void EmitIsbeWriteMapU32(EmitContext& ctx, const IR::Value& address, Id value) { + // The primitive count: the map is written from its end, so its first word is at byte 3. + if (ctx.stage != Stage::Mesh || !address.IsImmediate() || address.U32() != 3) { + throw NotImplementedException("32-bit stage buffer map write"); + } + ctx.OpStore(ctx.mesh_primitive_count, value); +} + +void EmitMeshPrologue(EmitContext& ctx) { + if (ctx.stage != Stage::Mesh) { + return; + } + // Every invocation stores the zero: no branch here, the entry block's label is a phi + // predecessor for later blocks. + ctx.OpStore(ctx.mesh_primitive_count, ctx.u32_zero_value); + Barrier(ctx); +} + +void EmitMeshEpilogue(EmitContext& ctx) { + if (ctx.stage == Stage::Task) { + TaskEpilogue(ctx); + } else if (ctx.stage == Stage::Mesh) { + MeshEpilogue(ctx); + } +} + +} // namespace Shader::Backend::SPIRV diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp index 8eca1fac29..2b36c1fd8d 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp @@ -137,9 +137,16 @@ void EmitPrologue(EmitContext& ctx) { if (ctx.stage == Stage::VertexB || ctx.stage == Stage::Geometry) { SetFixedPipelinePointSize(ctx); } + if (ctx.stage == Stage::Task || ctx.stage == Stage::Mesh) { + EmitMeshPrologue(ctx); + } } void EmitEpilogue(EmitContext& ctx) { + if (ctx.stage == Stage::Task || ctx.stage == Stage::Mesh) { + EmitMeshEpilogue(ctx); + return; + } if (ctx.stage == Stage::VertexB && ctx.runtime_info.convert_depth_mode && !ctx.profile.support_native_ndc) { ConvertDepthMode(ctx); diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp index b562e82983..f80f0440df 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp @@ -257,6 +257,10 @@ std::string_view StageName(Stage stage) { return "fs"; case Stage::Compute: return "cs"; + case Stage::Task: + return "ts"; + case Stage::Mesh: + return "ms"; } throw InvalidArgument("Invalid stage {}", stage); } @@ -475,6 +479,7 @@ EmitContext::EmitContext(const Profile& profile_, const RuntimeInfo& runtime_inf DefineCommonTypes(program.info); DefineCommonConstants(); DefineInterfaces(program); + DefineMesh(program); DefineLocalMemory(program); DefineSharedMemory(program); DefineSharedMemoryFunctions(program); @@ -1586,6 +1591,16 @@ void EmitContext::DefineInputs(const IR::Program& program) { } const bool is_integer = input_type == AttributeType::SignedInt || input_type == AttributeType::UnsignedInt; + if (runtime_info.per_primitive_generics.test(index)) { + // Written per primitive by a mesh program: constant over the primitive. + AddExtension("SPV_EXT_mesh_shader"); + AddCapability(spv::Capability::MeshShadingEXT); + Decorate(id, spv::Decoration::PerPrimitiveEXT); + if (is_integer) { + Decorate(id, spv::Decoration::Flat); + } + continue; + } if (is_integer) { Decorate(id, spv::Decoration::Flat); } else { @@ -1614,8 +1629,120 @@ void EmitContext::DefineInputs(const IR::Program& program) { } } +void EmitContext::DefineMesh(const IR::Program& program) { + if (stage != Stage::Task && stage != Stage::Mesh) { + return; + } + AddExtension("SPV_EXT_mesh_shader"); + AddCapability(spv::Capability::MeshShadingEXT); + mesh_info = program.mesh; + mesh_threads = program.workgroup_size[0]; + // The task's output region, which its mesh workgroups read as their input region. + const Id payload_type{TypeArray(U32[1], Const(mesh_info.payload_words))}; + mesh_payload = DefineVariable(*this, payload_type, std::nullopt, + spv::StorageClass::TaskPayloadWorkgroupEXT); + Name(mesh_payload, "task_payload"); + mesh_payload_u32 = TypePointer(spv::StorageClass::TaskPayloadWorkgroupEXT, U32[1]); + if (stage == Stage::Task) { + return; + } + // Output attribute region: groups of 32 vertices, then groups of 32 primitives, one 32-word + // line per attribute. The map region keeps one map byte per word. + const u32 vertex_lines{static_cast<u32>(mesh_info.vertex_lines.size())}; + const u32 primitive_lines{static_cast<u32>(mesh_info.primitive_lines.size())}; + mesh_primitive_base = Common::DivCeil(mesh_info.max_vertices, 32U) * vertex_lines * 32; + mesh_attribute_words = (std::max)( + mesh_primitive_base + Common::DivCeil(mesh_info.max_primitives, 32U) * primitive_lines * 32, + 1U); + mesh_map_bytes = 4 + 3 * mesh_info.max_primitives; + mesh_shared_u32 = TypePointer(spv::StorageClass::Workgroup, U32[1]); + mesh_attributes = DefineVariable(*this, TypeArray(U32[1], Const(mesh_attribute_words)), + std::nullopt, spv::StorageClass::Workgroup); + Name(mesh_attributes, "isbe_attributes"); + mesh_map = DefineVariable(*this, TypeArray(U32[1], Const(mesh_map_bytes)), std::nullopt, + spv::StorageClass::Workgroup); + Name(mesh_map, "isbe_map"); + mesh_primitive_count = + DefineVariable(*this, U32[1], std::nullopt, spv::StorageClass::Workgroup); + Name(mesh_primitive_count, "isbe_primitive_count"); + + const Id vertex_count{Const(mesh_info.max_vertices)}; + const Id primitive_count{Const(mesh_info.max_primitives)}; + mesh_primitive_indices = + DefineVariable(*this, TypeArray(U32[3], primitive_count), + spv::BuiltIn::PrimitiveTriangleIndicesEXT, spv::StorageClass::Output); + std::array<Id, IR::NUM_GENERICS> generics[2]{}; + Id point_size{}, primitive_layer{}, primitive_viewport{}, primitive_id_output{}; + const auto per_primitive_builtin = [&](Id& variable, spv::BuiltIn builtin) { + if (!Sirit::ValidId(variable)) { + variable = DefineVariable(*this, TypeArray(U32[1], primitive_count), builtin, + spv::StorageClass::Output); + Decorate(variable, spv::Decoration::PerPrimitiveEXT); + } + return MeshLineTarget{variable, ~0U, false}; + }; + const auto target = [&](u32 line, bool per_primitive) -> MeshLineTarget { + const IR::Attribute attribute{line}; + if (IR::IsGeneric(attribute)) { + const u32 index{IR::GenericAttributeIndex(attribute)}; + Id& variable{generics[per_primitive][index]}; + if (!Sirit::ValidId(variable)) { + variable = DefineVariable( + *this, TypeArray(F32[4], per_primitive ? primitive_count : vertex_count), + std::nullopt, spv::StorageClass::Output); + Decorate(variable, spv::Decoration::Location, index); + if (per_primitive) { + Decorate(variable, spv::Decoration::PerPrimitiveEXT); + } + } + return {variable, IR::GenericAttributeElement(attribute), true}; + } + if (per_primitive) { + switch (attribute) { + case IR::Attribute::PrimitiveId: + return per_primitive_builtin(primitive_id_output, spv::BuiltIn::PrimitiveId); + case IR::Attribute::Layer: + return per_primitive_builtin(primitive_layer, spv::BuiltIn::Layer); + case IR::Attribute::ViewportIndex: + return per_primitive_builtin(primitive_viewport, spv::BuiltIn::ViewportIndex); + default: + return {}; + } + } + switch (attribute) { + case IR::Attribute::PositionX: + case IR::Attribute::PositionY: + case IR::Attribute::PositionZ: + case IR::Attribute::PositionW: + if (!Sirit::ValidId(output_position)) { + output_position = DefineVariable(*this, TypeArray(F32[4], vertex_count), + spv::BuiltIn::Position, spv::StorageClass::Output); + } + return {output_position, + static_cast<u32>(attribute) - static_cast<u32>(IR::Attribute::PositionX), true}; + case IR::Attribute::PointSize: + if (!Sirit::ValidId(point_size)) { + point_size = DefineVariable(*this, TypeArray(F32[1], vertex_count), + spv::BuiltIn::PointSize, spv::StorageClass::Output); + } + return {point_size, ~0U, true}; + default: + return {}; // per-primitive values in vertex lines have no vertex output + } + }; + for (const u32 line : mesh_info.vertex_lines) { + mesh_vertex_targets.push_back(target(line, false)); + } + for (const u32 line : mesh_info.primitive_lines) { + mesh_primitive_targets.push_back(target(line, true)); + } +} + void EmitContext::DefineOutputs(const IR::Program& program) { const Info& info{program.info}; + if (stage == Stage::Task || stage == Stage::Mesh) { + return; // DefineMesh + } const std::optional<u32> invocations{program.invocations}; if (runtime_info.convert_depth_mode || info.stores.AnyComponent(IR::Attribute::PositionX) || stage == Stage::VertexB) { diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.h b/src/shader_recompiler/backend/spirv/spirv_emit_context.h index a5821e9e93..1f9b534aaf 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.h +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.h @@ -363,6 +363,32 @@ public: std::vector<Id> interfaces; std::vector<std::pair<u32, spv::Op>> opcode_rewrites; + // Task and mesh programs. Their stage buffer: the task's output region is the task payload + // (the mesh stage's input region); the mesh output regions live in workgroup memory until + // the epilogue copies them to the outputs. The two mesh instructions Sirit lacks are emitted + // as OpStore markers with these operands and replaced after assembly (PatchMeshInstructions). + struct MeshLineTarget { + Id variable; ///< arrayed output + u32 element; ///< vector element, or ~0 for a scalar output + bool is_float; ///< bitcast the stage buffer word to F32 + }; + Id mesh_payload{}; + Id mesh_payload_u32{}; ///< pointer type to one payload word + Id mesh_attributes{}; + Id mesh_map{}; + Id mesh_primitive_count{}; + Id mesh_shared_u32{}; ///< pointer type to one workgroup word + Id mesh_primitive_indices{}; + MeshInfo mesh_info{}; + u32 mesh_threads{}; + u32 mesh_attribute_words{}; + u32 mesh_map_bytes{}; + u32 mesh_primitive_base{}; ///< first word of the per-primitive groups + std::vector<MeshLineTarget> mesh_vertex_targets; + std::vector<MeshLineTarget> mesh_primitive_targets; + std::array<Id, 2> mesh_outputs_marker{}; + std::array<Id, 4> mesh_tasks_marker{}; + Id load_const_func_u8{}; Id load_const_func_u16{}; Id load_const_func_u32{}; @@ -402,6 +428,7 @@ private: void DefineInputs(const IR::Program& program); void DefineOutputs(const IR::Program& program); + void DefineMesh(const IR::Program& program); }; } // namespace Shader::Backend::SPIRV diff --git a/src/shader_recompiler/environment.h b/src/shader_recompiler/environment.h index 67486430c8..20364944cf 100644 --- a/src/shader_recompiler/environment.h +++ b/src/shader_recompiler/environment.h @@ -66,6 +66,7 @@ struct ConstBufferAddr { IR::U32 dynamic_offset; u32 count; bool has_secondary; + u32 handle_program{~0U}; ///< index in Info::handle_programs for a handle computed per draw }; class Environment { @@ -121,9 +122,22 @@ public: return is_proprietary_driver; } + /// Task and mesh programs: SET_MESH_SHADER_A, SET_MESH_SHADER_B and SET_MESH_INIT_SHADER. + [[nodiscard]] const std::array<u32, 3>& MeshRegisters() const noexcept { + return mesh_registers; + } + + /// Mesh programs: the header of the pass-through geometry program bound with them, whose + /// output map lists the per-primitive outputs. + [[nodiscard]] const ProgramHeader& MeshPrimitiveHeader() const noexcept { + return mesh_primitive_header; + } + protected: ProgramHeader sph{}; std::array<u32, 8> gp_passthrough_mask{}; + std::array<u32, 3> mesh_registers{}; + ProgramHeader mesh_primitive_header{}; Stage stage{}; u32 start_address{}; bool is_proprietary_driver{}; diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h index 489d003e60..024ed27ea3 100644 --- a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h +++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h @@ -34,6 +34,9 @@ struct LoweredProgram { std::vector<u32> source_instruction_offsets; u32 source_size{}; u32 required_texture_buffer = ~0U; + // UR28 and up seen without a mirror: lowering again with this many registers below R216 + // reserved for them (even, so pairs stay adjacent) maps them. + u32 high_uniform_registers{}; u32 error_offset{}; std::string_view error; @@ -43,8 +46,10 @@ struct LoweredProgram { // Lower only operations whose semantics are representable by the existing // Maxwell frontend. These are new instructions, not reinterpretations of the // native bytes. Scheduling/control words are added by the environment adapter. -inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { +inline LoweredProgram LowerToMaxwellWith(std::span<const Instruction> code, u32 high_uniforms) { LoweredProgram result; + // UR28 and up (when used) live just below R216: UR28 at R(216 - high_uniforms). + const u32 high_uniform_base = 216 - high_uniforms; struct Relocation { u32 output_index; s64 target; }; std::vector<Relocation> relocations; std::vector<u32> barrier_tokens; @@ -55,8 +60,8 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { bool uses_uniform_bank = false; for (const auto& instruction : code) { const u32 op = instruction.Bits(0, 12); - uses_uniform_bank |= op == 0x3c4 || op == 0x2bd || op == 0xc0c || op == 0x986 || op == 0x9a8 || op == 0xc12 || op == 0xc17 || op == 0xe24 || op == 0xc24 || op == 0xd00 || op == 0xd09 || op == 0x9c3 || op == 0xc45 || op == 0x2a4 || op == 0x891 || op == 0x890 || op == 0x290 || op == 0x28c || op == 0x88c || op == 0x287 || op == 0x887 || op == 0x958 || op == 0xab9 || op == 0xabb || op == 0x899 || op == 0xc20 || op == 0xe21 || op == 0xe23 || op == 0xc02 || - op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || op == 0x292 || op == 0x892 || op == 0x291 || op == 0x981 || op == 0xc13 || op == 0xc10 || op == 0xc23 || op == 0x526 || op == 0x2be || op == 0x8a4 || op == 0xd05 || (op == 0x984 && instruction.Bits(91, 1)) || op == 0x89c || op == 0x883 || op == 0x2a5 || op == 0xd06 || op == 0x299 || op == 0xc09 || (op == 0x81c && instruction.Bits(67, 1)) || op == 0xf60 || op == 0xc11 || op == 0xc0a; + uses_uniform_bank |= op == 0x3c4 || op == 0x2bd || op == 0x896 || op == 0xf60 || op == 0xc11 || op == 0xd08 || op == 0xc0c || op == 0x986 || op == 0x9a8 || op == 0xc12 || op == 0xc17 || op == 0xe24 || op == 0xc24 || op == 0xd00 || op == 0xd09 || op == 0x9c3 || op == 0xc45 || op == 0x2a4 || op == 0x891 || op == 0x890 || op == 0x290 || op == 0x28c || op == 0x88c || op == 0x287 || op == 0x887 || op == 0x958 || op == 0xab9 || op == 0xabb || op == 0x899 || op == 0xc20 || op == 0xe21 || op == 0xe23 || op == 0xc02 || + op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || op == 0x292 || op == 0x892 || op == 0x291 || op == 0x981 || op == 0xc13 || op == 0xc10 || op == 0xc23 || op == 0x526 || op == 0x2be || op == 0x8a4 || op == 0xd05 || (op == 0x984 && instruction.Bits(91, 1)) || op == 0x89c || op == 0x883 || op == 0x2a5 || op == 0xd06 || op == 0xf63 || op == 0xf69 || op == 0xc0b || op == 0x8a5 || op == 0x299 || op == 0xc09 || (op == 0x81c && instruction.Bits(67, 1)) || op == 0xc0a; } const auto address_of = [](u32 index) { return 0x50U + index * 8 + ((index + 1) / 3) * 8; }; const auto fail = [&](u32 index, std::string_view reason) { @@ -104,25 +109,26 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { // Scratch values and emulated uniform state occupy reserved high GPRs. // Reject native accesses that could alias that state. for (u32 bit : {16U, 24U, 32U, 64U}) { - if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66 && opcode != 0x361 && - opcode != 0xf60) continue; + if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66) continue; const u32 reg = insn.Bits(bit, 8); - if (reg >= (uses_uniform_bank ? 216U : 246U) && reg < 255) return fail(index, "native program uses lowering scratch registers"); + if (reg >= (uses_uniform_bank ? high_uniform_base : 246U) && reg < 255) return fail(index, "native program uses lowering scratch registers"); } // UR0-UR21 live in R216-R237 and UR22-UR27 in R240-R245 (R239 holds the uniform - // predicates), so even/odd pairs stay adjacent. Sites that still check reg < 16 rely on - // UR0-UR15 being R216-R231. + // predicates), so even/odd pairs stay adjacent; UR28 and up below R216. Sites that + // still check reg < 16 rely on UR0-UR15 being R216-R231. const auto uniform_register = [&](u32 reg) -> u32 { if (reg < 22) return 216 + reg; if (reg < 28) return 218 + reg; if (reg == 63) return 255; + if (reg < 63 && reg - 28 < high_uniforms) return high_uniform_base + (reg - 28); + if (reg < 63) result.high_uniform_registers = (std::max)(result.high_uniform_registers, (reg - 26) & ~1U); return ~0U; }; bool rewritten_uniform = false; bool uniform_guard = false; u32 uniform_output = 7; // ULOP3's UPn output: computed in P(6-n), then merged into R239 if (opcode == 0x2a4 || opcode == 0x8a4 || opcode == 0x891 || opcode == 0x890 || opcode == 0x290 || - opcode == 0x292 || opcode == 0x892 || opcode == 0x291) { + opcode == 0x292 || opcode == 0x892 || opcode == 0x291 || opcode == 0x896 || opcode == 0x8a5) { if (insn.Bits(12, 4) != 7 && opcode != 0x292 && opcode != 0x892) return fail(index, "native uniform arithmetic predication"); uniform_guard = insn.Bits(12, 4) != 7; @@ -144,7 +150,8 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { insn.words[2] = (insn.words[2] & ~(7U << 17)) | ((6 - uniform_output) << 17); } opcode = opcode == 0x2a4 ? 0x224 : opcode == 0x8a4 ? 0x824 : opcode == 0x891 ? 0x811 : opcode == 0x890 ? 0x810 - : opcode == 0x292 ? 0x212 : opcode == 0x892 ? 0x812 : opcode == 0x291 ? 0x211 : 0x210; + : opcode == 0x292 ? 0x212 : opcode == 0x892 ? 0x812 : opcode == 0x291 ? 0x211 + : opcode == 0x896 ? 0x816 : opcode == 0x8a5 ? 0x825 : 0x210; } const bool uniform_predication = opcode == 0xabb || opcode == 0xab9 || opcode == 0xc82 || opcode == 0x882; const u32 native_predicate = insn.Bits(12, 4); @@ -727,8 +734,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0x5c30000000070000ULL | 250 | (u64{250} << 20)); emit(0xef10000000070000ULL | 251 | (u64{insn.Bits(24, 8)} << 8) | (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); - emit(0x5ca0000000070000ULL | target | (u64{251} << 8) | - (u64{target} << 20) | (u64{6} << 39)); + if (native_predicate == 7) { + // Every lane executing it is in the ballot, so no lane-less fallback can be taken. + // Leaving the old value selectable would make it a source of the result for the + // texture and global memory tracking (a handle traced to the uniform's last load). + emit(0x5c98000000070000ULL | target | (u64{15} << 39) | (u64{251} << 20)); + } else { + emit(0x5ca0000000070000ULL | target | (u64{251} << 8) | + (u64{target} << 20) | (u64{6} << 39)); + } restore_predicate(); break; } @@ -821,36 +835,42 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } case 0x899: { // USHF.L.U32; accept a zero word outside the selected half. const u32 dst = insn.Bits(16, 8), low = insn.Bits(24, 8), high = insn.Bits(64, 8); - if (insn.words[2] == (0x08011400U | high) && low == 63 && high < 28 && - dst < 28 && native_predicate == 7 && insn.words[1] <= 32) { - emit(0x3828000000000000ULL | predicate | uniform_register(dst) | - (u64{uniform_register(high)} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48)); + const u32 dst_reg = uniform_register(dst), low_reg = uniform_register(low); + const u32 high_reg = uniform_register(high); + const bool dst_ok = dst != 63 && dst_reg != ~0U; + const bool low_ok = low != 63 && low_reg != ~0U, high_ok = high != 63 && high_reg != ~0U; + if (insn.words[2] == (0x08011400U | high) && low == 63 && high_ok && + dst_ok && native_predicate == 7 && insn.words[1] <= 32) { + emit(0x3828000000000000ULL | predicate | dst_reg | + (u64{high_reg} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48)); break; } // Logical right shifts of one register (bit 76 right, bit 80 high word, bits 73-74 // U32): USHF.R.U32 URd, URa, n, URZ and USHF.R.U32.HI URd, URZ, n, URb. - const bool right_low = insn.words[2] == 0x0800163f && low < 28; - const bool right_high = insn.words[2] == (0x08011600U | high) && low == 63 && high < 28; - if ((right_low || right_high) && dst < 28 && native_predicate == 7 && insn.words[1] <= 32) { - emit(0x3828000000000000ULL | predicate | uniform_register(dst) | - (u64{uniform_register(right_high ? high : low)} << 8) | (u64{insn.words[1]} << 20)); + const bool right_low = insn.words[2] == 0x0800163f && low_ok; + const bool right_high = insn.words[2] == (0x08011600U | high) && low == 63 && high_ok; + if ((right_low || right_high) && dst_ok && native_predicate == 7 && insn.words[1] <= 32) { + emit(0x3828000000000000ULL | predicate | dst_reg | + (u64{right_high ? high_reg : low_reg} << 8) | (u64{insn.words[1]} << 20)); break; } - const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high < 28; - const bool low_result = insn.words[2] == 0x0800063f && low < 28; - if (dst >= 28 || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result)) + const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high_ok; + const bool low_result = insn.words[2] == 0x0800063f && low_ok; + if (!dst_ok || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result)) return fail(index, "native uniform shift mode"); - emit(0x3848000000000000ULL | predicate | uniform_register(dst) | - (u64{uniform_register(high_result ? high : low)} << 8) | (u64{insn.words[1]} << 20)); + emit(0x3848000000000000ULL | predicate | dst_reg | + (u64{high_result ? high_reg : low_reg} << 8) | (u64{insn.words[1]} << 20)); break; } - case 0xb82: { // LDC.64 indexed byte offset - const u32 width = insn.Bits(73, 3); - if ((width != 4 && width != 5) || insn.Bits(78, 2) || insn.Bits(91, 1) || + case 0xb82: { // LDC.64 indexed byte offset; bits 78-79 the mode as Maxwell's (2 = IS) + // .IS: Ra + offset addresses all constant buffers as one space, the buffer in its high + // half (Kirby Air Riders' vertex shaders), which Eden's Maxwell LDC implements. + const u32 width = insn.Bits(73, 3), mode = insn.Bits(78, 2); + if ((width != 4 && width != 5) || (mode != 0 && mode != 2) || insn.Bits(91, 1) || bank() >= 18 || (width == 5 && ((dest & 1) || dest >= 254))) return fail(index, "native indexed constant load mode"); emit(base(0xef90000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | - (u64{insn.Bits(38, 16)} << 20) | (u64{bank()} << 36) | (u64{width} << 48)); + (u64{insn.Bits(38, 16)} << 20) | (u64{bank()} << 36) | (u64{mode} << 44) | (u64{width} << 48)); break; } case 0xf89: { // SHFL with immediate lane and segmentation/clamp mask, any mode. @@ -880,16 +900,17 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { const u32 dimension = insn.Bits(61, 3); const u32 high = insn.Bits(64, 8); // destination of the third and fourth channels const u32 mask = insn.Bits(72, 4), channels = static_cast<u32>(std::popcount(mask)); + // Bit 90 is forwarded below (Maxwell's NODEP); Splatoon Raiders sets it. if ((insn.words[2] & ~(0xfffU | (1U << 26))) != 0x001e0000 || mask == 0 || insn.Bits(59, 2) || (dimension != 1 && dimension != 5) || binding >= 18 || offset > 0x1fff || (channels > 1 && (dest & 1)) || dest >= 248 || - (channels > 2 ? (high & 1) || high >= 248 : high != 255) || + (channels > 2 ? high != 255 && ((high & 1) || high >= 248) : high != 255) || coordinates >= 248 || derivatives >= 247) return fail(index, "native gradient texture mode"); if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) return fail(index, "native texture descriptor buffer conflict"); result.required_texture_buffer = binding; - // Maxwell writes four consecutive registers: when the second pair is elsewhere (Splatoon + // Maxwell writes consecutive registers: when the second pair is elsewhere (Splatoon // Raiders: R12/R18, R18/R44), write them to scratch and move each pair into place. const bool consecutive = channels <= 2 || high == dest + 2; const u64 target = consecutive ? dest : 248; @@ -898,6 +919,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (u64{offset} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51)); if (!consecutive) { for (u32 i = 0; i < channels; ++i) { + if (i >= 2 && high == 255) break; emit(0x5c98000000000000ULL | predicate | (i < 2 ? dest + i : high + i - 2) | (u64{15} << 39) | (u64{248 + i} << 20)); } @@ -950,35 +972,55 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0x3828000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 8) | (u64{1} << 39) | (u64{1} << 40)); break; - case 0x3a1: { // MATCH.ALL.U32/U64 (bit 73) with only the all-equal predicate retained. + case 0x2bf: // UPOPC, as POPC of the uniform's register. + if (dest >= 16 || insn.words[1] >= 16 || insn.words[2] != 0x08000000 || native_predicate != 7) + return fail(index, "native uniform population count mode"); + emit(0x5c08000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 20)); + break; + case 0x3a1: { // MATCH.ALL (.U32, or .U64 on bit 73) with only the all-equal predicate retained. const u32 source = insn.Bits(24, 8), target = insn.Bits(81, 3); - const bool wide = insn.Bits(73, 1); - if (dest != 255 || source >= 254 || (wide && (source & 1)) || target >= 6 || - native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0200U)) + const u32 words = insn.Bits(73, 1) ? 2 : 1; + const u32 scratch = target == 6 ? 5U : 6U; // the comparison, saved around the match + if (dest != 255 || source >= 254 || (words == 2 && (source & 1)) || target == 7 || + native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0200U) != 0) return fail(index, "native warp match mode"); - save_predicate(); + save_predicate(1U << scratch); + // Compare every lane with the lowest active one, then require all to agree. emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49)); emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48)); emit(0x5c30000000070000ULL | 250 | (u64{250} << 20)); - for (u32 word = 0; word < (wide ? 2U : 1U); ++word) + for (u32 word = 0; word < words; ++word) emit(0xef10000000070000ULL | (251 + word) | (u64{source + word} << 8) | (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); - emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source} << 8) | + emit(0x5b60000000070000ULL | 7 | (u64{scratch} << 3) | (u64{source} << 8) | (u64{251} << 20) | (u64{7} << 39) | (u64{2} << 49)); - if (wide) - emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source + 1} << 8) | - (u64{252} << 20) | (u64{6} << 39) | (u64{2} << 49)); - emit(0x50d8000000070000ULL | 255 | (u64{6} << 39) | (u64{target} << 45)); - restore_predicate(); - break; - } - case 0x3c4: { // REDUX.SUM (modulo 2^32), .OR, .MIN or .MAX (unsigned) over the actual active lanes. + if (words == 2) + emit(0x5b60000000070000ULL | 7 | (u64{scratch} << 3) | (u64{source + 1} << 8) | + (u64{252} << 20) | (u64{scratch} << 39) | (u64{2} << 49)); + emit(0x50d8000000070000ULL | 255 | (u64{scratch} << 39) | (u64{target} << 45)); + restore_predicate(1U << scratch); + break; + } + case 0x80d: { // CSMTEST: a test of shading-mode state Eden does not model, taken as false. + // Kirby Air Riders' vertex-side shaders run "CSMTEST.A.FCSM_TR P0" then "@P0 EXIT": + // never leave early. Outputs of PT discard the result. + const u32 first = insn.Bits(81, 3), second = insn.Bits(84, 3); + if (insn.Bits(87, 4) != 7 || insn.Bits(16, 16)) + return fail(index, "native CSM test mode"); + for (const u32 target : {first, second}) + if (target != 7) + emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 | (u64{7} << 12) | + (u64{1} << 15) | (u64{7} << 29) | (u64{7} << 39)); + break; + } + case 0x3c4: { // REDUX.SUM (modulo 2^32), .MIN/.MAX (unsigned), .OR or .XOR over the actual active lanes. const u32 source = insn.Bits(24, 8), target = uniform_register(static_cast<u32>(dest)); const bool maximum = insn.words[2] == 0x14000, minimum = insn.words[2] == 0x10000; - const bool bitwise_or = insn.words[2] == 0x4000; + // OR and XOR: inactive lanes contribute zero, as for the sum. Maxwell LOP: 1 = OR, 2 = XOR. + const u32 logical = insn.words[2] == 0x4000 ? 1U : insn.words[2] == 0x8000 ? 2U : 0U; if (target == ~0U || target == 255 || insn.words[1] || - (insn.words[2] != 0xc000 && !maximum && !minimum && !bitwise_or) || native_predicate != 7) + (insn.words[2] != 0xc000 && !maximum && !minimum && !logical) || native_predicate != 7) return fail(index, "native warp sum mode"); emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); mov_imm(target, minimum ? ~0U : 0U); @@ -988,15 +1030,16 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0x3828000000070000ULL | 252 | (u64{250} << 8) | (u64{lane} << 20)); emit(0x0400000000070000ULL | 252 | (u64{252} << 8) | (u64{1} << 20)); emit(0x5c10000000070000ULL | 252 | (u64{255} << 8) | (u64{252} << 20) | (u64{1} << 48)); - // Inactive lanes give the identity: zero, or all ones for MIN (OR with ~mask). - emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | - (minimum ? (u64{1} << 40) | (u64{1} << 41) : 0) | (u64{7} << 48)); - if (maximum || minimum) { - emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | - (u64{maximum ? 15U : 7U} << 39)); - } else if (bitwise_or) { + // Inactive lanes: 0, or ~0 for the minimum (OR with the inverted lane mask). + emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48) | + (u64{minimum} << 40) | (u64{minimum} << 41)); + if (minimum) { + emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | (u64{7} << 39)); + } else if (maximum) { + emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | (u64{15} << 39)); + } else if (logical) { emit(0x5c40000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | - (u64{1} << 41) | (u64{7} << 48)); + (u64{logical} << 41) | (u64{7} << 48)); } else { emit(0x5c10000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20)); } @@ -1004,8 +1047,9 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { break; } case 0xf8c: { // ATOMS.POPC.INC.32 with no returned value: one increment per active lane. - const u32 stride = insn.Bits(78, 2); - if (dest != 255 || insn.words[1] || (insn.words[2] & ~0xc000U) != 0x0d00003f) + const u32 stride = insn.Bits(78, 2), offset = insn.Bits(40, 24); + if (dest != 255 || (insn.words[1] & 0xff) || (offset & 3) || + (insn.words[2] & ~0xc000U) != 0x0d00003f) return fail(index, "native shared increment mode"); u32 address = insn.Bits(24, 8); if (stride && address != 255) { @@ -1014,18 +1058,18 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { address = 251; } mov_imm(250, 1); - emit(0xec00000000000000ULL | predicate | 255 | (u64{address} << 8) | (u64{250} << 20)); + emit(0xec00000000000000ULL | predicate | 255 | (u64{address} << 8) | (u64{250} << 20) | + (u64{offset >> 2} << 30)); break; } - case 0x38c: { // ATOMS.ADD/MIN/MAX/OR.U32, naturally aligned shared byte offset. - // The operation (bits 87-90) numbers ADD 0, MIN 1, MAX 2, OR 6 as Maxwell's (bits 52-55). + case 0x38c: { // ATOMS (ADD/MIN/MAX/INC/DEC/AND/OR/XOR/EXCH, U32 or S32), aligned byte offset. + // The operation (bits 87-90) is numbered as in Maxwell ATOMS (52-55); bit 73 is S32. const u32 offset = insn.Bits(40, 24), operation = insn.Bits(87, 4); - if ((operation != 0 && operation != 1 && operation != 2 && operation != 6) || - insn.words[2] != (operation << 23) || (offset & 3)) + if ((insn.words[2] & ~0x07800200U) || operation > 8 || (offset & 3)) return fail(index, "native shared atomic mode"); emit(0xec00000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) | - (u64{offset >> 2} << 30) | (u64{operation} << 52)); + (u64{insn.Bits(73, 1)} << 28) | (u64{offset >> 2} << 30) | (u64{operation} << 52)); break; } case 0x3a8: { // ATOMG.E.ADD.STRONG.GPU U32 at a 64-bit register address plus an offset. @@ -1050,14 +1094,14 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (u64{1} << 48)); break; } - case 0xb97: { // SULD.P.2D/2D_ARRAY.IGN, direct descriptor, channel mask at 72-75. + case 0xb97: { // SULD.P.2D/2D_ARRAY/3D.IGN, direct descriptor, channel mask at 72-75. const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); const u32 coordinate = insn.Bits(24, 8); const u32 channels = insn.Bits(72, 4); - const u32 dimension = insn.Bits(61, 3); // Maxwell's codes: 3 = 2D, 4 = 2D_ARRAY - if ((insn.words[2] & ~0xf00U) != 0x001e0000 || channels == 0 || (dimension != 3 && dimension != 4) || + const u32 dimension = insn.Bits(61, 3); // Maxwell's codes: 3 = 2D, 4 = 2D_ARRAY, 5 = 3D + if ((insn.words[2] & ~0xf00U) != 0x001e0000 || channels == 0 || dimension < 3 || dimension > 5 || binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || dest == 255 || - coordinate > (dimension == 4 ? 252U : 253U)) + coordinate > (dimension >= 4 ? 252U : 253U)) return fail(index, "native formatted surface load mode"); if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) return fail(index, "native surface descriptor buffer conflict"); @@ -1086,6 +1130,21 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { if (strong) emit(0xef98000000000000ULL | predicate | (u64{1} << 8)); break; } + case 0x59f: { // SURED.D.2D.ADD.STRONG.GPU.IGN U32, direct descriptor (title 1's race compute). + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const u32 data = insn.Bits(32, 8), coordinate = insn.Bits(24, 8); + if (insn.words[2] != 0x0010e000 || insn.Bits(61, 3) != 3 || binding >= 18 || + offset > 0x1fff || insn.Bits(59, 2) || coordinate > 253) + return fail(index, "native surface reduction mode"); + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native surface descriptor buffer conflict"); + result.required_texture_buffer = binding; + // Maxwell SURED (as the frontend decodes it): operand 0-7, coordinates 8-15, ADD and + // U32 are zero at 24 and 20, type 33 (3 = 2D), bound offset 36, bound 51. + emit(0xeb40000000000000ULL | predicate | data | (u64{coordinate} << 8) | + (u64{3} << 33) | (u64{offset} << 36) | (u64{1} << 51)); + break; + } case 0x301: // BREV register. if (insn.words[2] || insn.words[1] > 255) return fail(index, "native bit reverse mode"); emit(base(0x3828000000000000ULL) | (u64{insn.words[1]} << 8) | @@ -1326,10 +1385,21 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0x919: case 0x9c3: { // S2R / S2UR thread and workgroup identities. + case 0x919: case 0x9c3: { // S2R / S2UR: special registers the Maxwell S2R handles. + // Same numbering as Maxwell: lane id (0), invocation id/info, Y direction, thread kill + // (17-19, 29), W scale factors (30-31, Kirby Air Riders), thread and workgroup ids, + // lane masks (56-60). const u32 special = insn.Bits(72, 8); + if (special == 132 && !insn.words[1] && insn.words[2] == (special << 8)) { + // SR_VARIABLE_RATE: without variable-rate shading every fragment is 1x1, its width + // and height in bytes 1 and 2 (Nintendo Switch Sports Resort multiplies them). + const u32 target = opcode == 0x9c3 ? uniform_register(static_cast<u32>(dest)) : static_cast<u32>(dest); + if (target == ~0U) return fail(index, "native uniform special destination"); + mov_imm(target, 0x00010100); + break; + } if (insn.words[1] || insn.words[2] != (special << 8) || - (special != 0 && special != 33 && special != 34 && special != 35 && + (special != 0 && (special < 17 || special > 19) && (special < 29 || special > 35) && special != 37 && special != 38 && special != 39 && special != 56 && special != 57 && special != 58 && special != 59 && special != 60)) return fail(index, "native special register"); @@ -1361,10 +1431,25 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { if (condition > 6 || target == ~0U || a == ~0U || b == ~0U || insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x07800000U) != 0x08000000U) return fail(index, "native uniform select mode"); - if (opcode == 0x887 && index > 0 && condition == 0 && insn.words[1] <= 0x7ffff && - code[index - 1].Bits(0, 12) == 0x88c && - code[index - 1].Bits(24, 8) == insn.Bits(24, 8) && - code[index - 1].words[1] == insn.words[1] && code[index - 1].words[2] == 0x0bf01070 && + // The clamp's UISETP may sit a few instructions back, with only instructions between + // that write no uniform state and do not branch (Nintendo Switch Sports' jump tables), + // and no branch seen so far landing between them. + u32 compare = index; + for (u32 back = 1; back <= 8 && back <= index; ++back) { + const u32 earlier = code[index - back].Bits(0, 12), datapath = earlier & 0x1ff; + if (earlier == 0x88c) { + compare = index - back; + break; + } + if ((datapath >= 0x080 && datapath <= 0x0bf) || (datapath >= 0x1c2 && datapath <= 0x1c4) || + (earlier >= 0x940 && earlier <= 0x95f)) + break; + } + const bool target_between = std::any_of(relocations.begin(), relocations.end(), + [&](const Relocation& relocation) { return relocation.target > compare && relocation.target <= index; }); + if (opcode == 0x887 && compare < index && !target_between && condition == 0 && insn.words[1] <= 0x7ffff && + code[compare].Bits(12, 4) == 7 && code[compare].Bits(24, 8) == insn.Bits(24, 8) && + code[compare].words[1] == insn.words[1] && code[compare].words[2] == 0x0bf01070 && insn.words[2] == 0x08000000) { // Preserve the unsigned clamp idiom for indirect-table analysis. emit(0x3820000000070000ULL | target | (u64{a} << 8) | @@ -1530,30 +1615,42 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0xb6f: { // TXQ texture dimensions (mask 1-3) or mip level count (mask 8). - const u32 mask = insn.Bits(72, 4); - if (insn.Bits(59, 5) || insn.Bits(64, 8) != 255 || (mask == 0 || (mask > 3 && mask != 8)) || - insn.Bits(76, 14) || insn.Bits(91, 1) || insn.Bits(54, 5) >= 18 || - insn.Bits(40, 14) > 0x1fff || (mask == 3 && (dest & 1)) || dest >= 254) + case 0xb6f: case 0x370: { // TXQ texture dimensions (0x370: TXQ.B). + // TXQ.B takes the handle in Ra and the level in Ra+1, as Maxwell's. Up to two outputs + // go to Rd, Rd+1 directly; more, or a second destination (bits 64-71) for the third and + // fourth, come out packed in scratch and are copied as for TEX. + const bool bindless = opcode == 0x370; + const u32 binding = insn.Bits(54, 5); // descriptor buffer, as for TLD + const u32 mask = insn.Bits(72, 4), components = static_cast<u32>(std::popcount(mask)); + const u32 second_dest = insn.Bits(64, 8); + const bool direct = second_dest == 255; + if (insn.Bits(59, 5) != (bindless ? 1U : 0U) || mask == 0 || (direct && components > 2) || + insn.Bits(76, 14) || insn.Bits(91, 1) || (!bindless && binding >= 18) || + (bindless && (insn.Bits(32, 27) || insn.Bits(24, 8) >= 245)) || + (!direct && (second_dest >= 245 || dest >= 245)) || + insn.Bits(40, 14) > 0x1fff || (direct && components == 2 && (dest & 1)) || dest >= 254) return fail(index, "native texture dimension query mode"); - if (!bind_texture_buffer(insn.Bits(54, 5))) - return fail(index, "native texture descriptor buffer conflict"); - emit(0xdf48000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | - (u64{1} << 22) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | - (u64{insn.Bits(90, 1)} << 49)); - break; - } - case 0x370: { // TXQ.B dimensions or mip count: handle in Ra, mip level in Ra+1. - const u32 mask = insn.Bits(72, 4), source = insn.Bits(24, 8); - if (insn.words[1] != 0x08000000 || insn.Bits(64, 8) != 255 || (mask == 0 || (mask > 3 && mask != 8)) || - (insn.words[2] & ~0x04000f00U) != 255 || source > 245 || (mask == 3 && (dest & 1)) || dest >= 254) - return fail(index, "native bindless texture dimension query mode"); - emit(0xdf50000000000000ULL | predicate | dest | (u64{source} << 8) | - (u64{1} << 22) | (u64{mask} << 31) | (u64{insn.Bits(90, 1)} << 49)); + if (!bindless) { + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native texture descriptor buffer conflict"); + result.required_texture_buffer = binding; + } + emit((bindless ? 0xdf50000000000000ULL : 0xdf48000000000000ULL) | predicate | (direct ? dest : 250) | + (u64{insn.Bits(24, 8)} << 8) | (u64{1} << 22) | (u64{mask} << 31) | + (u64{bindless ? 0U : insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49)); + for (u32 i = 0; !direct && i < components; ++i) { + const u32 target = i < 2 ? (dest == 255 ? 255U : static_cast<u32>(dest) + i) : second_dest + (i - 2); + if (target != 255) + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{250 + i} << 20)); + } break; } - case 0xb66: { // TLD.SCR.LZ, direct texel fetch + case 0xb66: { // TLD.SCR, direct texel fetch at level zero (.LZ) or a given level (.LL) const u32 mask = insn.Bits(72, 4); + const bool level = insn.Bits(87, 3) == 3; // .LL: the level follows the coordinates + // Bit 76: a texel offset after the coordinates and level. 2D only, as Maxwell TLDS + // 2D.LZ.AOFFI (x, y in Ra, Ra+1, offsets in Rb) or 2D.LL.AOFFI (level, offsets in Rb, Rb+1). + const bool offsets = insn.Bits(76, 1) != 0; const u32 dimension = insn.Bits(61, 3); const bool array = dimension == 5; const bool volume = dimension == 2; @@ -1569,8 +1666,12 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i; } if (!insn.Bits(60, 1) || (dimension != 0 && dimension != 1 && !array && !volume) || - swizzle == ~0U || insn.Bits(76, 5) || - insn.Bits(81, 3) != 7 || insn.Bits(87, 3) != 1 || + swizzle == ~0U || (insn.Bits(76, 5) & ~1U) || + (offsets && (dimension != 1 || (source_a & 1) || source_a >= 254 || source_b == 255 || + (level && ((source_b & 1) || source_b >= 254)))) || + insn.Bits(81, 3) != 7 || (insn.Bits(87, 3) != 1 && !level) || (level && volume) || + (level && dimension == 1 && ((source_a & 1) || source_a >= 254)) || + (level && array && ((source_b & 1) || source_b >= 254)) || insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 || dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) || @@ -1581,6 +1682,8 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } const u32 binding = insn.Bits(54, 5); if (!bind_texture_buffer(binding)) { + if (level || offsets) + return fail(index, "native texture descriptor buffer conflict"); // A second descriptor buffer: TLD.B (LZ) with the coordinates in R250.. (array // index or Z after Ra) and the handle, loaded from that buffer, in R253. const auto copy = [&](u32 target, u32 source) { @@ -1599,6 +1702,25 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { copy(i < 2 || !split ? static_cast<u32>(dest) + i : second_dest + (i - 2), 250 + i); break; } + if (array && level) { + // TLDS has no array form with a level: TLD takes (layer, x, y) from Ra and the level + // from Rb (layer, x in Ra, Ra+1; y, level in Rb, Rb+1 here). Results come out packed, + // then go where the Ampere destinations (Rd, Rd+1, then the second pair) take them. + for (u32 i = 0; i < 2; ++i) { + emit(0x5c98000000000000ULL | predicate | (250 + i) | (u64{15} << 39) | (u64{source_a + i} << 20)); + emit(0x5c98000000000000ULL | predicate | (252 + i) | (u64{15} << 39) | (u64{source_b + i} << 20)); + } + emit(0xdc00000000000000ULL | predicate | 246 | (u64{250} << 8) | (u64{253} << 20) | + (u64{3} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | (u64{1} << 55)); + const u32 components = static_cast<u32>(std::popcount(mask)); + for (u32 i = 0; i < components; ++i) { + const u32 target = !split || i < 2 ? static_cast<u32>(dest) + i : second_dest + (i - 2); + if (target != 255 && (split || i < 2)) + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{246 + i} << 20)); + } + break; + } if (array) { emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a + 1} << 20)); emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{source_b} << 20)); @@ -1607,7 +1729,42 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0xda00000000000000ULL | predicate | dest | (u64{source_a} << 8) | (u64{source_b} << 20) | (u64{insn.Bits(64, 8)} << 28) | (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | - (u64{swizzle} << 50) | (u64{array ? 8U : volume ? 7U : dimension == 0 ? 0U : 2U} << 53)); + (u64{swizzle} << 50) | + (u64{offsets ? (level ? 12U : 4U) + : array ? 8U : volume ? 7U : dimension == 0 ? (level ? 1U : 0U) : (level ? 5U : 2U)} << 53)); + break; + } + case 0x367: { // TLD.B at level zero (.LZ), the handle in a register. + // The values are the coordinates, then the handle, split over Ra and Rb as for TEX.B + // (ARRAY_2D: layer, x | y, handle); Maxwell TLD.B takes the handle first in Rb. + constexpr std::array<u32, 8> coordinate_count{1, 2, 3, 0, 0, 3, 0, 0}; + constexpr std::array<u32, 8> maxwell_type{0, 2, 4, 0, 0, 3, 0, 0}; + const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8); + const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + const u32 dimension = insn.Bits(61, 3), coordinates = coordinate_count[dimension]; + const u32 count = coordinates + 1, in_a = (count + 1) / 2; + if (coordinates == 0 || !insn.Bits(59, 1) || !insn.Bits(60, 1) || insn.Bits(40, 19) || + insn.Bits(76, 5) || insn.Bits(81, 3) != 7 || insn.Bits(87, 3) != 1 || insn.Bits(91, 1) || + mask == 0 || source_a == 255 || source_a + in_a > 246 || source_b == 255 || + source_b + (count - in_a) > 246) + return fail(index, "native bindless texel fetch mode"); + const auto value = [&](u32 i) { return i < in_a ? source_a + i : source_b + (i - in_a); }; + const auto copy = [&](u32 target, u32 source) { + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20)); + }; + for (u32 i = 0; i < coordinates; ++i) copy(250 + i, value(i)); + copy(246, value(coordinates)); + emit(0xdd00000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{246} << 20) | + (u64{maxwell_type[dimension]} << 28) | (u64{mask} << 31) | (u64{insn.Bits(90, 1)} << 49) | + (u64{7} << 51)); + const u32 components = static_cast<u32>(std::popcount(mask)); + const bool split = second_dest != 255; + for (u32 i = 0; i < components; ++i) { + u32 target = 255; + if (!split || i < 2) target = dest == 255 ? 255U : static_cast<u32>(dest) + i; + else target = second_dest + (i - 2); + if (target != 255) copy(target, 250 + i); + } break; } case 0x91a: // DEPBAR.LE SB5, 1: scoreboard wait, represented by IR data dependencies. @@ -1615,22 +1772,35 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { return fail(index, "native dependency barrier mode"); } break; - case 0xb69: { // TMML (LOD query), 2D. Results are 8.8 fixed point, as on Maxwell. + case 0xb69: case 0xf69: case 0x36a: { // TMML (LOD query), 1D/2D/3D/CUBE. Results are 8.8 fixed point, as on Maxwell. + // 0xf69: the handle in a uniform register (bit 91); 0x36a, TMML.B: the handle in Rb and + // every coordinate in Ra. Both become Maxwell TMML.B with the handle in Rb. + const bool bindless = opcode != 0xb69; + const u32 handle = opcode == 0xf69 ? uniform_register(insn.Bits(40, 6)) + : opcode == 0x36a ? insn.Bits(32, 8) : 255; const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8); const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); - const u32 binding = insn.Bits(54, 5); - if (insn.Bits(61, 3) != 1 || insn.Bits(76, 4) || insn.Bits(80, 10) || insn.Bits(59, 1) || - insn.Bits(40, 14) > 0x1fff || binding >= 18 || mask == 0 || source_a >= 246 || - (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)) + const u32 binding = insn.Bits(54, 5), dimension = insn.Bits(61, 3); + // Coordinates: all from Ra when Rb is RZ, else split as for TEX (2D: Ra, Rb). + constexpr std::array<u32, 4> coordinate_count{1, 2, 3, 3}; + const u32 coordinates = dimension < 4 ? coordinate_count[dimension] : 0; + const u32 in_a = source_b == 255 || opcode == 0x36a ? coordinates : (coordinates + 1) / 2; + if (coordinates == 0 || insn.Bits(76, 4) || insn.Bits(80, 10) || mask == 0 || + insn.Bits(59, 1) != (opcode == 0x36a ? 1U : 0U) || source_a + in_a > 246 || + (in_a < coordinates && source_b + (coordinates - in_a) > 246) || + (opcode == 0xf69 && (handle >= 246 || insn.Bits(46, 13) || !insn.Bits(91, 1))) || + (opcode == 0x36a && (handle >= 246 || insn.Bits(40, 19) || insn.Bits(91, 1))) || + (!bindless && (insn.Bits(40, 14) > 0x1fff || binding >= 18 || + (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)))) return fail(index, "native texture LOD query mode"); - result.required_texture_buffer = binding; - // Coordinates: (Ra, Ra+1) when Rb is RZ, else (Ra, Rb). - emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a} << 20)); - emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | - (u64{source_b == 255 ? source_a + 1 : source_b} << 20)); - emit(0xdf58000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | - (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | - (u64{insn.Bits(90, 1)} << 49)); + if (!bindless) result.required_texture_buffer = binding; + for (u32 i = 0; i < coordinates; ++i) + emit(0x5c98000000000000ULL | predicate | (250 + i) | (u64{15} << 39) | + (u64{i < in_a ? source_a + i : source_b + (i - in_a)} << 20)); + if (bindless) emit(0x5c98000000000000ULL | predicate | 246 | (u64{15} << 39) | (u64{handle} << 20)); + emit((bindless ? 0xdf60000000000000ULL : 0xdf58000000000000ULL) | predicate | 250 | (u64{250} << 8) | + (u64{bindless ? 246U : 255U} << 20) | (u64{dimension * 2} << 28) | (u64{mask} << 31) | + (u64{bindless ? 0U : insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49)); const u32 components = static_cast<u32>(std::popcount(mask)); const bool split = second_dest != 255; for (u32 i = 0; i < components; ++i) { @@ -1642,22 +1812,43 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0xb63: { // TLD4 (gather), 2D without offsets or depth compare; component at bits 87-88. + case 0xb63: case 0xf63: { // TLD4 (gather), 2D without offsets or depth compare; component at bits 87-88. + // 0xf63: the handle in a uniform register (bit 91), Maxwell TLD4.B with the handle in Rb. + const bool bindless = opcode == 0xf63; + const u32 handle = bindless ? uniform_register(insn.Bits(40, 6)) : 255; const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8); const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); const u32 binding = insn.Bits(54, 5); - if (insn.Bits(61, 3) != 1 || insn.Bits(76, 4) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || - insn.Bits(40, 14) > 0x1fff || binding >= 18 || insn.Bits(89, 1) || mask == 0 || - (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)) + // Bit 76: a texel offset after the coordinates (x, y in Ra, Ra+1, offsets in Rb, as + // for TEX); Maxwell TLD4 takes it as AOFFI (bits 54-55) from Rb. + const bool offsets = insn.Bits(76, 1) != 0; + if (insn.Bits(61, 3) != 1 || insn.Bits(77, 3) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || + (offsets && (bindless || source_b == 255 || (source_a != 255 && source_a >= 245))) || + insn.Bits(89, 1) || mask == 0 || + (bindless ? (handle >= 246 || insn.Bits(46, 13) || !insn.Bits(91, 1)) + : (insn.Bits(40, 14) > 0x1fff || binding >= 18 || + (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)))) return fail(index, "native texture gather mode"); - result.required_texture_buffer = binding; - // Two coordinates: x in Ra, y in Rb (the coordinate list splits evenly across them). + if (!bindless) result.required_texture_buffer = binding; + // Two coordinates: x in Ra, y in Rb (the coordinate list splits evenly across them); + // with an offset, x and y in Ra, Ra+1 and the offset in Rb. + const u32 y = !offsets ? source_b : source_a == 255 ? 255U : source_a + 1; emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a} << 20)); - emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | - (u64{source_b == 255 ? 255U : source_b} << 20)); - emit(0xc800000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | - (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | - (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | (u64{insn.Bits(87, 2)} << 56)); + emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{y} << 20)); + if (offsets) { + emit(0x5c98000000000000ULL | predicate | 246 | (u64{15} << 39) | (u64{source_b} << 20)); + } + if (bindless) { + emit(0x5c98000000000000ULL | predicate | 246 | (u64{15} << 39) | (u64{handle} << 20)); + emit(0xdec0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{246} << 20) | + (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(87, 2)} << 38) | + (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51)); + } else { + emit(0xc800000000000000ULL | predicate | 250 | (u64{250} << 8) | + (u64{offsets ? 246U : 255U} << 20) | (u64{2} << 28) | (u64{mask} << 31) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | + (u64{offsets ? 1U : 0U} << 54) | (u64{insn.Bits(87, 2)} << 56)); + } const u32 components = static_cast<u32>(std::popcount(mask)); const bool split = second_dest != 255; for (u32 i = 0; i < components; ++i) { @@ -1669,10 +1860,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0x361: // TEX with the descriptor handle in a register (after the coordinates) - case 0xf60: // TEX with the descriptor handle in a uniform register - case 0xb60: { // TEX, scalar 2D implicit-LOD sample - const bool register_handle = opcode == 0x361, uniform_handle = opcode == 0xf60; + case 0xb60: case 0xf60: case 0x361: { // TEX; 0xf60: handle in a uniform register; 0x361: TEX.B + // Bindless forms become Maxwell TEX.B, the handle first among the meta registers (the + // texture pass tracks it back to its constant buffer words). 0x361 lists the handle + // right after the coordinates, then the LOD and the offsets (see general_tex). + const bool bindless = opcode != 0xb60; + const u32 handle = opcode == 0xf60 ? uniform_register(insn.Bits(40, 6)) : 255; + if (bindless && ((opcode == 0xf60 && (handle == ~0U || handle >= 246 || insn.Bits(46, 13))) || + (opcode == 0x361 && (insn.Bits(40, 19) || insn.Bits(87, 3) > 3)))) + return fail(index, "native bindless texture mode"); const u32 mask = insn.Bits(72, 4); const u32 second_dest = insn.Bits(64, 8); constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12}; @@ -1682,8 +1878,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { const bool depth_compare = insn.Bits(78, 1); const u32 dimension = insn.Bits(61, 3), lod = insn.Bits(87, 3); const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); - if (dimension == 5 && depth_compare && lod == 0 && !split && mask == 1 && !register_handle && - !uniform_handle) { + if (!bindless && dimension == 5 && depth_compare && lod == 0 && !split && mask == 1) { if (!insn.Bits(60, 1) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || (source_a & 1) || (source_b & 1) || source_a >= 248 || source_b >= 248 || dest >= 248) @@ -1708,24 +1903,23 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { constexpr std::array<u32, 8> coordinate_count{1, 2, 3, 3, 2, 3, 0, 4}; constexpr std::array<u32, 8> maxwell_type{0, 2, 4, 6, 1, 3, 0, 7}; const bool offsets = insn.Bits(76, 1) != 0; - // Handle forms: 0x361 has no descriptor fields (bit 59 set), 0xf60 names a UR - // in bits 40-45 (bit 91 set) and is only known with packed operands. - const u32 handle_register = uniform_handle ? uniform_register(insn.Bits(40, 6)) : ~0U; - if (register_handle ? !insn.Bits(59, 1) || insn.Bits(40, 19) - : uniform_handle ? !insn.Bits(60, 1) || insn.Bits(59, 1) || insn.Bits(46, 13) || - !insn.Bits(91, 1) || handle_register == ~0U || handle_register == 255 - : insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18) - return false; + // Bit 59 is .B, set in 0x361's encoding. if (dimension == 6 || insn.Bits(77, 1) || insn.Bits(81, 3) != 7 || + insn.Bits(59, 1) != (opcode == 0x361 ? 1U : 0U) || + (!bindless && (insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18)) || (lod > 3 && lod != 6 && lod != 7) || (depth_compare && mask != 1) || mask == 0) return false; const u32 coordinates = coordinate_count[dimension]; const bool has_lod = lod == 2 || lod == 3 || lod == 6 || lod == 7; - const u32 count = coordinates + (register_handle ? 1 : 0) + (has_lod ? 1 : 0) + - (offsets ? 1 : 0) + (depth_compare ? 1 : 0); + // 0x361's handle is one more value after the coordinates: 2D.DC is (x, y | handle, + // reference), ARRAY_2D is (layer, x | y, handle); with five or more values the + // coordinates keep Ra (CUBE.LL is (x, y, z | handle, LOD)). + const u32 handle_slot = opcode == 0x361 ? 1 : 0; + const u32 count = coordinates + handle_slot + (has_lod ? 1 : 0) + (offsets ? 1 : 0) + + (depth_compare ? 1 : 0); // Unpacked 0x361 (bit 60 clear): Ra holds the coordinates, Rb the handle and the rest. - const u32 in_a = register_handle && !insn.Bits(60, 1) ? coordinates - : count <= 4 ? (count + 1) / 2 : 4; + const u32 in_a = opcode == 0x361 && !insn.Bits(60, 1) ? coordinates + : count <= 4 ? (count + 1) / 2 : opcode == 0x361 ? coordinates : 4; const auto value = [&](u32 i) -> u32 { if (i < in_a) return source_a == 255 ? 255U : source_a + i; return source_b == 255 ? 255U : source_b + (i - in_a); @@ -1733,34 +1927,40 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { if ((source_a != 255 && source_a + in_a > 246) || (count > in_a && source_b != 255 && source_b + (count - in_a) > 246)) return false; - // Handle forms and a second descriptor buffer are read bindless: TEX.B takes the - // handle (the register's, or loaded from that buffer) as its first meta register. - const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); - const bool bindless = register_handle || uniform_handle || !bind_texture_buffer(binding); + const u32 texture_handle = opcode == 0x361 ? value(coordinates) : handle; + if (bindless && texture_handle >= 246) return false; + // A second descriptor buffer is read bindless: TEX.B takes the handle, loaded from + // that buffer, as its first meta register. + const u32 binding = insn.Bits(54, 5); + const bool second_buffer = !bindless && !bind_texture_buffer(binding); + const bool tex_b = bindless || second_buffer; const auto copy = [&](u32 target, u32 source) { emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20)); }; - // Coordinates to R250.., then the handle, LOD/bias, offsets and reference to R246... + // Coordinates to R250.., then LOD/bias, offsets and reference to R246... for (u32 i = 0; i < coordinates; ++i) copy(250 + i, value(i)); u32 meta = 246; - u32 next = coordinates; - // The handle is copied unpredicated, so it is the same value on every path. - if (register_handle || uniform_handle) { - emit(0x5c98000000070000ULL | meta++ | (u64{15} << 39) | - (u64{register_handle ? value(next++) : handle_register} << 20)); - } else if (bindless) { - emit(0x4c98000000070000ULL | meta++ | (u64{15} << 39) | (u64{offset} << 20) | + u32 next = coordinates + handle_slot; + if (bindless) copy(meta++, texture_handle); + else if (second_buffer) + emit(0x4c98000000070000ULL | meta++ | (u64{15} << 39) | (u64{insn.Bits(40, 14)} << 20) | (u64{binding} << 34)); - } if (has_lod) copy(meta++, value(next++)); if (offsets) copy(meta++, value(next++)); if (depth_compare) copy(meta++, value(next++)); - emit((bindless ? 0xde80000000000000ULL : 0xc000000000000000ULL) | predicate | 250 | (u64{250} << 8) | + if (tex_b) { + // Maxwell TEX.B: AOFFI at 36 and the LOD mode at 37 instead of 54 and 55. + emit(0xde80000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{246} << 20) | + (u64{maxwell_type[dimension]} << 28) | (u64{mask} << 31) | (u64{offsets} << 36) | + (u64{lod} << 37) | (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) | + (u64{7} << 51)); + } else { + emit(0xc000000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{meta == 246 ? 255U : 246U} << 20) | (u64{maxwell_type[dimension]} << 28) | - (u64{mask} << 31) | (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) | - (u64{7} << 51) | - (bindless ? (u64{offsets} << 36) | (u64{lod} << 37) - : (u64{offset} << 36) | (u64{offsets} << 54) | (u64{lod} << 55))); + (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) | (u64{7} << 51) | + (u64{offsets} << 54) | (u64{lod} << 55)); + } const u32 components = static_cast<u32>(std::popcount(mask)); if (insn.Bits(79, 1)) { // FP16 results: (c0, c1) packed into the first destination, (c2, c3) into @@ -1795,7 +1995,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { // SM70 scalar coordinates and split RG/BA destinations match TEXS. // Descriptor offsets are words in both encodings, but only native // explicitly encodes the constant-buffer binding. - if (!insn.Bits(60, 1) || (dimension != 1 && dimension != 5) || + if (bindless || !insn.Bits(60, 1) || (dimension != 1 && dimension != 5) || swizzle == ~0U || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 || (lod > 1 && (lod != 3 || dimension != 1 || depth_compare)) || (lod == 3 && ((source_a & 1) || source_a >= 249)) || @@ -1804,7 +2004,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) || (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) || (depth_compare && (dimension != 1 || split || mask != 1 || (source_a & 1) || source_a >= 249)) || - (dimension == 5 && ((source_a & 1) || source_a >= 249)) || register_handle || uniform_handle || + (dimension == 5 && ((source_a & 1) || source_a >= 249)) || bindless || !bind_texture_buffer(2)) { if (!general_tex()) return fail(index, "native texture mode or descriptor binding"); break; @@ -1848,11 +2048,12 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0x307: { // FRND.F32, round to an integral floating-point value + case 0x307: case 0xb07: { // FRND.F32, round to an integral floating-point value (0xb07: cbuf) if (insn.Bits(75, 2) != 2 || insn.Bits(84, 2) != 2 || insn.Bits(81, 3) || insn.Bits(86, 6)) return fail(index, "native floating round format"); - emit(base(0x5ca8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | - (u64{insn.Bits(32, 8)} << 20) | (u64{8 + insn.Bits(78, 2)} << 39) | + if (opcode == 0xb07 && !valid_cbuf()) return fail(index, "native constant buffer address"); + emit(base(opcode == 0xb07 ? 0x4ca8000000000000ULL : 0x5ca8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | + (opcode == 0xb07 ? cbuf() : u64{insn.Bits(32, 8)} << 20) | (u64{8 + insn.Bits(78, 2)} << 39) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(63, 1)} << 45) | (u64{insn.Bits(62, 1)} << 49)); break; @@ -1883,15 +2084,22 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { result.instructions.push_back(translated); break; } - case 0x387: case 0x983: { // Captured scalar local spill at an immediate byte offset. + case 0x387: case 0x983: { // Local spill: 32, 64 or 128 bits at [Ra + immediate byte offset]. + // The size is bits 73-75 (4, 5, 6 = 32, 64, 128 bits); wider accesses become 32-bit ones. + // A load's bit 85 (.LU) is a cache hint. const bool store = opcode == 0x387; - const u32 data = insn.Bits(32, 8), offset = insn.Bits(40, 24); - if (insn.Bits(24, 8) != 255 || (store && (insn.Bits(16, 8) || data > 247)) || - (!store && (data || dest >= 248)) || (offset & 3) || offset > 0x7fffff || - insn.words[2] != (store ? 0x00100800U : 0x00300800U)) + const u32 size = insn.Bits(73, 3); + const u32 data = insn.Bits(32, 8), offset = insn.Bits(40, 24), address = insn.Bits(24, 8); + const u32 words = size == 6 ? 4 : size == 5 ? 2 : 1, value = store ? data : static_cast<u32>(dest); + if ((address != 255 && address >= 246) || (store && insn.Bits(16, 8)) || (!store && data) || + (value != 255 && (value + words > 248 || (value & (words - 1)))) || (!store && value == 255) || + (offset & 3) || offset + 4 * words > 0x800000 || size < 4 || size > 6 || + (insn.words[2] & ~(store ? 0xe00U : 0x200e00U)) != 0x00100000U) return fail(index, "native local scalar spill mode"); - emit((store ? 0xef50000000000000ULL : 0xef40000000000000ULL) | predicate | - (store ? data : dest) | (u64{255} << 8) | (u64{offset} << 20) | (u64{4} << 48)); + for (u32 word = 0; word < words; ++word) + emit((store ? 0xef50000000000000ULL : 0xef40000000000000ULL) | predicate | + (value == 255 ? 255U : value + word) | (u64{address} << 8) | + (u64{offset + 4 * word} << 20) | (u64{4} << 48)); break; } case 0x803: { // P2R PR, register merge source, immediate mask, byte select (76-77) @@ -1974,9 +2182,12 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0x308: case 0xb08: { // MUFU + case 0x308: case 0xb08: case 0xd08: { // MUFU (register, constant or uniform register source) const u32 operation = insn.Bits(74, 6); - if (operation > 8 || operation == 6 || operation == 7 || insn.Bits(80, 12)) return fail(index, "native MUFU mode"); + // Bit 91 is part of the uniform-register form's encoding. + if (operation > 8 || operation == 6 || operation == 7 || + insn.Bits(80, 12) != (opcode == 0xd08 ? 0x800U : 0U)) + return fail(index, "native MUFU mode"); // SM70+ MUFU.COS/SIN take the angle in turns (compilers multiply by 1/2pi first); // Eden's Maxwell MUFU takes radians (its RRO is a plain move), so scale by 2pi. const bool turns = operation == 0 || operation == 1; @@ -1995,7 +2206,8 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0x5080000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{operation} << 20)); emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) | (u64{250} << 20)); } else { - u32 src = operand(1); + u32 src = opcode == 0xd08 ? uniform_register(insn.Bits(32, 8)) : operand(1); + if (src == ~0U) return fail(index, "native MUFU uniform source"); if (turns) src = to_radians(src); emit(base(0x5080000000000000ULL) | (u64{src} << 8) | (u64{operation} << 20)); } @@ -2065,7 +2277,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { mov_imm(static_cast<u32>(dest) + 1, 0); break; } - case 0x306: case 0xd06: { // I2F.F32.U8, low byte, round-to-nearest (0xd06: uniform source) + case 0x306: case 0xd06: { // I2F.F32.U8, round-to-nearest (0xd06: uniform source) const u32 source = opcode == 0xd06 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); if (opcode == 0x306 && insn.words[2] == 0x00101000 && !(insn.words[1] & 0xcfffff00U)) { // I2F.U16 Rd, Rs(.H1): Maxwell's 16-bit source format, byte selector 0 or 2. @@ -2073,10 +2285,12 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (u64{insn.Bits(60, 2) * 2} << 41)); break; } + // I2F.U8 Rd, Rs.B0-B3: the byte selector (bits 60-61) as Maxwell's (41-42). if (insn.words[2] != (opcode == 0xd06 ? 0x08001000U : 0x1000U) || - (insn.words[1] & 0xffffff00U) || source == ~0U) + (insn.words[1] & (opcode == 0xd06 ? 0xffffff00U : 0xcfffff00U)) || source == ~0U) return fail(index, "native byte-to-float conversion mode"); - emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{source} << 20)); + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{source} << 20) | + (u64{insn.Bits(60, 2)} << 41)); break; } case 0x245: case 0xa45: case 0xc45: { // I2FP.F32.U32 @@ -2105,58 +2319,60 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (u64{insn.Bits(76, 4)} << 48) | (u64{1} << 52) | (u64{insn.Bits(80, 1)} << 55)); break; } - case 0x434: { // HSETP2 with an immediate half pair: A swizzle 74-75 (neg 72, abs 73), + case 0x234: case 0x434: { // HSETP2 with a register (B swizzle 60-61, abs 62, neg 63) or an + // immediate half pair: A swizzle 74-75 (neg 72, abs 73), // combine 69-70, H_AND 71, compare 76-79, FTZ 80, P/Q 81-86, input 87-90. - mov_imm(250, insn.words[1]); + u64 source = u64{250} << 20; + if (opcode == 0x234) { + if ((insn.words[1] & 0x0fffff00U) || insn.Bits(64, 5)) + return fail(index, "native half comparison mode"); + source = (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(60, 2)} << 28) | + (u64{insn.Bits(62, 1)} << 30) | (u64{insn.Bits(63, 1)} << 31); + } else { + mov_imm(250, insn.words[1]); + } emit(0x5d20000000000000ULL | predicate | insn.Bits(84, 3) | (u64{insn.Bits(81, 3)} << 3) | - (u64{insn.Bits(80, 1)} << 6) | (u64{insn.Bits(24, 8)} << 8) | (u64{250} << 20) | + (u64{insn.Bits(80, 1)} << 6) | (u64{insn.Bits(24, 8)} << 8) | source | (u64{insn.Bits(76, 4)} << 35) | (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | (u64{insn.Bits(72, 1)} << 43) | (u64{insn.Bits(73, 1)} << 44) | (u64{insn.Bits(69, 2)} << 45) | (u64{insn.Bits(74, 2)} << 47) | (u64{insn.Bits(71, 1)} << 49)); break; } - case 0x20b: case 0x80b: case 0xa0b: // FSETP + case 0x20b: case 0x80b: case 0xa0b: case 0xc0b: // FSETP case 0x20c: case 0x80c: case 0xa0c: case 0xc0c: { // ISETP const bool floating = (opcode & 0x1ff) == 0xb; if (!floating && insn.Bits(72, 1)) { - // ISETP.EX: the high words, chained to the low words' predicate in bits 68-71. - const u32 comparison = insn.Bits(76, 3), low_comparison = insn.Bits(68, 4); - if (insn.Bits(74, 2) || insn.Bits(84, 3) != 7 || insn.Bits(87, 4) != 7) - return fail(index, "native extended integer comparison"); - if (comparison == 1 || comparison == 3 || comparison == 4 || comparison == 6) { - // Unsigned LT/LE/GT/GE: (a < b, or > for GT/GE) || (a == b && low predicate), - // built from 0/~0 ISET masks, then one ISETP.NE into the output predicate. - const u32 a = insn.Bits(24, 8), b = insn.Bits(32, 8); - if (opcode != 0x20c || insn.Bits(73, 1) || insn.Bits(64, 4) || insn.Bits(80, 1) || - (insn.words[1] & ~0xffU)) - return fail(index, "native extended ordered comparison mode"); - const auto set_mask = [&](u32 target, u32 operation, u32 lhs, u32 rhs) { - emit(0x5b50000000000000ULL | predicate | target | (u64{lhs} << 8) | - (u64{rhs} << 20) | (u64{7} << 39) | (u64{operation} << 49)); - }; - if ((low_comparison & 7) == 7) { - mov_imm(252, (low_comparison & 8) ? 0U : ~0U); - } else { - emit(0x38e8000000000000ULL | predicate | 252 | (u64{255} << 8) | - (u64{1} << (20 + (low_comparison & 7)))); - set_mask(252, (low_comparison & 8) ? 2U : 5U, 252, 255); - } - set_mask(250, comparison == 1 || comparison == 3 ? 1U : 4U, a, b); - set_mask(251, 2, a, b); - emit(0x5c40000000000000ULL | predicate | 251 | (u64{251} << 8) | (u64{252} << 20) | - (u64{7} << 48)); - emit(0x5c40000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{251} << 20) | - (u64{1} << 41) | (u64{7} << 48)); - emit(0x5b60000000000000ULL | predicate | 7 | (u64{insn.Bits(81, 3)} << 3) | - (u64{250} << 8) | (u64{255} << 20) | (u64{7} << 39) | (u64{5} << 49)); - break; + // ISETP.EX (bit 72): the high-word half of a 64-bit compare. The low words' + // result comes in as the predicate at bits 68-70 (negated by 71), e.g. + // ISETP.NE P0 = lo != 0, then ISETP.NE.EX P0 = (hi != 0) OR P0. EQ: high equal + // AND low; NE: high different OR low; the orderings: high strictly ordered OR + // (high equal AND low). One or two Maxwell ISETPs into the destination; only + // the plain form (no second destination, AND with PT) is handled. + const u32 compare = insn.Bits(76, 3); + const u32 destination = insn.Bits(81, 3); + const u32 low = insn.Bits(68, 4); + if (insn.Bits(84, 3) != 7 || insn.Bits(87, 4) != 7 || insn.Bits(74, 2) != 0 || + compare == 0 || compare == 7 || + (destination != 7 && insn.Bits(12, 3) == destination)) + return fail(index, "native extended integer comparison mode"); + const u32 a = operand(0, false), b = operand(1, false); + const auto isetp = [&](u32 op, u32 input, u32 combine) { + return 0x5b60000000000000ULL | predicate | 7 | (u64{destination} << 3) | + (u64{a} << 8) | (u64{b} << 20) | (u64{input & 7} << 39) | + (u64{input >> 3} << 42) | (u64{combine} << 45) | + (u64{insn.Bits(73, 1)} << 48) | (u64{op} << 49); + }; + constexpr u32 And = 0, Or = 1, Lt = 1, Eq = 2, Gt = 4, Ne = 5; + if (compare == Eq) { + emit(isetp(Eq, low, And)); + } else if (compare == Ne) { + emit(isetp(Ne, low, Or)); + } else { + emit(isetp(Eq, low, And)); + emit(isetp(compare <= 3 ? Lt : Gt, destination, Or)); } - // EQ/NE: the high comparison ANDed (EQ) or ORed (NE) with that predicate. - if (comparison != 2 && comparison != 5) - return fail(index, "native extended integer comparison"); - insn.words[2] = (insn.words[2] & ~(0x100U | 0xc00U | 0x07800000U)) | - (insn.Bits(68, 4) << 23) | (comparison == 5 ? 0x400U : 0U); + break; } const u32 a = operand(0, floating), b = operand(1, floating); const u32 combine = insn.Bits(74, 2); @@ -2308,23 +2524,31 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - const u32 carry_output = insn.Bits(81, 3); + // A carry out (bits 81-83) is the low half of a 64-bit address; IADD3.X adds it next. + const u32 carry_out = insn.Bits(81, 3); if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || - (carry_output != 7 && (rewritten_uniform || (opcode != 0x811 && insn.Bits(63, 1)))) || + (carry_out != 7 && ((opcode != 0x811 && insn.Bits(63, 1)) || + (rewritten_uniform && (opcode != 0x211 || carry_out > 1)))) || (opcode == 0x211 && insn.Bits(62, 1)) || (opcode == 0xc11 && (insn.words[1] & ~0x8000003fU))) return fail(index, "native LEA carry/high mode"); - // 0x211/0xc11 bit 63 negates Rb (LEA Rd, Ra, -Rb, s), as Maxwell ISCADD's bit 48. + // 0x211 bit 63 negates Rb (LEA Rd, Ra, -Rb, s), as Maxwell ISCADD's bit 48. u32 b = opcode == 0xc11 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); if (b == ~0U) return fail(index, "native uniform LEA register"); if (opcode == 0x811) { mov_imm(251, insn.words[1]); b = 251; } - if (carry_output != 7) { - // LEA Rd, Pu, Ra, Rb, s: Pu is the carry of (Ra << s) + Rb, set when the sum wraps - // below Ra << s. The guard may be Pu, so Rd is written first. - emit(0x3848000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(75, 5)} << 20)); - emit(0x5c10000000000000ULL | predicate | 252 | (u64{250} << 8) | (u64{b} << 20)); - emit(base(0x5c98000000000000ULL) | (u64{252} << 20) | (u64{15} << 39)); - emit(0x5b60000000000000ULL | predicate | 7 | (u64{carry_output} << 3) | (u64{252} << 8) | - (u64{250} << 20) | (u64{7} << 39) | (u64{1} << 49)); + if (carry_out != 7) { + // The sum in R250, carry = sum < Rb (unsigned), as IADD3's carry out; a uniform + // predicate goes through P(6-n) into R239. + const u32 target = rewritten_uniform ? 6 - carry_out : carry_out, mask = 1U << target; + emit(0x5c18000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | + (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39)); + if (rewritten_uniform) emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{target} << 3) | (u64{250} << 8) | + (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); + if (rewritten_uniform) { + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20)); + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + } + emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{250} << 20)); break; } emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | @@ -2450,12 +2674,11 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { if ((insn.words[2] & ~(0x07800000U | (opcode == 0xc10 ? 0x08000000U : 0U))) != 0x007fe4ffU || (opcode == 0xc10 && (insn.words[1] & ~0x3fU))) return fail(index, "native extended constant add mode"); - const u32 b = operand(1, false); + const u32 a = insn.Bits(24, 8), b = operand(1, false); mov_imm(250, 1); emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); - if (insn.Bits(24, 8) != 255) - emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{insn.Bits(24, 8)} << 20)); + if (a != 255) emit(0x5c10000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{250} << 20)); emit(base(0x5c10000000000000ULL) | (u64{b} << 8) | (u64{250} << 20)); break; } @@ -2588,6 +2811,24 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { return finish(index); // the closing self-branch break; } + case 0x923: case 0x927: { // ISBERD / ISBEWR: task and mesh programs' stage buffer. + // Ampere: Ra 24-31, offset 40-63, ISBERD dest 16-23, ISBEWR data 32-39; 75 = .32 (else + // U8), 76-77 = 3 .ATTR (0 the map region), 78 = .SKEW, 79 = .O (the output buffer). + // Kept as an nx2 pseudo instruction (Maxwell 0xefc0, free there) that the frontend + // reads for the task or mesh stage: data/dest 0-7, Ra 8-15, offset 20-43, .32 44, + // .ATTR 45, .SKEW 46, .O 47, write 48. Addresses are linear bytes in the region. + const bool write = opcode == 0x927; + const u32 region = insn.Bits(76, 2); + const bool output = insn.Bits(79, 1) != 0; + if ((insn.words[2] & ~0xf800U) != 0 || (region != 0 && region != 3) || write != output || + (!write && region != 3) || (!write && dest == 255)) + return fail(index, "native stage buffer access mode"); + emit(0xefc0000000000000ULL | predicate | (write ? u64{insn.Bits(32, 8)} : dest) | + (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(40, 24)} << 20) | + (u64{insn.Bits(75, 1)} << 44) | (u64{region == 3} << 45) | + (u64{insn.Bits(78, 1)} << 46) | (u64{output} << 47) | (u64{write} << 48)); + break; + } case 0x94d: { // EXIT if (insn.Bits(84, 2)) { return fail(index, "native conditional/special EXIT"); @@ -2638,4 +2879,14 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { return fail(static_cast<u32>(code.size()), main_exited ? "native program lacks captured library return" : "native program has no unconditional EXIT"); } +inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { + // Programs using uniform registers above UR27 are lowered again with mirrors for them. + u32 high_uniforms = 0; + for (;;) { + LoweredProgram lowered = LowerToMaxwellWith(code, high_uniforms); + if (lowered || lowered.high_uniform_registers <= high_uniforms) return lowered; + high_uniforms = lowered.high_uniform_registers; + } +} + } // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.cpp b/src/shader_recompiler/frontend/ir/ir_emitter.cpp index 95815d7d51..23d1db5bf4 100644 --- a/src/shader_recompiler/frontend/ir/ir_emitter.cpp +++ b/src/shader_recompiler/frontend/ir/ir_emitter.cpp @@ -329,6 +329,22 @@ void IREmitter::SetPatch(Patch patch, const F32& value) { Inst(Opcode::SetPatch, patch, value); } +U32 IREmitter::IsbeRead(const U32& address) { + return Inst<U32>(Opcode::IsbeRead, address); +} + +void IREmitter::IsbeWriteAttribute(const U32& address, const U32& value) { + Inst(Opcode::IsbeWriteAttribute, address, value); +} + +void IREmitter::IsbeWriteMapU8(const U32& address, const U32& value) { + Inst(Opcode::IsbeWriteMapU8, address, value); +} + +void IREmitter::IsbeWriteMapU32(const U32& address, const U32& value) { + Inst(Opcode::IsbeWriteMapU32, address, value); +} + void IREmitter::SetFragColor(u32 index, u32 component, const F32& value) { Inst(Opcode::SetFragColor, Imm32(index), Imm32(component), value); } diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.h b/src/shader_recompiler/frontend/ir/ir_emitter.h index f22f0f75e2..d0cd88ca3c 100644 --- a/src/shader_recompiler/frontend/ir/ir_emitter.h +++ b/src/shader_recompiler/frontend/ir/ir_emitter.h @@ -88,6 +88,11 @@ public: [[nodiscard]] F32 GetPatch(Patch patch); void SetPatch(Patch patch, const F32& value); + [[nodiscard]] U32 IsbeRead(const U32& address); + void IsbeWriteAttribute(const U32& address, const U32& value); + void IsbeWriteMapU8(const U32& address, const U32& value); + void IsbeWriteMapU32(const U32& address, const U32& value); + void SetFragColor(u32 index, u32 component, const F32& value); void SetSampleMask(const U32& value); void SetFragDepth(const F32& value); diff --git a/src/shader_recompiler/frontend/ir/microinstruction.cpp b/src/shader_recompiler/frontend/ir/microinstruction.cpp index 84417980bc..a579ed1057 100644 --- a/src/shader_recompiler/frontend/ir/microinstruction.cpp +++ b/src/shader_recompiler/frontend/ir/microinstruction.cpp @@ -82,6 +82,9 @@ bool Inst::MayHaveSideEffects() const noexcept { case Opcode::SetAttribute: case Opcode::SetAttributeIndexed: case Opcode::SetPatch: + case Opcode::IsbeWriteAttribute: + case Opcode::IsbeWriteMapU8: + case Opcode::IsbeWriteMapU32: case Opcode::SetFragColor: case Opcode::SetSampleMask: case Opcode::SetFragDepth: diff --git a/src/shader_recompiler/frontend/ir/opcodes.cpp b/src/shader_recompiler/frontend/ir/opcodes.cpp index 94ba0f6ae0..1999f877fe 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.cpp +++ b/src/shader_recompiler/frontend/ir/opcodes.cpp @@ -10,7 +10,7 @@ namespace Shader::IR { namespace Detail { -OpcodeMeta META_TABLE[534] = { +OpcodeMeta META_TABLE[538] = { #define OPCODE(name_token, type_token, ...) \ { \ .name{#name_token}, \ @@ -21,7 +21,7 @@ OpcodeMeta META_TABLE[534] = { #undef OPCODE }; -u8 NUM_ARGS[534] = { +u8 NUM_ARGS[538] = { #define OPCODE(name_token, type_token, ...) u8(CalculateNumArgsOf(Opcode::name_token)), #include "opcodes.inc" #undef OPCODE diff --git a/src/shader_recompiler/frontend/ir/opcodes.h b/src/shader_recompiler/frontend/ir/opcodes.h index 50f9d7b155..c59f50adfe 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.h +++ b/src/shader_recompiler/frontend/ir/opcodes.h @@ -57,12 +57,12 @@ static constexpr Type F64x2{Type::F64x2}; static constexpr Type F64x3{Type::F64x3}; static constexpr Type F64x4{Type::F64x4}; -extern OpcodeMeta META_TABLE[534]; +extern OpcodeMeta META_TABLE[538]; constexpr size_t CalculateNumArgsOf(Opcode op) noexcept { const auto& arg_types = META_TABLE[size_t(op)].arg_types; return size_t(std::distance(arg_types.begin(), std::ranges::find(arg_types, Type::Void))); } -extern u8 NUM_ARGS[534]; +extern u8 NUM_ARGS[538]; } // namespace Detail /// Get return type of an opcode diff --git a/src/shader_recompiler/frontend/ir/opcodes.inc b/src/shader_recompiler/frontend/ir/opcodes.inc index fee0afaf33..552ec4fbb6 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.inc +++ b/src/shader_recompiler/frontend/ir/opcodes.inc @@ -48,6 +48,10 @@ OPCODE(GetAttributeIndexed, F32, U32, OPCODE(SetAttributeIndexed, Void, U32, F32, U32, ) OPCODE(GetPatch, F32, Patch, ) OPCODE(SetPatch, Void, Patch, F32, ) +OPCODE(IsbeRead, U32, U32, ) +OPCODE(IsbeWriteAttribute, Void, U32, U32, ) +OPCODE(IsbeWriteMapU8, Void, U32, U32, ) +OPCODE(IsbeWriteMapU32, Void, U32, U32, ) OPCODE(SetFragColor, Void, U32, U32, F32, ) OPCODE(SetSampleMask, Void, U32, ) OPCODE(SetFragDepth, Void, F32, ) diff --git a/src/shader_recompiler/frontend/ir/program.h b/src/shader_recompiler/frontend/ir/program.h index 6b4a05c598..fee5cee721 100644 --- a/src/shader_recompiler/frontend/ir/program.h +++ b/src/shader_recompiler/frontend/ir/program.h @@ -27,6 +27,7 @@ struct Program { u32 local_memory_size{}; u32 shared_memory_size{}; bool is_geometry_passthrough{}; + MeshInfo mesh{}; }; [[nodiscard]] std::string DumpProgram(const Program& program); diff --git a/src/shader_recompiler/frontend/maxwell/maxwell.inc b/src/shader_recompiler/frontend/maxwell/maxwell.inc index e5dc9c8005..5facfb4803 100644 --- a/src/shader_recompiler/frontend/maxwell/maxwell.inc +++ b/src/shader_recompiler/frontend/maxwell/maxwell.inc @@ -55,6 +55,7 @@ INST(IMNMX_reg, "IMNMX (reg)", "0101 1100 0010 0---") INST(IMUL_cbuf, "IMUL (cbuf)", "0100 1100 0011 1---") INST(IMUL_reg, "IMUL (reg)", "0101 1100 0011 1---") INST(ISBERD, "ISBERD", "1110 1111 1101 0---") +INST(ISBE_nx2, "ISBE (nx2)", "1110 1111 1100 0---") INST(ISCADD_cbuf, "ISCADD (cbuf)", "0100 1100 0001 1---") INST(ISCADD_reg, "ISCADD (reg)", "0101 1100 0001 1---") INST(LDC, "LDC", "1110 1111 1001 0---") diff --git a/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp b/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp index 329d44443e..50cff52f88 100644 --- a/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp +++ b/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp @@ -314,7 +314,8 @@ bool NeedsLift(Node goto_stmt, Node label_stmt) noexcept { class GotoPass { public: - explicit GotoPass(Flow::CFG& cfg, ObjectPool<Statement>& stmt_pool) : pool{stmt_pool} { + explicit GotoPass(Flow::CFG& cfg, ObjectPool<Statement>& stmt_pool, bool converge_exits_) + : pool{stmt_pool}, converge_exits{converge_exits_} { std::vector gotos{BuildTree(cfg)}; const auto end{gotos.rend()}; for (auto goto_stmt = gotos.rbegin(); goto_stmt != end; ++goto_stmt) { @@ -381,7 +382,17 @@ private: u32 label_id{0}; std::vector<Node> gotos; Flow::Function& first_function{cfg.Functions().front()}; - BuildTree(cfg, first_function, label_id, gotos, root_stmt.children.end(), std::nullopt); + Node insert_point{root_stmt.children.end()}; + if (converge_exits) { + // Every EXIT jumps to one return at the end, so all invocations reach the same + // epilogue (task and mesh programs emit their outputs there, after a barrier). + Tree& root{root_stmt.children}; + exit_label = root.insert(root.end(), *pool.Create(Label{}, label_id, &root_stmt)); + ++label_id; + root.insert(root.end(), *pool.Create(Return{}, &root_stmt)); + insert_point = *exit_label; + } + BuildTree(cfg, first_function, label_id, gotos, insert_point, std::nullopt); return gotos; } @@ -454,7 +465,14 @@ private: break; } case Flow::EndClass::Exit: - root.insert(ip, *pool.Create(Return{}, &root_stmt)); + if (exit_label) { + Statement* const always_cond{ + pool.Create(Identity{}, IR::Condition{true}, &root_stmt)}; + gotos.push_back(root.insert( + ip, *pool.Create(Goto{}, always_cond, *exit_label, &root_stmt))); + } else { + root.insert(ip, *pool.Create(Return{}, &root_stmt)); + } break; case Flow::EndClass::Return: { Statement* const always_cond{pool.Create(Identity{}, block.cond, &root_stmt)}; @@ -618,6 +636,8 @@ private: ObjectPool<Statement>& pool; Statement root_stmt{FunctionTag{}}; + bool converge_exits{}; + std::optional<Node> exit_label; }; [[nodiscard]] Statement* TryFindForwardBlock(Statement& stmt) { @@ -987,7 +1007,8 @@ IR::AbstractSyntaxList BuildASL(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR:: Environment& env, Flow::CFG& cfg, const HostTranslateInfo& host_info) { ObjectPool<Statement> stmt_pool{64}; - GotoPass goto_pass{cfg, stmt_pool}; + const Stage stage{env.ShaderStage()}; + GotoPass goto_pass{cfg, stmt_pool, stage == Stage::Task || stage == Stage::Mesh}; Statement& root{goto_pass.RootStatement()}; IR::AbstractSyntaxList syntax_list; TranslatePass{inst_pool, block_pool, stmt_pool, env, root, syntax_list, host_info}; diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp index 46660563a1..27de2a0171 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp @@ -125,4 +125,40 @@ void TranslatorVisitor::ISBERD(u64 insn) { X(isberd.dest_reg.Value(), X(isberd.src_reg.Value())); } +// nx2 pseudo instruction for the ISBERD/ISBEWR of Switch 2 task and mesh programs (see +// frontend/ampere/lower_to_maxwell.h). Addresses are linear bytes in the stage buffer: reads come +// from the input attribute region, writes go to the output attribute or map region. +void TranslatorVisitor::ISBE_nx2(u64 insn) { + union { + u64 raw; + BitField<0, 8, IR::Reg> reg; + BitField<8, 8, IR::Reg> address_reg; + BitField<20, 24, u64> offset; + BitField<44, 1, u64> is_32; + BitField<45, 1, u64> attribute; + BitField<48, 1, u64> write; + } const isbe{insn}; + + const Stage stage{env.ShaderStage()}; + if (stage != Stage::Task && stage != Stage::Mesh) { + throw NotImplementedException("Stage buffer access outside task and mesh programs"); + } + const IR::U32 address{ir.IAdd(X(isbe.address_reg), ir.Imm32(static_cast<u32>(isbe.offset)))}; + if (isbe.write == 0) { + X(isbe.reg, ir.IsbeRead(address)); + return; + } + const IR::U32 value{X(isbe.reg)}; + if (isbe.attribute != 0) { + if (isbe.is_32 == 0) { + throw NotImplementedException("U8 stage buffer attribute write"); + } + ir.IsbeWriteAttribute(address, value); + } else if (isbe.is_32 != 0) { + ir.IsbeWriteMapU32(address, value); + } else { + ir.IsbeWriteMapU8(address, value); + } +} + } // namespace Shader::Maxwell diff --git a/src/shader_recompiler/frontend/maxwell/translate_program.cpp b/src/shader_recompiler/frontend/maxwell/translate_program.cpp index cd4678cef6..fe6d8d8913 100644 --- a/src/shader_recompiler/frontend/maxwell/translate_program.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate_program.cpp @@ -101,10 +101,12 @@ void AddNVNStorageBuffers(IR::Program& program) { switch (program.stage) { case Stage::VertexA: case Stage::VertexB: + case Stage::Task: // bound in the vertex slot return 0x110u; case Stage::TessellationControl: return 0x210u; case Stage::TessellationEval: + case Stage::Mesh: // bound in the tessellation slot return 0x310u; case Stage::Geometry: return 0x410u; @@ -304,6 +306,30 @@ void LowerGeometryPassthrough(const IR::Program& program, const HostTranslateInf } } +// The lines of a 32-element group in a mesh program's stage buffer: one per attribute an output +// map enables, in attribute order (system values, then generics). +void AddMeshLines(const ProgramHeader& sph, boost::container::static_vector<u32, 136>& lines) { + const auto& c{sph.vtg.omap_systemc}; + if (c.clip_distances != 0 || c.point_sprite_s != 0 || c.point_sprite_t != 0 || + c.fog_coordinate != 0 || c.tessellation_eval_point_u != 0 || + c.tessellation_eval_point_v != 0 || c.instance_id != 0 || c.vertex_id != 0) { + throw NotImplementedException("Mesh outputs past the generic attributes"); + } + for (u32 bit = 0; bit < 8; ++bit) { + if (((sph.vtg.omap_systemb.raw >> bit) & 1) != 0) { + lines.push_back(static_cast<u32>(IR::Attribute::PrimitiveId) + bit); + } + } + for (u32 generic = 0; generic < 32; ++generic) { + const std::array<bool, 4> mask{sph.vtg.OutputGeneric(generic)}; + for (u32 element = 0; element < 4; ++element) { + if (mask[element]) { + lines.push_back(static_cast<u32>(IR::Attribute::Generic0X) + generic * 4 + element); + } + } + } +} + void TightenOutputVertices(IR::Program& program) { if (program.stage != Stage::Geometry || program.is_geometry_passthrough) { return; @@ -368,6 +394,30 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo program.workgroup_size = env.WorkgroupSize(); program.shared_memory_size = env.SharedMemorySize(); break; + case Stage::Task: + case Stage::Mesh: { + // SET_MESH_SHADER_A: topology 3:0, max vertex 12:4, max primitive 22:13; + // SET_MESH_SHADER_B: mesh threads 25:20; SET_MESH_INIT_SHADER: task threads 5:0, local + // buffer lines 20:12, lines passed to the mesh stage 30:22 (lines are 128 bytes). + const auto& registers{env.MeshRegisters()}; + const u32 shader_a{registers[0]}, shader_b{registers[1]}, init{registers[2]}; + const u32 lines{(std::max)((init >> 12) & 0x1ff, (init >> 22) & 0x1ff)}; + program.mesh.payload_words = (std::max)(lines, 1U) * 32; + program.mesh.topology = shader_a & 0xf; + program.mesh.max_vertices = (shader_a >> 4) & 0x1ff; + program.mesh.max_primitives = (shader_a >> 13) & 0x3ff; + const u32 threads{program.stage == Stage::Task ? init & 0x3f : (shader_b >> 20) & 0x3f}; + program.workgroup_size = {(std::max)(threads, 1U), 1, 1}; + if (program.stage == Stage::Mesh) { + if (program.mesh.topology != 4) { + throw NotImplementedException("Mesh output topology {} (SET_MESH_SHADER_A {:#010x})", + program.mesh.topology, shader_a); + } + AddMeshLines(env.SPH(), program.mesh.vertex_lines); + AddMeshLines(env.MeshPrimitiveHeader(), program.mesh.primitive_lines); + } + break; + } default: break; } @@ -406,6 +456,18 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo Optimization::VerificationPass(program); } Optimization::CollectShaderInfoPass(env, program); + program.info.is_task = program.stage == Stage::Task; + program.info.is_mesh = program.stage == Stage::Mesh; + if (program.stage == Stage::Mesh) { + // Its outputs come from the stage buffer lines, not attribute stores; the next stage + // reads what they name. + for (const u32 line : program.mesh.vertex_lines) { + program.info.stores.Set(static_cast<IR::Attribute>(line), true); + } + for (const u32 line : program.mesh.primitive_lines) { + program.info.stores.Set(static_cast<IR::Attribute>(line), true); + } + } Optimization::LayerPass(program, normalized_host_info); Optimization::VendorWorkaroundPass(program); diff --git a/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp b/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp index 9010c2bf68..c212a25d37 100644 --- a/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp +++ b/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp @@ -963,10 +963,12 @@ void CollectShaderInfoPass(Environment& env, IR::Program& program) { switch (program.stage) { case Stage::VertexA: case Stage::VertexB: + case Stage::Task: // bound in the vertex slot return 0x110u; case Stage::TessellationControl: return 0x210u; case Stage::TessellationEval: + case Stage::Mesh: // bound in the tessellation slot return 0x310u; case Stage::Geometry: return 0x410u; @@ -978,6 +980,11 @@ void CollectShaderInfoPass(Environment& env, IR::Program& program) { throw InvalidArgument("Invalid stage {}", program.stage); }()}; info.nvn_buffer_base = base; + if (program.stage == Stage::Task || program.stage == Stage::Mesh) { + // The stage buffer reads the workgroup index; the epilogue splits work by invocation. + info.uses_workgroup_id = true; + info.uses_local_invocation_id = true; + } for (IR::Block* const block : program.post_order_blocks) { for (IR::Inst& inst : block->Instructions()) { diff --git a/src/shader_recompiler/ir_opt/texture_pass.cpp b/src/shader_recompiler/ir_opt/texture_pass.cpp index fae69ab91b..8a72f565dc 100644 --- a/src/shader_recompiler/ir_opt/texture_pass.cpp +++ b/src/shader_recompiler/ir_opt/texture_pass.cpp @@ -5,6 +5,10 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <queue> +#include <unordered_map> +#include <unordered_set> +#include <vector> #include <bit> #include <optional> #include "common/container/unordered_map.h" @@ -340,15 +344,20 @@ static inline u32 GetTextureHandleCached(Environment& env, const ConstBufferAddr } // Cached variants of existing helpers +// A handle computed per draw has no descriptor to read at translation time: assume an ordinary +// floating-point 2D texture, the instruction's own type deciding the rest. static inline TextureType ReadTextureTypeCached(Environment& env, const ConstBufferAddr& cbuf) { + if (cbuf.handle_program != ~0U) return TextureType::Color2D; return env.ReadTextureType(GetTextureHandleCached(env, cbuf)); } static inline TexturePixelFormat ReadTexturePixelFormatCached(Environment& env, const ConstBufferAddr& cbuf) { + if (cbuf.handle_program != ~0U) return TexturePixelFormat::A8B8G8R8_UNORM; return env.ReadTexturePixelFormat(GetTextureHandleCached(env, cbuf)); } static inline bool IsTexturePixelFormatIntegerCached(Environment& env, const ConstBufferAddr& cbuf) { + if (cbuf.handle_program != ~0U) return false; return env.IsTexturePixelFormatInteger(GetTextureHandleCached(env, cbuf)); } @@ -366,12 +375,389 @@ static inline std::optional<ConstBufferAddr> TrackCached(const IR::Value& v, Env std::optional<ConstBufferAddr> TryGetConstBuffer(const IR::Inst* inst, Environment& env, const HostTranslateInfo& host_info, const IR::Block* block); +bool IsMemoryLoad(IR::Opcode opcode) { + switch (opcode) { + case IR::Opcode::LoadGlobalU8: case IR::Opcode::LoadGlobalS8: case IR::Opcode::LoadGlobalU16: + case IR::Opcode::LoadGlobalS16: case IR::Opcode::LoadGlobal32: case IR::Opcode::LoadGlobal64: + case IR::Opcode::LoadGlobal128: case IR::Opcode::LoadStorageU8: case IR::Opcode::LoadStorageS8: + case IR::Opcode::LoadStorageU16: case IR::Opcode::LoadStorageS16: case IR::Opcode::LoadStorage32: + case IR::Opcode::LoadStorage64: case IR::Opcode::LoadStorage128: case IR::Opcode::LoadLocal: + case IR::Opcode::LoadSharedU8: case IR::Opcode::LoadSharedS8: case IR::Opcode::LoadSharedU16: + case IR::Opcode::LoadSharedS16: case IR::Opcode::LoadSharedU32: case IR::Opcode::LoadSharedU64: + case IR::Opcode::LoadSharedU128: + return true; + default: + return false; + } +} + std::optional<ConstBufferAddr> Track(const IR::Value& value, Environment& env, const HostTranslateInfo& host_info, const IR::Block* block) { - return IR::BreadthFirstSearch(value, [&env, &host_info, block](const IR::Inst* inst) { - return TryGetConstBuffer(inst, env, host_info, block); - }, [&](IR::Value operand) { return ResolveConditionalPhi(operand, block); }); + // A value loaded from memory is not a constant buffer word, whatever its address is built + // from: the search does not look into a load's operands (a handle read from a storage + // buffer would otherwise be taken from the buffer's address in the constant buffer). + const IR::Value start{ResolveConditionalPhi(value, block)}; + if (start.IsImmediate()) { + return std::nullopt; + } + std::unordered_set<const IR::Inst*> visited; + std::queue<const IR::Inst*> queue; + queue.push(start.InstRecursive()); + visited.insert(start.InstRecursive()); + while (!queue.empty()) { + const IR::Inst* const inst{queue.front()}; + queue.pop(); + if (const std::optional result = TryGetConstBuffer(inst, env, host_info, block)) { + return result; + } + if (IsMemoryLoad(inst->GetOpcode())) { + continue; + } + for (size_t arg = inst->NumArgs(); arg--;) { + const IR::Value arg_value{ResolveConditionalPhi(inst->Arg(arg), block)}; + if (arg_value.IsImmediate()) { + continue; + } + const IR::Inst* const arg_inst{arg_value.InstRecursive()}; + if (visited.insert(arg_inst).second) { + queue.push(arg_inst); + } + } + } + return std::nullopt; +} + +// Builds a handle program (shader_info.h) for a value computed from constant buffer and storage +// buffer words; false when the value depends on anything else. +thread_local IR::Opcode g_handle_failure{IR::Opcode::Void}; // the first operation not supported + +bool IsUndefined(IR::Opcode opcode) { + return opcode == IR::Opcode::UndefU1 || opcode == IR::Opcode::UndefU8 || + opcode == IR::Opcode::UndefU16 || opcode == IR::Opcode::UndefU32 || + opcode == IR::Opcode::UndefU64; +} + +// Values already emitted by the program being built, by the slot that keeps them, and how deep +// in nested BuildHandleProgram calls the build is. +thread_local std::unordered_map<const IR::Inst*, u32> g_handle_slots; +thread_local int g_handle_depth{}; + +bool BuildHandleNode(const IR::Value& value, std::vector<u32>& code, int& budget, + const IR::Block* block); + +bool BuildHandleProgram(const IR::Value& value, std::vector<u32>& code, int& budget, + const IR::Block* block) { + using Op = HandleOp; + if (value.IsImmediate()) { + return BuildHandleNode(value, code, budget, block); + } + const bool outermost = g_handle_depth++ == 0; + if (outermost) { + g_handle_slots.clear(); + } + const IR::Inst* const inst{value.InstRecursive()}; + bool result; + if (const auto it = g_handle_slots.find(inst); it != g_handle_slots.end()) { + if (it->second == ~0U) { + // Reached again while building it: a loop carries the value, which no single + // evaluation can give. + g_handle_failure = IR::Opcode::Phi; + result = false; + } else { + code.push_back(static_cast<u32>(Op::Load)); + code.push_back(it->second); + result = true; + } + } else { + g_handle_slots.emplace(inst, ~0U); + const size_t start = code.size(); + result = BuildHandleNode(value, code, budget, block); + g_handle_slots.erase(inst); + if (!result) { + code.resize(start); + } + if (result && inst->UseCount() > 1 && g_handle_slots.size() < 256) { + const u32 slot = static_cast<u32>( + std::ranges::count_if(g_handle_slots, [](const auto& entry) { return entry.second != ~0U; })); + g_handle_slots.emplace(inst, slot); + code.push_back(static_cast<u32>(Op::Tee)); + code.push_back(slot); + } + } + --g_handle_depth; + if (outermost) { + g_handle_slots.clear(); + } + return result; } +bool BuildHandleNode(const IR::Value& value, std::vector<u32>& code, int& budget, + const IR::Block* block) { + if (--budget < 0) { + return false; + } + using Op = HandleOp; + const auto emit = [&](Op op) { code.push_back(static_cast<u32>(op)); }; + if (value.IsImmediate()) { + emit(Op::Immediate); + code.push_back(value.Type() == IR::Type::U1 ? (value.U1() ? 1U : 0U) : value.U32()); + return true; + } + const IR::Inst* const inst{value.InstRecursive()}; + const auto build = [&](size_t arg) { return BuildHandleProgram(inst->Arg(arg), code, budget, block); }; + const auto binary = [&](Op op) { + if (!build(0) || !build(1)) return false; + emit(op); + return true; + }; + const auto unary = [&](Op op) { + if (!build(0)) return false; + emit(op); + return true; + }; + // A word of a constant buffer read or of a storage buffer load, at byte offset +extra. + const auto load_word = [&](const IR::Inst* source, u32 extra) { + const auto add_extra = [&] { + if (extra != 0) { + emit(Op::Immediate); + code.push_back(extra); + emit(Op::Add); + } + }; + switch (source->GetOpcode()) { + case IR::Opcode::GetCbufU32: + case IR::Opcode::GetCbufU32x2: { + const IR::Value index{source->Arg(0)}; + const IR::Value offset{source->Arg(1)}; + if (!index.IsImmediate()) return false; + if (offset.IsImmediate()) { + emit(Op::Cbuf); + code.push_back(index.U32()); + code.push_back(offset.U32() + extra); + return true; + } + if (!BuildHandleProgram(offset, code, budget, block)) return false; + add_extra(); + emit(Op::CbufDynamic); + code.push_back(index.U32()); + return true; + } + case IR::Opcode::LoadStorage32: + case IR::Opcode::LoadStorage64: + case IR::Opcode::LoadStorage128: { + const IR::Value binding{source->Arg(0)}; + if (!binding.IsImmediate()) return false; + if (!BuildHandleProgram(source->Arg(1), code, budget, block)) return false; + add_extra(); + emit(Op::Storage); + code.push_back(binding.U32()); + return true; + } + default: + return false; + } + }; + switch (inst->GetOpcode()) { + case IR::Opcode::Identity: + case IR::Opcode::ConditionRef: + case IR::Opcode::ShuffleIndex: // a value made uniform (R2UR): the same in every lane + case IR::Opcode::VoteAny: // of a condition on per-draw words only, the same in every lane + case IR::Opcode::VoteAll: + return build(0); + case IR::Opcode::GetCbufU32: + case IR::Opcode::LoadStorage32: + return load_word(inst, 0); + case IR::Opcode::CompositeExtractU32x2: { + const IR::Value element{inst->Arg(1)}; + if (!element.IsImmediate() || inst->Arg(0).IsImmediate()) return false; + return load_word(inst->Arg(0).InstRecursive(), element.U32() * 4); + } + case IR::Opcode::Phi: { + // Keep the incoming value that is defined; of two, the one made uniform (the + // register-to-uniform move a conditional block makes for the handle). + const IR::Value resolved{ResolveConditionalPhi(value, block)}; + if (resolved.IsImmediate() || resolved.InstRecursive() != inst) { + return BuildHandleProgram(resolved, code, budget, block); + } + std::optional<size_t> chosen; + for (size_t arg = 0; arg < inst->NumArgs(); ++arg) { + const IR::Value incoming{inst->Arg(arg)}; + if (!incoming.IsImmediate() && IsUndefined(incoming.InstRecursive()->GetOpcode())) { + continue; + } + IR::Value probe{incoming}; + bool shuffled = false; + for (int depth = 0; depth < 8 && !probe.IsImmediate(); ++depth) { + const IR::Inst* const probe_inst{probe.InstRecursive()}; + if (probe_inst->GetOpcode() == IR::Opcode::ShuffleIndex) { + shuffled = true; + break; + } + if (probe_inst->GetOpcode() != IR::Opcode::Identity) break; + probe = probe_inst->Arg(0); + } + if (!chosen || shuffled) chosen = arg; + } + if (chosen && build(*chosen)) { + return true; + } + for (size_t arg = 0; arg < inst->NumArgs(); ++arg) { + const IR::Value incoming{inst->Arg(arg)}; + if (arg == chosen || (!incoming.IsImmediate() && IsUndefined(incoming.InstRecursive()->GetOpcode()))) { + continue; + } + if (build(arg)) { + return true; + } + } + return false; + } + case IR::Opcode::IAdd32: return binary(Op::Add); + case IR::Opcode::ISub32: return binary(Op::Sub); + case IR::Opcode::IMul32: return binary(Op::Mul); + case IR::Opcode::BitwiseAnd32: return binary(Op::And); + case IR::Opcode::BitwiseOr32: return binary(Op::Or); + case IR::Opcode::BitwiseXor32: return binary(Op::Xor); + case IR::Opcode::ShiftLeftLogical32: return binary(Op::Shl); + case IR::Opcode::ShiftRightLogical32: return binary(Op::Shr); + case IR::Opcode::ShiftRightArithmetic32: return binary(Op::Sar); + case IR::Opcode::INeg32: return unary(Op::Neg); + case IR::Opcode::BitwiseNot32: return unary(Op::Not); + case IR::Opcode::UMin32: return binary(Op::UMin); + case IR::Opcode::UMax32: return binary(Op::UMax); + case IR::Opcode::SMin32: return binary(Op::SMin); + case IR::Opcode::SMax32: return binary(Op::SMax); + case IR::Opcode::SLessThan: return binary(Op::SLt); + case IR::Opcode::SLessThanEqual: return binary(Op::SLe); + case IR::Opcode::SGreaterThan: return binary(Op::SGt); + case IR::Opcode::SGreaterThanEqual: return binary(Op::SGe); + case IR::Opcode::ULessThan: return binary(Op::ULt); + case IR::Opcode::ULessThanEqual: return binary(Op::ULe); + case IR::Opcode::UGreaterThan: return binary(Op::UGt); + case IR::Opcode::UGreaterThanEqual: return binary(Op::UGe); + case IR::Opcode::IEqual: return binary(Op::Eq); + case IR::Opcode::INotEqual: return binary(Op::Ne); + case IR::Opcode::LogicalAnd: return binary(Op::LAnd); + case IR::Opcode::LogicalOr: return binary(Op::LOr); + case IR::Opcode::LogicalXor: return binary(Op::LXor); + case IR::Opcode::LogicalNot: return unary(Op::LNot); + case IR::Opcode::GetZeroFromOp: // the flag a producing operation sets on its result + if (!build(0)) return false; + emit(Op::Immediate); + code.push_back(0); + emit(Op::Eq); + return true; + case IR::Opcode::GetSignFromOp: + if (!build(0)) return false; + emit(Op::Immediate); + code.push_back(0); + emit(Op::SLt); + return true; + case IR::Opcode::GetCarryFromOp: { + // The carry of a 32-bit addition: the sum below an operand. + const IR::Value op{inst->Arg(0)}; + if (op.IsImmediate() || op.InstRecursive()->GetOpcode() != IR::Opcode::IAdd32) { + g_handle_failure = inst->GetOpcode(); + return false; + } + const IR::Inst* const add{op.InstRecursive()}; + if (!BuildHandleProgram(op, code, budget, block) || + !BuildHandleProgram(add->Arg(0), code, budget, block)) { + return false; + } + emit(Op::ULt); + return true; + } + case IR::Opcode::BitCastU32F32: + case IR::Opcode::BitCastF32U32: + return build(0); + case IR::Opcode::UndefU1: + case IR::Opcode::UndefU32: + // Read before any write on some path: no value to compute, take zero. + emit(Op::Immediate); + code.push_back(0); + return true; + case IR::Opcode::BitFieldUExtract: { + // (base >> offset) & ((1 << count) - 1), the shift by 32 giving the all-ones mask. + const auto mask = [&] { + emit(Op::Immediate); + code.push_back(1); + if (!build(2)) return false; + emit(Op::Shl); + emit(Op::Immediate); + code.push_back(1); + emit(Op::Sub); + return true; + }; + if (!build(0) || !build(1)) return false; + emit(Op::Shr); + if (!mask()) return false; + emit(Op::And); + return true; + } + case IR::Opcode::BitFieldSExtract: { + // (base << (32 - offset - count)) >> (32 - count), arithmetic. + if (!build(0)) return false; + emit(Op::Immediate); + code.push_back(32); + if (!build(1)) return false; + emit(Op::Sub); + if (!build(2)) return false; + emit(Op::Sub); + emit(Op::Shl); + emit(Op::Immediate); + code.push_back(32); + if (!build(2)) return false; + emit(Op::Sub); + emit(Op::Sar); + return true; + } + case IR::Opcode::BitFieldInsert: { + // (base & ~mask) | ((insert << offset) & mask), mask = ((1 << count) - 1) << offset. + const auto mask = [&] { + emit(Op::Immediate); + code.push_back(1); + if (!build(3)) return false; + emit(Op::Shl); + emit(Op::Immediate); + code.push_back(1); + emit(Op::Sub); + if (!build(2)) return false; + emit(Op::Shl); + return true; + }; + if (!build(0) || !mask()) return false; + emit(Op::Not); + emit(Op::And); + if (!build(1) || !build(2)) return false; + emit(Op::Shl); + if (!mask()) return false; + emit(Op::And); + emit(Op::Or); + return true; + } + case IR::Opcode::UClamp32: + case IR::Opcode::SClamp32: { + const bool is_signed = inst->GetOpcode() == IR::Opcode::SClamp32; + if (!build(0) || !build(1)) return false; + emit(is_signed ? Op::SMax : Op::UMax); + if (!build(2)) return false; + emit(is_signed ? Op::SMin : Op::UMin); + return true; + } + case IR::Opcode::SelectU1: + case IR::Opcode::SelectU32: + if (!build(0) || !build(1) || !build(2)) return false; + emit(Op::Select); + return true; + default: + g_handle_failure = inst->GetOpcode(); + return false; + } +} + +// Programs built while making the texture instructions of the pass running on this thread. +thread_local std::vector<std::vector<u32>>* g_handle_programs{}; + std::optional<u32> TryGetConstant(IR::Value& value, Environment& env) { const IR::Inst* inst = value.InstRecursive(); if (inst->GetOpcode() != IR::Opcode::GetCbufU32) { @@ -525,10 +911,44 @@ TextureInst MakeInst(Environment& env, IR::Block* block, IR::Inst& inst, const H if (IsBindless(inst)) { const std::optional<ConstBufferAddr> track_addr{TrackCached(inst.Arg(0), env, host_info, block)}; - if (!track_addr) { - throw NotImplementedException("Failed to track bindless texture constant buffer"); - } else { + if (track_addr) { addr = *track_addr; + } else { + // Read from a storage buffer (a handle table): computed for each draw. + std::vector<u32> code; + int budget = 4096; + g_handle_failure = IR::Opcode::Void; + if (!g_handle_programs || !BuildHandleProgram(inst.Arg(0), code, budget, block)) { + // Not computable for a whole draw (a handle that changes in a loop): keep the + // earlier search through loads, which at least gives the shader a texture. + const std::optional<ConstBufferAddr> legacy{IR::BreadthFirstSearch(inst.Arg(0), + [&env, &host_info, block](const IR::Inst* candidate) { + return TryGetConstBuffer(candidate, env, host_info, block); + }, [&](IR::Value operand) { return ResolveConditionalPhi(operand, block); })}; + if (!legacy) { + throw NotImplementedException("Failed to track bindless texture constant buffer (stopped at {}, budget {})", + g_handle_failure, budget); + } + return TextureInst{.cbuf = *legacy, .inst = &inst, .block = block}; + } + auto& programs{*g_handle_programs}; + auto it{std::ranges::find(programs, code)}; + if (it == programs.end()) { + programs.push_back(std::move(code)); + it = programs.end() - 1; + } + addr = ConstBufferAddr{ + .index = 0, + .offset = 0, + .shift_left = 0, + .secondary_index = 0, + .secondary_offset = 0, + .secondary_shift_left = 0, + .dynamic_offset = {}, + .count = 1, + .has_secondary = false, + .handle_program = static_cast<u32>(it - programs.begin()), + }; } } else { addr = ConstBufferAddr{ @@ -724,6 +1144,7 @@ void TexturePass(Environment& env, IR::Program& program, const HostTranslateInfo env.handle_cache.clear(); env.track_cache.clear(); TextureInstVector to_replace; + g_handle_programs = &program.info.handle_programs; for (IR::Block* const block : program.post_order_blocks) { env.track_cache.clear(); for (IR::Inst& inst : block->Instructions()) { @@ -881,6 +1302,7 @@ void TexturePass(Environment& env, IR::Program& program, const HostTranslateInfo .secondary_shift_left = cbuf.secondary_shift_left, .count = count, .size_shift = size_shift, + .handle_program = cbuf.handle_program, }); flags.is_integer.Assign(is_integer ? 1 : 0); } diff --git a/src/shader_recompiler/runtime_info.h b/src/shader_recompiler/runtime_info.h index e6e1284762..560afb3648 100644 --- a/src/shader_recompiler/runtime_info.h +++ b/src/shader_recompiler/runtime_info.h @@ -7,6 +7,7 @@ #pragma once #include <array> +#include <bitset> #include <map> #include <optional> #include <vector> @@ -85,6 +86,8 @@ struct TransformFeedbackVarying { struct RuntimeInfo { std::array<AttributeType, 32> generic_input_types{}; VaryingState previous_stage_stores; + /// Fragment stage after a mesh program: generic inputs it writes per primitive. + std::bitset<32> per_primitive_generics{}; std::map<IR::Attribute, IR::Attribute> previous_stage_legacy_stores_mapping; bool convert_depth_mode{}; diff --git a/src/shader_recompiler/shader_info.h b/src/shader_recompiler/shader_info.h index 71309e040e..eeb8d768d7 100644 --- a/src/shader_recompiler/shader_info.h +++ b/src/shader_recompiler/shader_info.h @@ -7,6 +7,9 @@ #pragma once #include <array> +#include <optional> +#include <span> +#include <vector> #include <bitset> #include <map> @@ -219,6 +222,9 @@ struct TextureDescriptor { u32 secondary_shift_left; u32 count; u32 size_shift; + /// Index in Info::handle_programs when the handle is computed at draw time (read from a + /// storage buffer), ~0 when it is the constant buffer word above. + u32 handle_program{~0U}; auto operator<=>(const TextureDescriptor&) const = default; }; @@ -239,11 +245,26 @@ struct ImageDescriptor { }; using ImageDescriptors = boost::container::small_vector<ImageDescriptor, 4>; +/// Switch 2 (NVN) task and mesh programs: the sizes their mesh registers set and, for mesh +/// programs, the attribute (IR::Attribute index) held by each 128-byte line of a 32-element group +/// in the emulated stage buffer: per-vertex lines from the program's output map, per-primitive +/// lines from the pass-through geometry header bound with it, in attribute order. +struct MeshInfo { + u32 payload_words{}; ///< Task output region / mesh input region, in 32-bit words + u32 max_vertices{}; + u32 max_primitives{}; + u32 topology{}; ///< SET_MESH_SHADER_A output topology (4 = triangles) + boost::container::static_vector<u32, 136> vertex_lines; + boost::container::static_vector<u32, 136> primitive_lines; +}; + struct Info { static constexpr size_t MAX_INDIRECT_CBUFS{14}; static constexpr size_t MAX_CBUFS{18}; static constexpr size_t MAX_SSBOS{32}; + bool is_task{}; ///< Switch 2 task program (Stage::Task) + bool is_mesh{}; ///< Switch 2 mesh program (Stage::Mesh) bool uses_workgroup_id{}; bool uses_local_invocation_id{}; bool uses_invocation_id{}; @@ -337,6 +358,9 @@ struct Info { TextureBufferDescriptors texture_buffer_descriptors; ImageBufferDescriptors image_buffer_descriptors; TextureDescriptors texture_descriptors; + /// Texture handles the shader reads from storage buffers, as programs over constant buffer + /// and storage buffer words (see EvaluateHandleProgram), evaluated for each draw. + std::vector<std::vector<u32>> handle_programs; ImageDescriptors image_descriptors; }; @@ -349,4 +373,111 @@ u32 NumDescriptors(const Descriptors& descriptors) { return num; } + +/// Operations of a handle program, a stack machine over 32-bit words. Operands follow the +/// operation code in the program; values are popped right operand first. +enum class HandleOp : u32 { + Immediate, ///< value + Cbuf, ///< index, offset: the constant buffer word + CbufDynamic, ///< index: the word at the popped byte offset + Storage, ///< binding: the storage buffer word at the popped byte offset + Add, Sub, Mul, And, Or, Xor, Shl, Shr, Sar, Neg, Not, + UMin, UMax, SMin, SMax, + SLt, SLe, SGt, SGe, ULt, ULe, UGt, UGe, Eq, Ne, + LAnd, LOr, LXor, LNot, + Select, ///< condition, true value, false value + Tee, ///< slot: keeps the top value in the slot (shared subexpressions) + Load, ///< slot: the value kept there +}; + +/// Runs a handle program; read_cbuf(index, offset) and read_storage(binding, offset) return the +/// words it reads. Returns nothing for a malformed program. +template <typename ReadCbuf, typename ReadStorage> +std::optional<u32> EvaluateHandleProgram(std::span<const u32> code, ReadCbuf&& read_cbuf, + ReadStorage&& read_storage) { + std::array<u32, 64> stack{}; + std::array<u32, 256> slots{}; + size_t size = 0; + const auto pop = [&]() -> u32 { return size > 0 ? stack[--size] : 0; }; + const auto push = [&](u32 value) { + if (size < stack.size()) stack[size++] = value; + }; + for (size_t pc = 0; pc < code.size();) { + const HandleOp op = static_cast<HandleOp>(code[pc++]); + const auto operand = [&]() -> u32 { return pc < code.size() ? code[pc++] : 0; }; + if (op == HandleOp::Immediate) { + push(operand()); + continue; + } + if (op == HandleOp::Cbuf) { + const u32 index = operand(); + push(read_cbuf(index, operand())); + continue; + } + if (op == HandleOp::CbufDynamic) { + const u32 index = operand(); + push(read_cbuf(index, pop())); + continue; + } + if (op == HandleOp::Storage) { + const u32 binding = operand(); + push(read_storage(binding, pop())); + continue; + } + if (op == HandleOp::Neg || op == HandleOp::Not || op == HandleOp::LNot) { + const u32 a = pop(); + push(op == HandleOp::Neg ? 0U - a : op == HandleOp::Not ? ~a : (a == 0 ? 1U : 0U)); + continue; + } + if (op == HandleOp::Tee) { + const u32 slot = operand(); + if (slot < slots.size() && size > 0) slots[slot] = stack[size - 1]; + continue; + } + if (op == HandleOp::Load) { + const u32 slot = operand(); + push(slot < slots.size() ? slots[slot] : 0U); + continue; + } + if (op == HandleOp::Select) { + const u32 false_value = pop(), true_value = pop(), condition = pop(); + push(condition != 0 ? true_value : false_value); + continue; + } + const u32 b = pop(), a = pop(); + const s32 sa = static_cast<s32>(a), sb = static_cast<s32>(b); + switch (op) { + case HandleOp::Add: push(a + b); break; + case HandleOp::Sub: push(a - b); break; + case HandleOp::Mul: push(a * b); break; + case HandleOp::And: push(a & b); break; + case HandleOp::Or: push(a | b); break; + case HandleOp::Xor: push(a ^ b); break; + case HandleOp::Shl: push(b >= 32 ? 0U : a << b); break; + case HandleOp::Shr: push(b >= 32 ? 0U : a >> b); break; + case HandleOp::Sar: push(static_cast<u32>(sa >> (b >= 32 ? 31 : b))); break; + case HandleOp::UMin: push(a < b ? a : b); break; + case HandleOp::UMax: push(a > b ? a : b); break; + case HandleOp::SMin: push(static_cast<u32>(sa < sb ? sa : sb)); break; + case HandleOp::SMax: push(static_cast<u32>(sa > sb ? sa : sb)); break; + case HandleOp::SLt: push(sa < sb); break; + case HandleOp::SLe: push(sa <= sb); break; + case HandleOp::SGt: push(sa > sb); break; + case HandleOp::SGe: push(sa >= sb); break; + case HandleOp::ULt: push(a < b); break; + case HandleOp::ULe: push(a <= b); break; + case HandleOp::UGt: push(a > b); break; + case HandleOp::UGe: push(a >= b); break; + case HandleOp::Eq: push(a == b); break; + case HandleOp::Ne: push(a != b); break; + case HandleOp::LAnd: push(a != 0 && b != 0); break; + case HandleOp::LOr: push(a != 0 || b != 0); break; + case HandleOp::LXor: push((a != 0) != (b != 0)); break; + default: return std::nullopt; + } + } + if (size != 1) return std::nullopt; + return stack[0]; +} + } // namespace Shader diff --git a/src/shader_recompiler/stage.h b/src/shader_recompiler/stage.h index 9af0a1b720..34172cfee2 100644 --- a/src/shader_recompiler/stage.h +++ b/src/shader_recompiler/stage.h @@ -17,6 +17,11 @@ enum class Stage : u32 { Compute, VertexA, + + // Switch 2 (NVN) task and mesh programs, bound in the vertex and tessellation slots while + // SET_MESH_CONTROL is enabled. + Task, + Mesh, }; constexpr u32 MaxStageTypes = 6; diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 5ed43fc903..b3ef88c5f1 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -235,6 +235,7 @@ add_library(video_core STATIC shader_cache.h shader_environment.cpp shader_environment.h + shader_handle_program.h shader_notify.cpp shader_notify.h smaa_area_tex.h diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index a3358048dc..53bbae7610 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -17,6 +17,7 @@ #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/guest_memory.h" #include "video_core/host1x/gpu_device_memory_manager.h" +#include "video_core/nx2.h" #include "video_core/nx2_profile.h" #include "video_core/texture_cache/util.h" @@ -100,6 +101,12 @@ void BufferCache<P>::TickFrame() { return; } runtime.TickFrame(slot_buffers); + if (frame_tick % 600 == 0 && !uploaded_buffer_ranges.empty()) { + for (const auto& [address, size] : uploaded_buffer_ranges) { + device_memory.HintCold(address, size); + } + uploaded_buffer_ranges.clear(); + } if (static u64 frames{}; ++frames % 600 == 0) { u64 total{}; size_t count{}; @@ -141,9 +148,10 @@ void BufferCache<P>::TickFrame() { // title 1's buffers were ~200 MiB of an 8 GB GPU's ~6.8 GB in use, yet this collector // ran aggressively every frame on the device-wide figure, evicting up to 64 buffers (and // downloading the GPU-written ones, ~5% of the GPU thread) that were recreated later. - // Collect when the buffers themselves are large. - if (buffer_bytes >= 1_GiB) { - RunGarbageCollector(buffer_bytes >= 2_GiB); + // Collect when the buffers themselves are large. At 1 GiB it held geometry of scenes + // long gone: title 1's title screen grew to ~0.9 GB of buffers in ten minutes. + if (buffer_bytes >= 512_MiB) { + RunGarbageCollector(buffer_bytes >= 1_GiB); } } else if (total_used_memory >= minimum_memory) { RunGarbageCollector(total_used_memory >= critical_memory); @@ -291,6 +299,9 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am const bool has_new_downloads = tmp_intervals.size() != 0; for (const auto& pair : tmp_intervals) { gpu_modified_ranges.Add(pair.first, pair.second); + if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { + Tegra::Nx2MarkGpuWritten(pair.first, pair.second); + } } const auto& copy = copies[0]; src_buffer.MarkUsage(copy.src_offset, copy.size); @@ -1647,6 +1658,9 @@ void BufferCache<P>::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 } else { ArmReadTraps(device_addr, size); } + if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { + Tegra::Nx2MarkGpuWritten(device_addr, size); + } } template <class P> @@ -1964,6 +1978,11 @@ void BufferCache<P>::MappedUploadMemory([[maybe_unused]] Buffer& buffer, u8* const src_pointer = staging_pointer.data() + copy.src_offset; const DAddr device_addr = buffer.CpuAddr() + copy.dst_offset; device_memory.ReadBlockUnsafe(device_addr, src_pointer, copy.size); + // Large uploads are mostly geometry the host GPU now has its own copy of: TickFrame + // marks their guest memory cold (see TextureCache::RefreshContents). + if (copy.size >= 256_KiB) { + uploaded_buffer_ranges.emplace_back(device_addr, copy.size); + } // Apply the staging offset copy.src_offset += upload_staging.offset; } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index d8a61b7913..47f342932d 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -580,6 +580,8 @@ private: size_t immediate_buffer_capacity = 0; Common::ScratchBuffer<u8> immediate_buffer_alloc; + /// Guest memory large uploads came from since the last cold hint (device address, size). + std::vector<std::pair<DAddr, u64>> uploaded_buffer_ranges; struct LRUItemParams { using ObjectType = BufferId; diff --git a/src/video_core/dirty_flags.cpp b/src/video_core/dirty_flags.cpp index e6856aaca3..2f359749aa 100644 --- a/src/video_core/dirty_flags.cpp +++ b/src/video_core/dirty_flags.cpp @@ -64,6 +64,8 @@ void SetupDirtyRenderTargets(Maxwell3D::DirtyState::Tables& tables) { void SetupDirtyShaders(Maxwell3D::DirtyState::Tables& tables) { FillBlock(tables[0], OFF(pipelines), NUM(pipelines[0]) * Maxwell3D::Regs::MaxShaderProgram, Shaders); + // Task and mesh programs are translated with the mesh registers. + FillBlock(tables[1], Maxwell3D::Regs::MeshControl, 4, Shaders); } } // Anonymous namespace diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 88d571ae5f..1144488c91 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -155,13 +155,31 @@ bool DmaPusher::Step() { // (the split only starts with unsafe reads; the settings can change mid-game). const bool use_safe = !VideoCommon::GpuSplit::Get().Active() && (Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe()); - if (use_safe) { + // The words as submitted, unless this is a resumed entry or the GPU has newer ones the + // safe read below would flush (with the split, the front can't tell: it reads memory + // unflushed anyway). Entries on pages the GPU writes (culling) always take the flushing + // read: their words at submit time are an older frame's. + const bool submitted = command_resume_offset == 0 && + dma_pushbuffer_subindex < command_list.submitted_offsets.size() && + !IsGpuWrittenCommands(dma_state.dma_get, + size_t{header.size} * sizeof(u32)) && + !(use_safe && memory_manager.IsMemoryDirty(dma_state.dma_get, + header.size * sizeof(u32))); + if (submitted) { + const std::span<const CommandHeader> commands{ + command_list.submitted_words.data() + + command_list.submitted_offsets[dma_pushbuffer_subindex], + header.size}; + RecordParsedEntry(header, commands); + command_resume_offset += ProcessCommands(commands, 0); + } else if (use_safe) { Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::SafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers); const std::span<const CommandHeader> commands{headers.data(), headers.size()}; RecordParsedEntry(header, commands); command_resume_offset += ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); } else { + FlushGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::UnsafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers); const std::span<const CommandHeader> commands{headers.data(), headers.size()}; RecordParsedEntry(header, commands); @@ -445,6 +463,34 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { } } +bool DmaPusher::IsGpuWrittenCommands(GPUVAddr address, size_t size) { + if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || size == 0) { + return false; + } + for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < address + size; page += 0x1000) { + if (const auto device = memory_manager.GpuToCpuAddress(page); + device && Tegra::Nx2IsGpuWritten(*device)) { + return true; + } + } + return false; +} + +void DmaPusher::FlushGpuWrittenCommands(GPUVAddr address, size_t size) { + // Title 1 runs small command lists its culling dispatches fill in each frame. Accurate GPU + // mode reads every command list through the caches (safe reads); here only the ones on pages + // the GPU has written are, or their draws run with an older frame's words (trees vanish). + if (!IsGpuWrittenCommands(address, size)) { + return; + } + // The caches belong to the split's back: it first runs what the front queued (the dispatch + // that writes these words among it). + VideoCommon::GpuSplit::Get().Drain(); + if (memory_manager.IsMemoryDirty(address, size, VideoCommon::CacheType::BufferCache)) { + memory_manager.FlushRegion(address, size, VideoCommon::CacheType::BufferCache); + } +} + void DmaPusher::ReportUnboundSubchannel() { // Methods sent to a subchannel with no engine (no SetObject yet) are dropped. static std::atomic<u32> reported{}; diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 82d29edb4b..e9dd5e2b6b 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -117,6 +117,10 @@ struct CommandList final { boost::container::small_vector<CommandHeader, 512> prefetch_command_list; /// EDEN_NX2_PB_CHECK: hash of each entry's words when it was submitted. std::vector<u64> submit_hashes; + /// The entries' words when they were submitted (NX2), each entry's at submitted_offsets[i]: + /// executed instead of the memory, which the game may already be refilling for a later frame. + std::vector<CommandHeader> submitted_words; + std::vector<u32> submitted_offsets; }; /// FNV-1a over pushbuffer words (EDEN_NX2_PB_CHECK). @@ -182,6 +186,10 @@ public: void DumpCurrentEntry(const char* reason); void CheckSubmitHash(const CommandListHeader& header, u64 submit_hash); void RecordParsedEntry(const CommandListHeader& header, std::span<const CommandHeader> commands); + /// Whether the entry lies on device pages the GPU has written (NX2). + bool IsGpuWrittenCommands(GPUVAddr address, size_t size); + /// Brings command list words the GPU wrote (still only in host buffers) to guest memory. + void FlushGpuWrittenCommands(GPUVAddr address, size_t size); private: diff --git a/src/video_core/engines/draw_manager.cpp b/src/video_core/engines/draw_manager.cpp index 5891c25622..038e5a2e69 100644 --- a/src/video_core/engines/draw_manager.cpp +++ b/src/video_core/engines/draw_manager.cpp @@ -80,6 +80,33 @@ void Maxwell3D::DrawManager::DrawDeferred(Maxwell3D& maxwell3d) { draw_state.instance_count = 0; } +bool Maxwell3D::DrawManager::IsInstanceLoopMethod(u32 method) noexcept { + switch (method) { + case MAXWELL3D_REG_INDEX(draw.begin): + case MAXWELL3D_REG_INDEX(draw.end): + case MAXWELL3D_REG_INDEX(vertex_buffer.first): + case MAXWELL3D_REG_INDEX(vertex_buffer.count): + case MAXWELL3D_REG_INDEX(index_buffer.first): + case MAXWELL3D_REG_INDEX(index_buffer.count): + case MAXWELL3D_REG_INDEX(index_buffer32_first): + case MAXWELL3D_REG_INDEX(index_buffer16_first): + case MAXWELL3D_REG_INDEX(index_buffer8_first): + case MAXWELL3D_REG_INDEX(index_buffer32_subsequent): + case MAXWELL3D_REG_INDEX(index_buffer16_subsequent): + case MAXWELL3D_REG_INDEX(index_buffer8_subsequent): + case MAXWELL3D_REG_INDEX(draw_inline_index): + case MAXWELL3D_REG_INDEX(inline_index_2x16.count): + case MAXWELL3D_REG_INDEX(inline_index_2x16.even): + case MAXWELL3D_REG_INDEX(inline_index_4x8.count): + case MAXWELL3D_REG_INDEX(inline_index_4x8.index0): + case MAXWELL3D_REG_INDEX(vertex_array_instance_first): + case MAXWELL3D_REG_INDEX(vertex_array_instance_subsequent): + return true; + default: + return false; + } +} + void Maxwell3D::DrawManager::DrawArray(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 vertex_first, u32 vertex_count, u32 base_instance, u32 num_instances) { draw_state.topology = topology; draw_state.vertex_buffer.first = vertex_first; diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index b8a6567907..4732167369 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -616,10 +616,12 @@ void KeplerCompute::RememberDlssMotionVectors() { // Title 4's upscaler reads the tonemapped frame through the texture handle at cbuf0+0x90 and // writes the output resolution image through the surface handle at cbuf0+0xa0 (from its -// TLD/SUST operands). Fire Emblem: Fortune's Weave's DLSS network ends with a pass that reads the -// input colour through cbuf0+0x328 and writes the DLSS output through cbuf0+0x348 (B10G11R11; 720p -// to 1440p docked). Blit one to the other with bilinear filtering when both are plain 2D images -// of the same format and the output is larger. +// TLD/SUST operands). Kirby Air Riders' reads a 2048x1152 image at cbuf0+0x80 and writes a +// 2560x1440 one at +0x90; Fire Emblem: Fortune's Weave's DLSS network ends with a pass that reads +// the input colour through cbuf0+0x328 and writes the DLSS output through cbuf0+0x348 (B10G11R11; +// 720p to 1440p docked). Blit one to the other with bilinear filtering when both are plain 2D +// images of the same format and the output is larger: title 4's slots first, then Fortune's +// Weave's, else the first two handles in cbuf0+0x80..0x11c that form such a pair. bool KeplerCompute::UpscaleInsteadOfNativeProgram() { const auto& cbuf = launch_description.const_buffer_config[0]; const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low; @@ -628,20 +630,38 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() { if (tic_index > regs.tic.limit) return std::nullopt; return GetTICEntry(tic_index); }; - const auto usable = [](const std::optional<Texture::TICEntry>& input, - const std::optional<Texture::TICEntry>& output) { - return input && output && input->format == output->format && input->Address() != 0 && - output->Address() != 0 && output->Width() > input->Width() && input->IsBlockLinear() && - output->IsBlockLinear(); + const auto upscale_pair = [](const std::optional<Texture::TICEntry>& from, + const std::optional<Texture::TICEntry>& to) { + return from && to && from->format == to->format && from->Address() != 0 && + to->Address() != 0 && to->Width() > from->Width() && from->IsBlockLinear() && + to->IsBlockLinear(); }; - auto input = image(0x90), output = image(0xa0); + std::optional<Texture::TICEntry> input = image(0x90), output = image(0xa0); bool fortunes_weave = false; - if (!usable(input, output)) { + if (!upscale_pair(input, output)) { input = image(0x328); output = image(0x348); - if (!usable(input, output)) return false; - fortunes_weave = true; + fortunes_weave = upscale_pair(input, output); + } + if (!upscale_pair(input, output)) { + input.reset(); + output.reset(); + for (u32 first = 0x80; first < 0x120 && !input; first += 4) { + const auto from = memory_manager.Read<u32>(cbuf_address + first) != 0 ? image(first) + : std::nullopt; + for (u32 second = first + 4; from && second < 0x120; second += 4) { + if (memory_manager.Read<u32>(cbuf_address + second) == 0) continue; + const auto to = image(second); + if (upscale_pair(from, to)) { + input = from; + output = to; + break; + } + } + } } + if (!input || !output) + return false; using RTFormat = Tegra::RenderTargetFormat; RTFormat format{}; switch (input->format) { diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 4001284127..9731f54584 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -338,6 +338,12 @@ void Maxwell3D::ConsumeSinkImpl(Core::System& system) { } void Maxwell3D::ProcessDirtyRegisters(u32 method, u32 argument) { + // The instances after the first are drawn together once their loop ends, and they must see + // the state they were begun with: NVN's draw macros reset the base instance (and its copy in + // c[0][0x644]) right after the loop, before the macro returns. + if (draw_manager.HasDeferredDraw() && !DrawManager::IsInstanceLoopMethod(method)) [[unlikely]] { + draw_manager.DrawDeferred(*this); + } regs.reg_array[method] = argument; for (auto const& table : dirty.tables) dirty.flags[table[method]] = true; @@ -529,6 +535,10 @@ void Maxwell3D::CallMultiMethod(Core::System& system, u32 method, const u32* bas ProcessMacro(system, method, base_start, amount, amount == methods_pending); return; } + // As in ProcessDirtyRegisters: the paths below write without it. + if (draw_manager.HasDeferredDraw() && !DrawManager::IsInstanceLoopMethod(method)) [[unlikely]] { + draw_manager.DrawDeferred(*this); + } switch (method) { case MAXWELL3D_REG_INDEX(const_buffer.buffer): case MAXWELL3D_REG_INDEX(const_buffer.buffer) + 1: diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index a32b31fd1e..af0e37978e 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -2541,6 +2541,19 @@ public: return IsShaderConfigEnabled(static_cast<std::size_t>(type)); } + // Switch 2 (NVN) task and mesh pipelines (clc597.h): SET_MESH_CONTROL, SET_MESH_SHADER_A + // and _B, SET_MESH_INIT_SHADER. While mesh control is on, the vertex slot holds the task + // program (or the mesh program when there is no task), the tessellation slot the mesh + // program and the geometry slot a pass-through header naming the per-primitive outputs. + static constexpr std::size_t MeshControl = 0x114c / 4; + static constexpr std::size_t MeshShaderA = 0x1150 / 4; + static constexpr std::size_t MeshShaderB = 0x1154 / 4; + static constexpr std::size_t MeshInitShader = 0x1158 / 4; + + bool IsMeshEnabled() const { + return (reg_array[MeshControl] & 1) != 0; + } + struct ConstantBuffer { u32 size; u32 address_high; @@ -3083,6 +3096,12 @@ public: void ProcessMethodCall(Maxwell3D& maxwell3d, u32 method, u32 argument); void Clear(Maxwell3D& maxwell3d, u32 layer_count); void DrawDeferred(Maxwell3D& maxwell3d); + /// Instances after the first wait to be drawn together (DrawDeferred). + [[nodiscard]] bool HasDeferredDraw() const noexcept { + return draw_state.draw_mode == DrawMode::Instance && draw_state.instance_count != 0; + } + /// Methods an instance loop writes between its draws; any other write ends the loop. + [[nodiscard]] static bool IsInstanceLoopMethod(u32 method) noexcept; void DrawArray(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 vertex_first, u32 vertex_count, u32 base_instance, u32 num_instances); void DrawArrayInstanced(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 vertex_first, u32 vertex_count, bool subsequent); void DrawIndex(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 index_first, u32 index_count, u32 base_index, u32 base_instance, u32 num_instances); diff --git a/src/video_core/engines/sw_blitter/blitter.cpp b/src/video_core/engines/sw_blitter/blitter.cpp index 8bcc2f7a76..78363bed11 100644 --- a/src/video_core/engines/sw_blitter/blitter.cpp +++ b/src/video_core/engines/sw_blitter/blitter.cpp @@ -5,6 +5,8 @@ #include <cmath> #include <vector> +#include "common/alignment.h" +#include "common/literals.h" #include "common/scratch_buffer.h" #include "video_core/engines/sw_blitter/blitter.h" #include "video_core/engines/sw_blitter/converter.h" @@ -23,6 +25,7 @@ using VideoCore::Surface::PixelFormatFromRenderTargetFormat; namespace Tegra::Engines::Blitter { using namespace Texture; +using namespace Common::Literals; namespace { @@ -125,6 +128,46 @@ void ProcessPitchLinear(std::span<const u8> input, std::span<u8> output, size_t } } +struct SubSurface { + u64 offset; ///< From the surface's address + size_t size; ///< Bytes + u32 height; ///< Lines the part has + u32 origin_y; ///< The blit's first line in it +}; + +// The part of a surface a blit covers: the rows of blocks (block linear) or the lines (pitch) from +// its first line to its last. A tile copied into a layer of a large texture used to read and write +// back the whole layer (64-256 MiB), which also gave every block of a Switch 2 sparse layer memory. +SubSurface CoveredRows(const Fermi2D::Surface& surface, u32 bytes_per_pixel, s32 x0, s32 y0, + u32 extent_x, u32 extent_y, size_t surface_size) { + const SubSurface whole{0, surface_size, surface.height, static_cast<u32>(y0)}; + if (x0 < 0 || y0 < 0 || extent_y == 0) { + return whole; + } + const u64 first_line = static_cast<u32>(y0); + if (surface.linear == Fermi2D::MemoryLayout::BlockLinear) { + if (surface.depth != 1) { + return whole; + } + const u32 block_height = surface.block_height; + const u32 block_depth = surface.block_depth; + const u32 lines_shift = GOB_SIZE_Y_SHIFT + block_height; + const u64 row_size = + u64{Common::AlignUpLog2(surface.width * bytes_per_pixel, GOB_SIZE_X_SHIFT)} + << (lines_shift + block_depth); + const u64 first_row = first_line >> lines_shift; + const u64 rows = ((first_line + extent_y - 1) >> lines_shift) - first_row + 1; + const SubSurface part{first_row * row_size, static_cast<size_t>(rows * row_size), + static_cast<u32>(rows << lines_shift), + static_cast<u32>(first_line - (first_row << lines_shift))}; + return part.offset + part.size <= surface_size ? part : whole; + } + const u64 size = (extent_y - 1) * u64{surface.pitch} + + (static_cast<u64>(x0) + extent_x) * bytes_per_pixel; + const SubSurface part{first_line * surface.pitch, static_cast<size_t>(size), extent_y, 0}; + return part.offset + part.size <= surface_size ? part : whole; +} + } // namespace struct SoftwareBlitEngine::BlitEngineImpl { @@ -133,6 +176,23 @@ struct SoftwareBlitEngine::BlitEngineImpl { Common::ScratchBuffer<u8> dst_buffer; Common::ScratchBuffer<f32> intermediate_src; Common::ScratchBuffer<f32> intermediate_dst; + + // Buffers grown for one large surface are freed after the blit rather than kept for the rest + // of the session; small ones stay. 16 MiB kept title 1's boot mip blits (64/16/4/1 MiB levels) + // in 16 such buffers, 256 MiB, across its 2D engines. + void ReleaseLargeBuffers() { + constexpr size_t LargeCapacity = 2_MiB; + for (auto* const buffer : {&tmp_buffer, &src_buffer, &dst_buffer}) { + if (buffer->capacity() > LargeCapacity) { + buffer->release(); + } + } + for (auto* const buffer : {&intermediate_src, &intermediate_dst}) { + if (buffer->capacity() * sizeof(f32) > LargeCapacity) { + buffer->release(); + } + } + } ConverterFactory converter_factory; }; @@ -161,9 +221,11 @@ bool SoftwareBlitEngine::Blit(Fermi2D::Surface& src, Fermi2D::Surface& dst, const auto src_bytes_per_pixel = BytesPerBlock(PixelFormatFromRenderTargetFormat(src.format)); const auto dst_bytes_per_pixel = BytesPerBlock(PixelFormatFromRenderTargetFormat(dst.format)); const size_t src_size = get_surface_size(src, src_bytes_per_pixel); + const SubSurface src_part = CoveredRows(src, src_bytes_per_pixel, config.src_x0, config.src_y0, + src_extent_x, src_extent_y, src_size); Tegra::Memory::GpuGuestMemory<u8, Tegra::Memory::GuestMemoryFlags::SafeRead> tmp_buffer( - memory_manager, src.Address(), src_size, &impl->tmp_buffer); + memory_manager, src.Address() + src_part.offset, src_part.size, &impl->tmp_buffer); const size_t src_copy_size = src_extent_x * src_extent_y * src_bytes_per_pixel; const size_t dst_copy_size = dst_extent_x * dst_extent_y * dst_bytes_per_pixel; @@ -202,12 +264,13 @@ bool SoftwareBlitEngine::Blit(Fermi2D::Surface& src, Fermi2D::Surface& dst, impl->dst_buffer.resize_destructive(dst_copy_size); if (src.linear == Fermi2D::MemoryLayout::BlockLinear) { - UnswizzleSubrect(impl->src_buffer, tmp_buffer, src_bytes_per_pixel, src.width, src.height, - src.depth, config.src_x0, config.src_y0, src_extent_x, src_extent_y, - src.block_height, src.block_depth, src_extent_x * src_bytes_per_pixel); + UnswizzleSubrect(impl->src_buffer, tmp_buffer, src_bytes_per_pixel, src.width, + src_part.height, src.depth, config.src_x0, src_part.origin_y, + src_extent_x, src_extent_y, src.block_height, src.block_depth, + src_extent_x * src_bytes_per_pixel); } else { ProcessPitchLinear<false>(tmp_buffer, impl->src_buffer, src_extent_x, src_extent_y, - src.pitch, config.src_x0, config.src_y0, src_bytes_per_pixel); + src.pitch, config.src_x0, src_part.origin_y, src_bytes_per_pixel); } // Conversion Phase @@ -222,18 +285,25 @@ bool SoftwareBlitEngine::Blit(Fermi2D::Surface& src, Fermi2D::Surface& dst, } const size_t dst_size = get_surface_size(dst, dst_bytes_per_pixel); - Tegra::Memory::GpuGuestMemoryScoped<u8, Tegra::Memory::GuestMemoryFlags::SafeReadWrite> - tmp_buffer2(memory_manager, dst.Address(), dst_size, &impl->tmp_buffer); - - if (dst.linear == Fermi2D::MemoryLayout::BlockLinear) { - SwizzleSubrect(tmp_buffer2, impl->dst_buffer, dst_bytes_per_pixel, dst.width, dst.height, - dst.depth, config.dst_x0, config.dst_y0, dst_extent_x, dst_extent_y, - dst.block_height, dst.block_depth, dst_extent_x * dst_bytes_per_pixel); - } else { - ProcessPitchLinear<true>(impl->dst_buffer, tmp_buffer2, dst_extent_x, dst_extent_y, - dst.pitch, config.dst_x0, config.dst_y0, - static_cast<size_t>(dst_bytes_per_pixel)); - } + const SubSurface dst_part = CoveredRows(dst, dst_bytes_per_pixel, config.dst_x0, config.dst_y0, + dst_extent_x, dst_extent_y, dst_size); + { + Tegra::Memory::GpuGuestMemoryScoped<u8, Tegra::Memory::GuestMemoryFlags::SafeReadWrite> + tmp_buffer2(memory_manager, dst.Address() + dst_part.offset, dst_part.size, + &impl->tmp_buffer); + + if (dst.linear == Fermi2D::MemoryLayout::BlockLinear) { + SwizzleSubrect(tmp_buffer2, impl->dst_buffer, dst_bytes_per_pixel, dst.width, + dst_part.height, dst.depth, config.dst_x0, dst_part.origin_y, + dst_extent_x, dst_extent_y, dst.block_height, dst.block_depth, + dst_extent_x * dst_bytes_per_pixel); + } else { + ProcessPitchLinear<true>(impl->dst_buffer, tmp_buffer2, dst_extent_x, dst_extent_y, + dst.pitch, config.dst_x0, dst_part.origin_y, + static_cast<size_t>(dst_bytes_per_pixel)); + } + } // tmp_buffer2 writes the destination back here + impl->ReleaseLargeBuffers(); return true; } diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index 61c9844cf2..d9f0ac2837 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -18,6 +18,7 @@ #include <queue> #include "common/common_types.h" +#include "common/logging.h" #include "common/settings.h" #include "common/thread.h" #include "video_core/delayed_destruction_ring.h" @@ -123,6 +124,13 @@ public: } void SignalSyncPoint(u32 value) { + // The id comes from a command list (the 3D engine's sync method, the puller's fence + // action). Past the syncpoint table, the increment would walk memory after it as an + // action list; command words read wrong carry any id (Splatoon Raiders on Linux). + if (value >= Tegra::Host1x::SyncpointManager::NUM_MAX_SYNCPOINTS) { + LOG_ERROR(HW_GPU, "Signal of syncpoint {} past the table, ignored", value); + return; + } syncpoint_manager.IncrementGuest(value); std::function<void()> func([this, value] { syncpoint_manager.IncrementHost(value); }); SignalFence(std::move(func)); diff --git a/src/video_core/host1x/syncpoint_manager.h b/src/video_core/host1x/syncpoint_manager.h index 847ed20c8d..d6e65af3eb 100644 --- a/src/video_core/host1x/syncpoint_manager.h +++ b/src/video_core/host1x/syncpoint_manager.h @@ -18,6 +18,8 @@ namespace Host1x { class SyncpointManager { public: + static constexpr size_t NUM_MAX_SYNCPOINTS = 192; + u32 GetGuestSyncpointValue(u32 id) const { return syncpoints_guest[id].load(std::memory_order_acquire); } @@ -78,8 +80,6 @@ private: void Wait(std::atomic<u32>& syncpoint, std::condition_variable& wait_cv, u32 expected_value); - static constexpr size_t NUM_MAX_SYNCPOINTS = 192; - std::array<std::atomic<u32>, NUM_MAX_SYNCPOINTS> syncpoints_guest{}; std::array<std::atomic<u32>, NUM_MAX_SYNCPOINTS> syncpoints_host{}; diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 9111ba10d2..ddfd2529fd 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -302,7 +302,7 @@ void MemoryManager::Write(GPUVAddr addr, T data) { return; } // A Switch 2 sparse block nobody wrote yet gets its memory now. - Tegra::Nx2BackSparseWrite(addr, sizeof(T)); + Tegra::Nx2BackSparseWrite(addr, sizeof(T), &data); if (auto page_pointer{GetPointer(addr)}; page_pointer) { std::memcpy(page_pointer, &data, sizeof(T)); return; @@ -431,11 +431,14 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: if (!unsafe) { rasterizer->FlushRegion(dev_addr_base, copy_amount, which); } - if (!IsBigPageContinuous(page_index)) [[unlikely]] { - memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount); - } else { - u8* physical = memory.GetPointer<u8>(dev_addr_base); + // A continuous big page may still have no host memory (a Switch 2 region backed as it is + // written, while its backing is pending): the per-page copy handles that. + u8* const physical = IsBigPageContinuous(page_index) ? memory.GetPointer<u8>(dev_addr_base) + : nullptr; + if (physical != nullptr) [[likely]] { std::memcpy(dest_buffer, physical, copy_amount); + } else { + memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount); } dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; }; @@ -466,7 +469,7 @@ void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, co void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which, bool unsafe) { // Give Switch 2 sparse pages the write lands on their memory first (otherwise it is dropped). - Tegra::Nx2BackSparseWrite(gpu_dest_addr, size); + Tegra::Nx2BackSparseWrite(gpu_dest_addr, size, src_buffer); auto just_advance = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; }; @@ -485,11 +488,12 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe if (!unsafe) { rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); } - if (!IsBigPageContinuous(page_index)) [[unlikely]] { - memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount); - } else { - u8* physical = memory.GetPointer<u8>(dev_addr_base); + u8* const physical = IsBigPageContinuous(page_index) ? memory.GetPointer<u8>(dev_addr_base) + : nullptr; + if (physical != nullptr) [[likely]] { std::memcpy(physical, src_buffer, copy_amount); + } else { + memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount); } src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; }; diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 318607384b..db8c92adfd 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -20,6 +20,44 @@ namespace Tegra { // Ampere classes and the Turing+ macro ISA, which Eden's Maxwell engines don't implement. inline std::atomic<bool> g_nx2_ampere{false}; +// Device pages the GPU has written through the buffer cache (shader stores, copies), ever. Title 1 +// runs command lists the GPU writes each frame (20-byte entries its culling fills in); read +// without a flush they are stale and the draws behind them (trees) go missing. The bits never +// clear: the dispatch that rewrites a page may still be queued on the split's back when its front +// reads the command list. One bit per 4 KiB of the 36-bit device address space (2 MiB). +inline constexpr u64 Nx2WrittenPageBits = 12; +inline std::array<std::atomic<u64>, (u64{1} << (36 - Nx2WrittenPageBits)) / 64> g_nx2_gpu_written_pages{}; + +inline void Nx2MarkGpuWritten(u64 device_address, u64 size) { + if (size == 0) { + return; + } + const u64 last = std::min<u64>((device_address + size - 1) >> Nx2WrittenPageBits, + g_nx2_gpu_written_pages.size() * 64 - 1); + for (u64 page = device_address >> Nx2WrittenPageBits; page <= last; ++page) { + auto& word = g_nx2_gpu_written_pages[page / 64]; + const u64 bit = u64{1} << (page % 64); + if ((word.load(std::memory_order_relaxed) & bit) == 0) { + word.fetch_or(bit, std::memory_order_relaxed); + } + } +} + +/// A new emulation session starts: no page has been written by its GPU yet. +inline void Nx2ClearGpuWrittenPages() { + for (auto& word : g_nx2_gpu_written_pages) { + word.store(0, std::memory_order_relaxed); + } +} + +[[nodiscard]] inline bool Nx2IsGpuWritten(u64 device_address) { + const u64 page = device_address >> Nx2WrittenPageBits; + if (page >= g_nx2_gpu_written_pages.size() * 64) { + return false; + } + return (g_nx2_gpu_written_pages[page / 64].load(std::memory_order_relaxed) >> (page % 64)) & 1; +} + // NVN2 queues release their progress values into the second page of a two-page GPU-accessible // region and GPU acquires read the same offsets in the first page (the CPU polls the second). // Something on the console copies one into the other; until that is known, the puller mirrors @@ -59,8 +97,9 @@ inline void Nx2ClearSemaphoreRegions() { // Switch 2 sparse regions (nx2:gpu 214, word 7 bit 0x200: title 1's terrain texture arrays and // layer pools, 170 MB-5.8 GB each, of which the game fills a little) get memory per block when the // GPU first writes there, not in full. nx2:gpu installs the handler and the span its regions cover; -// the GPU memory manager calls Nx2BackSparseWrite before each write. -using Nx2SparseBackingHandler = std::function<void(u64 gpu_address, u64 size)>; +// the GPU memory manager calls Nx2BackSparseWrite before each write, with the bytes it writes +// (blocks nobody wrote that only get zeros need no memory: they read zero already). +using Nx2SparseBackingHandler = std::function<void(u64 gpu_address, u64 size, const u8* data)>; inline std::recursive_mutex g_nx2_sparse_backing_mutex; inline Nx2SparseBackingHandler g_nx2_sparse_backing; inline std::atomic<u64> g_nx2_sparse_low{~u64{0}}; @@ -84,14 +123,14 @@ inline void Nx2AddSparseSpan(u64 gpu_address, u64 size) { } } -inline void Nx2BackSparseWrite(u64 gpu_address, u64 size) { +inline void Nx2BackSparseWrite(u64 gpu_address, u64 size, const void* data) { if (gpu_address + size <= g_nx2_sparse_low.load(std::memory_order_relaxed) || gpu_address >= g_nx2_sparse_high.load(std::memory_order_relaxed)) [[likely]] { return; } std::scoped_lock lk{g_nx2_sparse_backing_mutex}; if (g_nx2_sparse_backing) { - g_nx2_sparse_backing(gpu_address, size); + g_nx2_sparse_backing(gpu_address, size, static_cast<const u8*>(data)); } } @@ -136,6 +175,15 @@ inline void Nx2SparseBlockWritten(u64 region_address, u64 block) { } } +/// True unless the block of the region at region_address is known to hold nothing (no write from +/// the CPU or the GPU yet). +inline bool Nx2SparseBlockIsWritten(u64 region_address, u64 block) { + std::scoped_lock lk{g_nx2_sparse_state_mutex}; + const auto it = g_nx2_sparse_state.find(region_address); + return it == g_nx2_sparse_state.end() || block >= it->second.written.size() || + it->second.written[block]; +} + /// The game maps its own memory over [offset, +size) of the region at region_address (mapped), or /// unmaps it there. inline void Nx2SparseTilesMapped(u64 region_address, u64 offset, u64 size, bool mapped) { diff --git a/src/video_core/renderer_opengl/gl_compute_pipeline.cpp b/src/video_core/renderer_opengl/gl_compute_pipeline.cpp index f0a7baf9aa..eb134e86b5 100644 --- a/src/video_core/renderer_opengl/gl_compute_pipeline.cpp +++ b/src/video_core/renderer_opengl/gl_compute_pipeline.cpp @@ -12,6 +12,7 @@ #include "video_core/renderer_opengl/gl_compute_pipeline.h" #include "video_core/renderer_opengl/gl_shader_manager.h" #include "video_core/renderer_opengl/gl_shader_util.h" +#include "video_core/shader_handle_program.h" namespace OpenGL { @@ -105,6 +106,20 @@ void ComputePipeline::Configure() { const auto& cbufs{qmd.const_buffer_config}; const bool via_header_index{qmd.linked_tsc != 0}; const auto read_handle{[&](const auto& desc, u32 index) { + if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this dispatch. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, *gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && + ((qmd.const_buffer_enable_mask >> cbuf) & 1) != 0 + ? cbufs[cbuf].Address() + : GPUVAddr{}; + }), + via_header_index); + } + } ASSERT(((qmd.const_buffer_enable_mask >> desc.cbuf_index) & 1) != 0); const u32 index_offset{index << desc.size_shift}; const u32 offset{desc.cbuf_offset + index_offset}; diff --git a/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp b/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp index ecd6544b10..81950279d5 100644 --- a/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp +++ b/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp @@ -17,6 +17,7 @@ #include "video_core/renderer_opengl/gl_shader_manager.h" #include "video_core/renderer_opengl/gl_shader_util.h" #include "video_core/renderer_opengl/gl_state_tracker.h" +#include "video_core/shader_handle_program.h" #include "video_core/shader_notify.h" #include "video_core/texture_cache/texture_cache.h" @@ -308,6 +309,19 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { } const auto& cbufs{maxwell3d->state.shader_stages[stage].const_buffers}; const auto read_handle{[&](const auto& desc, u32 index) { + if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this draw. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, *gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && cbufs[cbuf].enabled + ? cbufs[cbuf].address + : GPUVAddr{}; + }), + via_header_index); + } + } ASSERT(cbufs[desc.cbuf_index].enabled); const u32 index_offset{index << desc.size_shift}; const u32 offset{desc.cbuf_offset + index_offset}; diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index 277cf222fa..ac750ab747 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -578,7 +578,8 @@ std::unique_ptr<ComputePipeline> ShaderCache::CreateComputePipeline( const ComputePipelineKey& key, const VideoCommon::ShaderInfo* shader) { const GPUVAddr program_base{kepler_compute->regs.code_loc.Address()}; const auto& qmd{kepler_compute->launch_description}; - ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start}; + ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start, + shader->lowered_native}; env.SetCachedSize(shader->size_bytes); main_pools.ReleaseContents(); diff --git a/src/video_core/renderer_vulkan/maxwell_to_vk.cpp b/src/video_core/renderer_vulkan/maxwell_to_vk.cpp index c8e43d0650..e03e5786fe 100644 --- a/src/video_core/renderer_vulkan/maxwell_to_vk.cpp +++ b/src/video_core/renderer_vulkan/maxwell_to_vk.cpp @@ -333,6 +333,10 @@ VkShaderStageFlagBits ShaderStage(Shader::Stage stage) { return VK_SHADER_STAGE_FRAGMENT_BIT; case Shader::Stage::Compute: return VK_SHADER_STAGE_COMPUTE_BIT; + case Shader::Stage::Task: + return VK_SHADER_STAGE_TASK_BIT_EXT; + case Shader::Stage::Mesh: + return VK_SHADER_STAGE_MESH_BIT_EXT; } UNIMPLEMENTED_MSG("Unimplemented shader stage={}", stage); return {}; diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp index 29356cfcec..1cfe391ba6 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp @@ -18,6 +18,7 @@ #include "video_core/renderer_vulkan/vk_pipeline_cache.h" #include "video_core/renderer_vulkan/vk_scheduler.h" #include "video_core/renderer_vulkan/vk_update_descriptor.h" +#include "video_core/shader_handle_program.h" #include "video_core/shader_notify.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -134,6 +135,9 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk if (pipeline_statistics) { pipeline_statistics->Collect(device, *pipeline); } + // Nothing reads the module once the pipeline exists; the driver keeps its SPIR-V while it + // lives. + spv_module.reset(); std::scoped_lock lock{build_mutex}; is_built = true; build_condvar.notify_one(); @@ -172,6 +176,20 @@ bool ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, const auto& cbufs{qmd.const_buffer_config}; const bool via_header_index{qmd.linked_tsc != 0}; const auto read_handle{[&](const auto& desc, u32 index) { + if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this dispatch. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && + ((qmd.const_buffer_enable_mask >> cbuf) & 1) != 0 + ? cbufs[cbuf].Address() + : GPUVAddr{}; + }), + via_header_index); + } + } ASSERT(((qmd.const_buffer_enable_mask >> desc.cbuf_index) & 1) != 0); const u32 index_offset{index << desc.size_shift}; const u32 offset{desc.cbuf_offset + index_offset}; diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 94bace432c..fa273f6ef9 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -24,6 +24,7 @@ #include "video_core/renderer_vulkan/vk_scheduler.h" #include "video_core/renderer_vulkan/vk_texture_cache.h" #include "video_core/renderer_vulkan/vk_update_descriptor.h" +#include "video_core/shader_handle_program.h" #include "video_core/shader_notify.h" #include "video_core/texture_cache/texture_cache.h" #include "video_core/vulkan_common/vulkan_device.h" @@ -54,6 +55,17 @@ using VideoCore::Surface::PixelFormatFromRenderTargetFormat; constexpr size_t NUM_STAGES = Maxwell::MaxShaderStage; +// The stage of a program: task and mesh programs sit in the vertex and tessellation slots. +VkShaderStageFlagBits StageFlag(const Shader::Info& info, VkShaderStageFlagBits slot_stage) { + if (info.is_task) { + return VK_SHADER_STAGE_TASK_BIT_EXT; + } + if (info.is_mesh) { + return VK_SHADER_STAGE_MESH_BIT_EXT; + } + return slot_stage; +} + DescriptorLayoutBuilder MakeBuilder(const Device& device, std::span<const Shader::Info> infos) { DescriptorLayoutBuilder builder{device}; for (size_t index = 0; index < infos.size(); ++index) { @@ -64,7 +76,7 @@ DescriptorLayoutBuilder MakeBuilder(const Device& device, std::span<const Shader VK_SHADER_STAGE_GEOMETRY_BIT, VK_SHADER_STAGE_FRAGMENT_BIT, }; - builder.Add(infos[index], stages.at(index)); + builder.Add(infos[index], StageFlag(infos[index], stages.at(index))); } return builder; } @@ -330,6 +342,11 @@ GraphicsPipeline::GraphicsPipeline( if (pipeline_statistics) { pipeline_statistics->Collect(device, *pipeline); } + // Nothing reads the shader modules once the pipeline exists, and the driver keeps a copy + // of each one's SPIR-V while it lives (title 1: ~1,500 pipelines, 2-5 modules each). + for (vk::ShaderModule& module : spv_modules) { + module.reset(); + } std::scoped_lock lock{build_mutex}; is_built = true; @@ -338,12 +355,13 @@ GraphicsPipeline::GraphicsPipeline( shader_notify->MarkShaderComplete(); } }}; + // Before the build: it frees the modules this looks at. + configure_func = ConfigureFunc(spv_modules, stage_infos); if (worker_thread) { worker_thread->QueueWork(std::move(func)); } else { func(); } - configure_func = ConfigureFunc(spv_modules, stage_infos); } void GraphicsPipeline::AddTransition(GraphicsPipeline* transition) { @@ -382,6 +400,19 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { } const auto& cbufs{maxwell3d->state.shader_stages[stage].const_buffers}; const auto read_handle{[&](const auto& desc, u32 index) { + if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this draw. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, *gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && cbufs[cbuf].enabled + ? cbufs[cbuf].address + : GPUVAddr{}; + }), + via_header_index); + } + } ASSERT(cbufs[desc.cbuf_index].enabled); const u32 index_offset{index << desc.size_shift}; const u32 offset{desc.cbuf_offset + index_offset}; @@ -1073,6 +1104,19 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { } } + const bool is_mesh{IsMeshPipeline()}; + if (is_mesh) { + // Mesh pipelines have no vertex input or input assembly. + dynamic_states.erase(std::remove_if(dynamic_states.begin(), dynamic_states.end(), + [](VkDynamicState state) { + return state == VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY_EXT || + state == VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE_EXT || + state == VK_DYNAMIC_STATE_VERTEX_INPUT_EXT || + state == VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE_EXT || + state == VK_DYNAMIC_STATE_PATCH_CONTROL_POINTS_EXT; + }), + dynamic_states.end()); + } const VkPipelineDynamicStateCreateInfo dynamic_state_ci{ .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO, .pNext = nullptr, @@ -1095,7 +1139,8 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = nullptr, .flags = 0, - .stage = MaxwellToVK::ShaderStage(Shader::StageFromIndex(stage)), + .stage = StageFlag(stage_infos[stage], + MaxwellToVK::ShaderStage(Shader::StageFromIndex(stage))), .module = *spv_modules[stage], .pName = "main", .pSpecializationInfo = nullptr, @@ -1115,9 +1160,9 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { .flags = flags, .stageCount = static_cast<u32>(shader_stages.size()), .pStages = shader_stages.data(), - .pVertexInputState = &vertex_input_ci, - .pInputAssemblyState = &input_assembly_ci, - .pTessellationState = &tessellation_ci, + .pVertexInputState = is_mesh ? nullptr : &vertex_input_ci, + .pInputAssemblyState = is_mesh ? nullptr : &input_assembly_ci, + .pTessellationState = is_mesh ? nullptr : &tessellation_ci, .pViewportState = &viewport_ci, .pRasterizationState = &rasterization_ci, .pMultisampleState = &multisample_ci, diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h index b1ac5a3fcc..6d6e2ab12d 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h @@ -98,6 +98,11 @@ public: bool UsesExtendedDynamicState() const noexcept { return key.state.extended_dynamic_state != 0; } + + /// Switch 2 task/mesh pipeline: drawn with vkCmdDrawMeshTasksEXT. + [[nodiscard]] bool IsMeshPipeline() const noexcept { + return stage_infos[0].is_task || stage_infos[0].is_mesh; + } GraphicsPipeline& operator=(GraphicsPipeline&&) noexcept = delete; GraphicsPipeline(GraphicsPipeline&&) noexcept = delete; @@ -119,6 +124,21 @@ public: : nullptr; } + /// The frame the pipeline cache last handed this pipeline out (evicts long unused ones). + void MarkUsed(u64 frame) noexcept { + last_use_frame = frame; + } + + [[nodiscard]] u64 LastUse() const noexcept { + return last_use_frame; + } + + /// Forgets the transitions, which may point to pipelines the cache evicted. + void ClearTransitions() noexcept { + transition_keys.clear(); + transitions.clear(); + } + [[nodiscard]] bool IsBuilt() const noexcept { return is_built.load(std::memory_order::relaxed); } @@ -159,6 +179,7 @@ private: std::vector<GraphicsPipelineCacheKey> transition_keys; std::vector<GraphicsPipeline*> transitions; + u64 last_use_frame{}; std::array<vk::ShaderModule, NUM_STAGES> spv_modules; diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index 05511c10e1..ff8e84b019 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -170,6 +170,14 @@ Shader::RuntimeInfo MakeRuntimeInfo(std::span<const Shader::IR::Program> program } else { info.previous_stage_stores.mask.set(); } + if (previous_program && previous_program->stage == Shader::Stage::Mesh) { + for (const u32 line : previous_program->mesh.primitive_lines) { + const auto attribute{static_cast<Shader::IR::Attribute>(line)}; + if (Shader::IR::IsGeneric(attribute)) { + info.per_primitive_generics.set(Shader::IR::GenericAttributeIndex(attribute)); + } + } + } const Shader::Stage stage{program.stage}; const bool has_geometry{key.unique_hashes[4] != 0 && !programs[4].is_geometry_passthrough}; const bool gl_ndc{key.state.ndc_minus_one_to_one != 0}; @@ -230,6 +238,9 @@ Shader::RuntimeInfo MakeRuntimeInfo(std::span<const Shader::IR::Program> program return Shader::TessSpacing::Equal; }(); break; + case Shader::Stage::Mesh: + info.convert_depth_mode = gl_ndc; + break; case Shader::Stage::Geometry: if (program.output_topology == Shader::OutputTopology::PointList) { info.fixed_state_point_size = point_size; @@ -345,6 +356,22 @@ bool GraphicsPipelineCacheKey::operator==(const GraphicsPipelineCacheKey& rhs) c return std::memcmp(&rhs, this, Size()) == 0; } +namespace { +// NVIDIA's and AMD's drivers keep a disk cache of built pipelines of their own. Eden's copy on top +// only kept every pipeline the title ever built in host memory (title 1: a 245 MB file, ~0.35 GB +// of RAM for the session) and copied it whole at each save, which stalled frames; pipelines built +// no slower without it. +bool DriverCachesPipelines(const Device& device) { + switch (device.GetDriverID()) { + case VK_DRIVER_ID_NVIDIA_PROPRIETARY: + case VK_DRIVER_ID_AMD_PROPRIETARY: + return true; + default: + return false; + } +} +} // Anonymous namespace + PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, @@ -358,7 +385,8 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, render_pass_cache{render_pass_cache_}, buffer_cache{buffer_cache_}, texture_cache{texture_cache_}, shader_notify{shader_notify_}, use_asynchronous_shaders{Settings::values.use_asynchronous_shaders.GetValue()}, - use_vulkan_pipeline_cache{Settings::values.use_vulkan_driver_pipeline_cache.GetValue()}, + use_vulkan_pipeline_cache{Settings::values.use_vulkan_driver_pipeline_cache.GetValue() && + !DriverCachesPipelines(device_)}, workers(device.HasBrokenParallelShaderCompiling() ? 1ULL : GetTotalPipelineWorkers(), "VkPipelineBuilder", {}, Common::ThreadPlacement::Background), serialization_thread(1, "VkPipelineSerialization", {}, @@ -378,7 +406,9 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, Shader::Stage::TessellationEval) | subgroup_stage_bit(VK_SHADER_STAGE_GEOMETRY_BIT, Shader::Stage::Geometry) | subgroup_stage_bit(VK_SHADER_STAGE_FRAGMENT_BIT, Shader::Stage::Fragment) | - subgroup_stage_bit(VK_SHADER_STAGE_COMPUTE_BIT, Shader::Stage::Compute)}; + subgroup_stage_bit(VK_SHADER_STAGE_COMPUTE_BIT, Shader::Stage::Compute) | + subgroup_stage_bit(VK_SHADER_STAGE_TASK_BIT_EXT, Shader::Stage::Task) | + subgroup_stage_bit(VK_SHADER_STAGE_MESH_BIT_EXT, Shader::Stage::Mesh)}; if (driver_id == VK_DRIVER_ID_MESA_TURNIP) { supported_subgroup_stages &= ~(1u << static_cast<u32>(Shader::Stage::Geometry)); } @@ -563,13 +593,96 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, device.SupportsTransformFeedbackProvokingVertexPreservation(); } +namespace { +// <title>/vulkan_used.bin: the graphics pipelines recent sessions used, as key hash + sessions +// since the last use. +constexpr u32 UsedPipelinesMagic = 0x31505545; // "EUP1" +// Pipelines the last two sessions used are built at boot. Each holds ~0.45 MB of driver memory: +// title 1 built 4,800 from eight sessions, 1.5 GB more than building them on first use (but that +// stalled frames 4x as often), while one session of its title screen uses ~1,500. +constexpr u8 UsedPipelinesMaxAge = 2; + +std::unordered_map<u64, u8> LoadUsedPipelines(const std::filesystem::path& path) { + std::unordered_map<u64, u8> used; + std::ifstream file{path, std::ios::binary}; + u32 magic{}, count{}; + if (!file.read(reinterpret_cast<char*>(&magic), sizeof(magic)) || magic != UsedPipelinesMagic || + !file.read(reinterpret_cast<char*>(&count), sizeof(count))) { + return used; + } + used.reserve(count); + for (u32 i = 0; i < count; ++i) { + u64 hash{}; + u8 age{}; + if (!file.read(reinterpret_cast<char*>(&hash), sizeof(hash)) || + !file.read(reinterpret_cast<char*>(&age), sizeof(age))) { + break; + } + if (age < UsedPipelinesMaxAge) { // lists saved with a longer window + used.emplace(hash, age); + } + } + return used; +} + +// Without a list of the pipelines recent sessions used (the first boot with this cache, or a session +// that ended before saving it), this many cached graphics pipelines at most are built at boot and the +// rest when first needed. A cache that grew over many builds held 48k pipelines, ~21 GB of driver +// memory built before the game started; the list is only saved at exit or after two minutes, so a +// boot that ran out of memory never saved one and the next boot did the same. +constexpr size_t MaxPipelinesWithoutUsedList = 3000; + +// EDEN_NX2_PRELOAD_ALL_PIPELINES=1 builds every cached pipeline at boot, as before. +bool PreloadAllPipelines() { + static const bool all = [] { + const char* value = std::getenv("EDEN_NX2_PRELOAD_ALL_PIPELINES"); + return value != nullptr && value[0] == '1'; + }(); + return all; +} +} // Anonymous namespace + PipelineCache::~PipelineCache() { + SaveUsedPipelines(); if (use_vulkan_pipeline_cache && !vulkan_pipeline_cache_filename.empty()) { SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache, CACHE_VERSION); } } +void PipelineCache::SaveUsedPipelines() { + used_pipelines_saved = std::chrono::steady_clock::now(); + // A session too short to say what the game uses (a boot to the menu) keeps the old list. + constexpr size_t MinimumUsed = 64; + if (used_pipelines_filename.empty() || + used_this_session.size() + used_and_evicted.size() < MinimumUsed) { + return; + } + std::unordered_map<u64, u8> next; + next.reserve(used_pipelines.size() + used_this_session.size()); + for (const auto& [key, pipeline] : graphics_cache) { + if (used_this_session.contains(pipeline.get())) { + next.emplace(key.Hash(), u8{0}); + } + } + for (const u64 hash : used_and_evicted) { + next.emplace(hash, u8{0}); + } + for (const auto& [hash, age] : used_pipelines) { + if (age + 1 < UsedPipelinesMaxAge) { + next.try_emplace(hash, static_cast<u8>(age + 1)); + } + } + std::ofstream file{used_pipelines_filename, std::ios::binary | std::ios::trunc}; + const u32 count = static_cast<u32>(next.size()); + file.write(reinterpret_cast<const char*>(&UsedPipelinesMagic), sizeof(UsedPipelinesMagic)); + file.write(reinterpret_cast<const char*>(&count), sizeof(count)); + for (const auto& [hash, age] : next) { + file.write(reinterpret_cast<const char*>(&hash), sizeof(hash)); + file.write(reinterpret_cast<const char*>(&age), sizeof(age)); + } +} + GraphicsPipeline* PipelineCache::CurrentGraphicsPipeline() { if (!RefreshStages(graphics_key.unique_hashes)) { @@ -582,6 +695,7 @@ GraphicsPipeline* PipelineCache::CurrentGraphicsPipeline() { GraphicsPipeline* const next{current_pipeline->Next(graphics_key)}; if (next) { current_pipeline = next; + current_pipeline->MarkUsed(frame_count); return BuiltPipeline(current_pipeline); } } @@ -629,6 +743,11 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading return; } pipeline_cache_filename = base_dir / "vulkan.bin"; + used_pipelines_filename = base_dir / "vulkan_used.bin"; + used_pipelines = LoadUsedPipelines(used_pipelines_filename); + used_pipelines_saved = std::chrono::steady_clock::now(); + size_t skipped_pipelines = 0; + size_t queued_graphics = 0; if (use_vulkan_pipeline_cache) { vulkan_pipeline_cache_filename = base_dir / "vulkan_pipelines.bin"; @@ -669,6 +788,15 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading GraphicsPipelineCacheKey key; file.read(reinterpret_cast<char*>(&key), sizeof(key)); + // Task/mesh pipelines are not stored (their mesh registers are not in the format); skip + // any that were. + if (std::ranges::any_of(envs, [](const FileEnvironment& env) { + return env.ShaderStage() == Shader::Stage::Task || + env.ShaderStage() == Shader::Stage::Mesh; + })) { + return; + } + if ((key.state.extended_dynamic_state != 0) != dynamic_features.has_extended_dynamic_state || (key.state.extended_dynamic_state_2 != 0) != @@ -697,6 +825,16 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading return; } + const u64 hash = key.Hash(); + disk_pipelines.insert(hash); + if (!PreloadAllPipelines() && + (used_pipelines.empty() ? queued_graphics >= MaxPipelinesWithoutUsedList + : !used_pipelines.contains(hash))) { + ++skipped_pipelines; + return; + } + ++queued_graphics; + workers.QueueWork([this, key, envs_ = std::move(envs), &state, &callback]() mutable { ShaderPools pools; boost::container::static_vector<Shader::Environment*, 5> env_ptrs; @@ -721,6 +859,11 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading load_graphics); LOG_INFO(Render_Vulkan, "Total Pipeline Count: {}", state.total); + if (skipped_pipelines != 0) { + LOG_INFO(Render_Vulkan, + "{} cached graphics pipelines are built when first needed", + skipped_pipelines); + } std::unique_lock lock{state.mutex}; callback(VideoCore::LoadCallbackStage::Build, 0, state.total); @@ -837,13 +980,66 @@ GraphicsPipeline* PipelineCache::CurrentGraphicsPipelineSlowPath() { if (!pipeline) { return nullptr; } + if (used_this_session.insert(pipeline.get()).second && + std::chrono::steady_clock::now() - used_pipelines_saved > std::chrono::minutes{2}) { + SaveUsedPipelines(); + } if (current_pipeline) { current_pipeline->AddTransition(pipeline.get()); } current_pipeline = pipeline.get(); + current_pipeline->MarkUsed(frame_count); return BuiltPipeline(current_pipeline); } +void PipelineCache::TickFrame() { + ++frame_count; + while (!evicted_pipelines.empty() && scheduler.IsFree(evicted_pipelines.front().first)) { + evicted_pipelines.pop_front(); + } + constexpr u64 EvictCheckFrames = 600; + if (frame_count % EvictCheckFrames == 0) { + EvictUnusedPipelines(); + } +} + +void PipelineCache::EvictUnusedPipelines() { + // Each built pipeline holds ~0.45 MB of driver memory and none was ever destroyed, so it grew + // with every place a session went (title 1: 1,500 on its title screen alone). One unused for + // this long is destroyed; it takes 2-3 ms to build again from the driver's pipeline cache. + constexpr u64 EvictAfterFrames = 9000; + if (frame_count <= EvictAfterFrames) { + return; + } + const u64 cutoff = frame_count - EvictAfterFrames; + const u64 tick = scheduler.CurrentTick(); + size_t evicted = 0; + for (auto it = graphics_cache.begin(); it != graphics_cache.end();) { + GraphicsPipeline* const pipeline = it->second.get(); + if (pipeline == nullptr || pipeline == current_pipeline || !pipeline->IsBuilt() || + pipeline->LastUse() >= cutoff) { + ++it; + continue; + } + if (used_this_session.erase(pipeline) != 0) { + used_and_evicted.insert(it->first.Hash()); + } + evicted_pipelines.emplace_back(tick, std::move(it->second)); + it = graphics_cache.erase(it); + ++evicted; + } + if (evicted == 0) { + return; + } + for (auto& [key, pipeline] : graphics_cache) { + if (pipeline) { + pipeline->ClearTransitions(); + } + } + LOG_INFO(Render_Vulkan, "Evicted {} graphics pipelines unused for {} frames, {} left", evicted, + EvictAfterFrames, graphics_cache.size()); +} + GraphicsPipeline* PipelineCache::BuiltPipeline(GraphicsPipeline* pipeline) const noexcept { if (pipeline->IsBuilt()) { return pipeline; @@ -985,7 +1181,12 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline( ++env_index; const u32 cfg_offset{static_cast<u32>(env.StartAddress() + sizeof(Shader::ProgramHeader))}; - Shader::Maxwell::Flow::CFG cfg(env, pools.flow_block, cfg_offset, index == 0); + try { + // Walks the program for the dump; the one that failed may fail here again, and an + // exception leaving this handler would end the process. + Shader::Maxwell::Flow::CFG cfg(env, pools.flow_block, cfg_offset, index == 0); + } catch (const Shader::Exception&) { + } env.Dump(hash, key.unique_hashes[index]); } LOG_ERROR(Render_Vulkan, "{}", exception.what()); @@ -1033,6 +1234,18 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() { if (!pipeline || pipeline_cache_filename.empty()) { return pipeline; } + if (pipeline->IsMeshPipeline()) { + // Not stored: task and mesh programs are translated with the mesh registers, which the + // cache format does not carry. The few of them rebuild quickly from the driver cache. + QueueVulkanPipelineCacheFlush(); + return pipeline; + } + if (disk_pipelines.contains(graphics_key.Hash())) { + // In the disk cache already, only not built at boot. + QueueVulkanPipelineCacheFlush(); + return pipeline; + } + disk_pipelines.insert(graphics_key.Hash()); serialization_thread.QueueWork([this, key = graphics_key, envs = std::move(environments.envs)] { boost::container::static_vector<const GenericEnvironment*, Maxwell::MaxShaderProgram> env_ptrs; @@ -1055,7 +1268,8 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline( }; const GPUVAddr program_base{kepler_compute->regs.code_loc.Address()}; const auto& qmd{kepler_compute->launch_description}; - ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start}; + ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start, + shader->lowered_native}; env.SetCachedSize(shader->size_bytes); main_pools.ReleaseContents(); diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.h b/src/video_core/renderer_vulkan/vk_pipeline_cache.h index 71dfb46e3d..66fa615ea5 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.h +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.h @@ -10,7 +10,10 @@ #include <atomic> #include <chrono> #include <cstddef> +#include <deque> #include <filesystem> +#include <unordered_map> +#include <unordered_set> #include <memory> #include <type_traits> #include "common/container/unordered_map.h" @@ -114,6 +117,8 @@ public: [[nodiscard]] GraphicsPipeline* CurrentGraphicsPipeline(); + void TickFrame(); + [[nodiscard]] ComputePipeline* CurrentComputePipeline(); void LoadDiskResources(u64 title_id, std::stop_token stop_loading, @@ -163,6 +168,12 @@ private: GraphicsPipelineCacheKey graphics_key{}; GraphicsPipeline* current_pipeline{}; + /// Destroys graphics pipelines unused for a while, once the GPU is done with them. + void EvictUnusedPipelines(); + u64 frame_count{}; + std::deque<std::pair<u64, std::unique_ptr<GraphicsPipeline>>> evicted_pipelines; ///< (tick, pipeline) + std::unordered_set<u64> used_and_evicted; ///< Key hashes of pipelines used this session, evicted since + ::Common::unordered_map<ComputePipelineCacheKey, std::unique_ptr<ComputePipeline>> compute_cache; ::Common::unordered_map<GraphicsPipelineCacheKey, std::unique_ptr<GraphicsPipeline>> graphics_cache; @@ -173,6 +184,16 @@ private: std::filesystem::path pipeline_cache_filename; + /// Key hashes of the graphics pipelines in the disk cache. Only those a recent session used + /// (used_pipelines) are built at boot; the rest are built when first needed, and not stored + /// again. Each built pipeline holds driver memory (title 1: 6.5k cached, a quarter used). + std::unordered_set<u64> disk_pipelines; + std::unordered_map<u64, u8> used_pipelines; ///< Key hash -> sessions since it was last used + std::unordered_set<const GraphicsPipeline*> used_this_session; + std::filesystem::path used_pipelines_filename; + std::chrono::steady_clock::time_point used_pipelines_saved{}; + void SaveUsedPipelines(); + std::filesystem::path vulkan_pipeline_cache_filename; vk::PipelineCache vulkan_pipeline_cache; size_t pipelines_since_flush{}; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index bbb49d5432..d3dbba6163 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -9,6 +9,7 @@ #include <memory> #include <cstdio> #include <cstdlib> +#include <filesystem> #include <tuple> #include <set> #include <fstream> @@ -279,6 +280,22 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { report_failure("no pipeline"); return; } + // Task/mesh pipelines: the draw's vertex count is the number of task workgroups. + mesh_draw = pipeline->IsMeshPipeline(); + // EDEN_NX2_NO_MESH=1: skip task/mesh draws as before they were supported (A/B comparisons). + // EDEN_NX2_NO_MESH_FILE=<path>: skip them while that file exists (checked every 1024 draws), + // to compare the same scene with and without them in one run. + static const bool no_mesh_always{std::getenv("EDEN_NX2_NO_MESH") != nullptr}; + static const char* const no_mesh_file{std::getenv("EDEN_NX2_NO_MESH_FILE")}; + static bool no_mesh{no_mesh_always}; + static u32 no_mesh_check{}; + if (no_mesh_file && (no_mesh_check++ % 1024) == 0) { + no_mesh = std::filesystem::exists(no_mesh_file); + } + if (mesh_draw && (no_mesh || !device.IsExtMeshShaderSupported())) { + report_failure(no_mesh ? "EDEN_NX2_NO_MESH" : "no VK_EXT_mesh_shader"); + return; + } std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; // update engine as channel may be different. pipeline->SetEngine(maxwell3d, gpu_memory); @@ -553,6 +570,12 @@ void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) { const u32 num_instances{instance_count}; const DrawParams draw_params{MakeDrawParams(draw_state, num_instances, is_indexed)}; + if (mesh_draw) { + scheduler.Record([groups = draw_params.num_vertices](vk::CommandBuffer cmdbuf) { + cmdbuf.DrawMeshTasksEXT(groups, 1, 1); + }); + return; + } scheduler.Record([draw_params](vk::CommandBuffer cmdbuf) { if (draw_params.is_indexed) { cmdbuf.DrawIndexed(draw_params.num_vertices, draw_params.num_instances, @@ -585,6 +608,14 @@ void RasterizerVulkan::DrawIndirect() { const auto& params = maxwell3d->draw_manager.indirect_state; buffer_cache.SetDrawIndirect(¶ms); PrepareDraw(params.is_indexed, [this, ¶ms] { + if (mesh_draw) { + // Indirect task/mesh draws would need vkCmdDrawMeshTasksIndirectEXT's layout. + static bool logged{}; + if (!std::exchange(logged, true)) { + LOG_WARNING(Render_Vulkan, "Indirect task/mesh draw skipped"); + } + return; + } const auto indirect_buffer = buffer_cache.GetDrawIndirectBuffer(); const auto& buffer = indirect_buffer.first; const auto& offset = indirect_buffer.second; @@ -1212,6 +1243,7 @@ void RasterizerVulkan::FlushCommands() { void RasterizerVulkan::TickFrame() { draw_counter = 0; + pipeline_cache.TickFrame(); guest_descriptor_queue.TickFrame(); compute_pass_descriptor_queue.TickFrame(); descriptor_buffer_ring.TickFrame(); diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index 7aca376614..fc282fbe8d 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -235,6 +235,7 @@ private: boost::container::static_vector<VkSampler, MAX_TEXTURES> sampler_handles; u32 draw_counter = 0; + bool mesh_draw{}; ///< the draw being prepared uses a task/mesh pipeline }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp index 7b6571d33d..98787b9add 100644 --- a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp @@ -36,11 +36,25 @@ constexpr VkDeviceSize MAX_ALIGNMENT = 256; #if defined(__FreeBSD__) constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 128_MiB; #else -constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 256_MiB; +// 128 MiB on Windows too: 256 MiB sat mostly unused in device memory on Switch 2 titles. +constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 128_MiB; #endif size_t GetStreamBufferSize(const Device& device) { if (!device.HasDebuggingToolAttached()) { + if (device.IsIntegrated()) { + // An integrated GPU's device-local host-visible heap can be no bigger than the stream + // buffer (Radeon 780M: 256 MiB, 243 MiB budget), which then sat over that budget. + // Leave a quarter of it free. + VkDeviceSize heap_size = 0; + ForEachDeviceLocalHostVisibleHeap( + device, [&heap_size](size_t, VkMemoryHeap& heap) { + heap_size = (std::max)(heap_size, heap.size); + }); + if (heap_size != 0 && heap_size <= MAX_STREAM_BUFFER_SIZE) { + return Common::AlignUp(heap_size * 3 / 4, MAX_ALIGNMENT); + } + } return MAX_STREAM_BUFFER_SIZE; } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index b0e5eacb32..30981fe413 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -244,6 +244,35 @@ enum class SparseTilesMode { return size >= (32U << 20) && Tegra::Nx2InSparseRegion(gpu_addr, size); } +// Vulkan's standard sparse block shape for single-sampled 2D images (64 KiB blocks), in texel +// blocks, for a texel block of `block_bytes`. Only valid when the device reports +// residencyStandard2DBlockShape. +[[nodiscard]] bool StandardSparseBlockShape(const Device& device, u64 block_bytes, + VkExtent3D& shape) { + if (!device.GetPhysical().GetProperties().sparseProperties.residencyStandard2DBlockShape) { + return false; + } + switch (block_bytes) { + case 1: + shape = {256, 256, 1}; + return true; + case 2: + shape = {256, 128, 1}; + return true; + case 4: + shape = {128, 128, 1}; + return true; + case 8: + shape = {128, 64, 1}; + return true; + case 16: + shape = {64, 64, 1}; + return true; + default: + return false; + } +} + // The most device memory one SparseLayers image binds: 768 MiB, 384 MiB on GPUs with less than // 7 GiB (title 1's BC6H arrays are 85 MB a layer). [[nodiscard]] VkDeviceSize SparseLayersCap(const Device& device) { @@ -304,6 +333,20 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM info.size.depth == 1; } +// The most a trimmed ASTC array keeps on the host: 512 MiB, 256 MiB on GPUs with less than 7 GiB +// (title 1's two terrain arrays took 724 MiB of a 6 GB card next to ~1.2 GB of other sparse terrain +// textures). EDEN_NX2_ASTC_ARRAY_MIB=<MiB> sets it. +[[nodiscard]] u64 TrimmedAstcArrayLimit(const Device& device) { + static const u64 forced = [] { + const char* value = std::getenv("EDEN_NX2_ASTC_ARRAY_MIB"); + return value != nullptr ? std::strtoull(value, nullptr, 10) << 20 : 0ULL; + }(); + if (forced != 0) { + return forced; + } + return device.GetDeviceLocalMemory() < (u64{7} << 30) ? (u64{256} << 20) : (u64{512} << 20); +} + // The host shape of an image whose decoded copy would not fit in VRAM: a GPU-decoded ASTC array // over 1 GiB as RGBA8 (title 1's 8192x8192x68 ASTC 8x8 virtual textures are 24 GB each) keeps only // its mip levels from `level_offset` on, so host mip 0 is guest mip `level_offset`. Normalized @@ -324,8 +367,9 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM if (decoded_bytes(0) <= (u64{1} << 30)) { return info; } + const u64 limit = TrimmedAstcArrayLimit(device); while (level_offset + 1 < static_cast<u32>(info.resources.levels) && - decoded_bytes(level_offset) > (u64{512} << 20)) { + decoded_bytes(level_offset) > limit) { ++level_offset; } ImageInfo host = info; @@ -2174,10 +2218,22 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 TextureCacheRuntime::CurrentTick() const noexcept { + return scheduler.CurrentTick(); +} + +bool TextureCacheRuntime::IsFree(u64 tick) const noexcept { + return scheduler.IsFree(tick); +} + u64 TextureCacheRuntime::GetDeviceMemoryBudget() const { return device.GetDeviceMemoryBudget(); } +u64 TextureCacheRuntime::GetDeviceMemorySlack() const { + return device.GetDeviceMemorySlack(); +} + bool TextureCacheRuntime::CanDownloadMsaa(const VideoCommon::ImageInfo& info) const { const VkImageAspectFlags aspect_mask = ImageAspectMask(info.format); if ((aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0) { @@ -2255,10 +2311,33 @@ void TextureCacheRuntime::ReleaseMsaaScratchImage(VkImage image) { } void TextureCacheRuntime::TickFrame() { + static constexpr u32 MAX_UNUSED_SCRATCH_FRAMES = 60; + // Finished decode images wait for the next texture of their shape, up to this much memory. + static constexpr u64 MaxFreeTranscodeBytes = u64{256} << 20; + const auto decode_bytes = [](const PendingTranscode& t) { + u64 bytes{}; + for (u32 level = 0; level < t.levels; ++level) { + bytes += u64{(std::max)(t.width >> level, 1U)} * (std::max)(t.height >> level, 1U) * 4; + } + return bytes * t.layers; + }; + u64 free_bytes{}; + for (const PendingTranscode& t : free_transcodes) { + free_bytes += decode_bytes(t); + } while (!pending_transcodes.empty() && scheduler.IsFree(pending_transcodes.front().tick)) { + PendingTranscode& done = pending_transcodes.front(); + const u64 bytes = decode_bytes(done); + if (free_bytes + bytes <= MaxFreeTranscodeBytes) { + done.unused_frames = 0; + free_bytes += bytes; + free_transcodes.push_back(std::move(done)); + } pending_transcodes.pop_front(); } - static constexpr u32 MAX_UNUSED_SCRATCH_FRAMES = 60; + std::erase_if(free_transcodes, [](PendingTranscode& t) { + return ++t.unused_frames > MAX_UNUSED_SCRATCH_FRAMES; + }); std::erase_if(msaa_scratch_images, [this](MsaaScratchImage& scratch) { if (!scheduler.IsFree(scratch.tick)) { scratch.unused_frames = 0; @@ -2798,7 +2877,24 @@ bool Image::InitSparseLayers() { const u32 block_width = huge_astc ? 1 : VideoCore::Surface::DefaultBlockWidth(info.format); const u32 block_height = huge_astc ? 1 : VideoCore::Surface::DefaultBlockHeight(info.format); const u64 block_bytes = huge_astc ? 4 : VideoCore::Surface::BytesPerBlock(info.format); - const VkExtent3D granularity = it->formatProperties.imageGranularity; + VkExtent3D granularity = it->formatProperties.imageGranularity; + if (granularity.width == 0 || granularity.height == 0) { + // Seen with AMD's driver: the sparse block size comes back as 0, and the size computation + // below divided by it. Use the standard 2D block shape when the device promises it. + if (!StandardSparseBlockShape(device, block_bytes, granularity)) { + LOG_WARNING(Render_Vulkan, + "Sparse layered image {:#x}: the driver reports no sparse block size and " + "no standard block shape; not using sparse layers", + gpu_addr); + return false; + } + granularity.width *= block_width; + granularity.height *= block_height; + LOG_INFO(Render_Vulkan, + "Sparse layered image {:#x}: no block size from the driver, using the standard " + "{}x{} texel block", + gpu_addr, granularity.width, granularity.height); + } s.tile_width = granularity.width; s.tile_height = granularity.height; const u64 tile = s.requirements.alignment; @@ -4235,30 +4331,47 @@ void TextureCacheRuntime::TranscodeAstcToBcn( const u32 levels = static_cast<u32>(info.resources.levels); const u32 layers = static_cast<u32>(info.resources.layers); constexpr VkFormat DecodeFormat = VK_FORMAT_A8B8G8R8_UNORM_PACK32; - PendingTranscode pending{ - .tick = 0, - .image = memory_allocator.CreateImage(VkImageCreateInfo{ - .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .imageType = VK_IMAGE_TYPE_2D, - .format = DecodeFormat, - .extent = {width, height, 1}, - .mipLevels = levels, - .arrayLayers = layers, - .samples = VK_SAMPLE_COUNT_1_BIT, - .tiling = VK_IMAGE_TILING_OPTIMAL, - .usage = VK_IMAGE_USAGE_STORAGE_BIT, - .sharingMode = VK_SHARING_MODE_EXCLUSIVE, - .queueFamilyIndexCount = 0, - .pQueueFamilyIndices = nullptr, - .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, - }, true), - .views = {}, - }; - for (u32 level = 0; level < levels; ++level) { - pending.views.push_back( - MakeStorageView(device.GetLogical(), level, *pending.image, DecodeFormat)); + // A decode image of the same shape the GPU is done with: title 1's title screen decodes + // dozens of textures a frame, mostly of a few shapes. Making one took 263 us; from VMA's + // shared blocks 90 us, but freeing them there left ~1 GB of those blocks unused. + const auto reusable = std::ranges::find_if(free_transcodes, [&](const PendingTranscode& t) { + return t.width == width && t.height == height && t.levels == levels && t.layers == layers; + }); + PendingTranscode pending; + if (reusable != free_transcodes.end()) { + pending = std::move(*reusable); + free_transcodes.erase(reusable); + } else { + pending = PendingTranscode{ + .tick = 0, + .image = memory_allocator.CreateImage(VkImageCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .imageType = VK_IMAGE_TYPE_2D, + .format = DecodeFormat, + .extent = {width, height, 1}, + .mipLevels = levels, + .arrayLayers = layers, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_STORAGE_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + }, true), + .views = {}, + .width = width, + .height = height, + .levels = levels, + .layers = layers, + .unused_frames = 0, + }; + for (u32 level = 0; level < levels; ++level) { + pending.views.push_back( + MakeStorageView(device.GetLogical(), level, *pending.image, DecodeFormat)); + } } astc_decoder_pass->AssembleInto( info, 0, image.guest_size_bytes, diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 25a6ce7c1e..564b2cbf9e 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -67,10 +67,22 @@ public: u64 GetDeviceMemoryUsage() const; + /// GPU progress: the texture cache frees a deleted image once the GPU is past its last use. + [[nodiscard]] u64 CurrentTick() const noexcept; + + [[nodiscard]] bool IsFree(u64 tick) const noexcept; + u64 GetDeviceMemoryBudget() const; + u64 GetDeviceMemorySlack() const; + bool CanReportMemoryUsage() const; + /// Images that did not fit the device memory budget so far (see MemoryAllocator). + u32 MemoryPressureEvents() const noexcept { + return memory_allocator.MemoryPressureEvents(); + } + void LogMemoryStatistics() const; /// True when ASTC is re-compressed to BC3 and opaque images may use BC1 instead. @@ -192,13 +204,20 @@ public: void TranscodeAstcToBcn(Image& image, const StagingBufferRef& map, std::span<const VideoCommon::SwizzleParameters> swizzles); - /// Temporary decode images, freed once the GPU is past `tick`. + /// Temporary decode images, reusable once the GPU is past `tick`. struct PendingTranscode { u64 tick; vk::Image image; std::vector<vk::ImageView> views; + u32 width; + u32 height; + u32 levels; + u32 layers; + u32 unused_frames; }; std::deque<PendingTranscode> pending_transcodes; + /// Decode images the GPU is done with, for the next texture of the same shape. + std::vector<PendingTranscode> free_transcodes; std::optional<BlockLinearUnswizzle3DPass> bl3d_unswizzle_pass; const Settings::ResolutionScalingInfo& resolution; @@ -390,6 +409,11 @@ public: return (this->*current_image).UsageFlags(); } + /// Where the image's memory lies, for the collector (TextureCache::RunGarbageCollector). + [[nodiscard]] vk::Image::MemoryBlock MemoryBlock() const noexcept { + return original_image.GetMemoryBlock(); + } + /// Returns true when the image is already initialized and mark it as initialized [[nodiscard]] bool ExchangeInitialization() noexcept { return std::exchange(initialized, true); diff --git a/src/video_core/shader_cache.cpp b/src/video_core/shader_cache.cpp index dfe80abac3..5faed0dba9 100644 --- a/src/video_core/shader_cache.cpp +++ b/src/video_core/shader_cache.cpp @@ -14,6 +14,7 @@ #include <boost/container/small_vector.hpp> #include "common/assert.h" +#include "common/cityhash.h" #include "shader_recompiler/frontend/maxwell/control_flow.h" #include "shader_recompiler/exception.h" #include "shader_recompiler/object_pool.h" @@ -48,6 +49,22 @@ void ShaderCache::SyncGuestHost() { ShaderCache::ShaderCache(Tegra::MaxwellDeviceMemoryManager& device_memory_) : device_memory{device_memory_} {} +// Task and mesh programs are translated with the mesh registers and the header of the pass-through +// geometry program (per-primitive outputs), so pipelines are keyed by them too. +u64 ShaderCache::MeshStateHash() const { + using Regs = Tegra::Engines::Maxwell3D::Regs; + const auto& regs{maxwell3d->regs}; + std::array<u32, 3 + 20> state{regs.reg_array[Regs::MeshShaderA], regs.reg_array[Regs::MeshShaderB], + regs.reg_array[Regs::MeshInitShader]}; + if (regs.IsShaderConfigEnabled(Regs::ShaderType::Geometry)) { + const auto& geometry{regs.pipelines[static_cast<size_t>(Regs::ShaderType::Geometry)]}; + const GPUVAddr address{(static_cast<u64>(geometry.reserved[0] & 0xff) << 32) | + geometry.reserved[1]}; + gpu_memory->ReadBlockUnsafe(address, state.data() + 3, 20 * sizeof(u32)); + } + return Common::CityHash64(reinterpret_cast<const char*>(state.data()), sizeof(state)) | 1; +} + bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { auto& dirty{maxwell3d->dirty.flags}; if (!dirty[VideoCommon::Dirty::Shaders]) { @@ -57,6 +74,7 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()}; const bool native = maxwell3d->GetNativeMacroEngine().native_turing; + const bool mesh{native && maxwell3d->regs.IsMeshEnabled()}; for (size_t index = 0; index < Tegra::Engines::Maxwell3D::Regs::MaxShaderProgram; ++index) { if (!maxwell3d->regs.IsShaderConfigEnabled(index)) { unique_hashes[index] = 0; @@ -69,6 +87,11 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { unique_hashes[index] = 0; continue; } + if (mesh && program == Tegra::Engines::Maxwell3D::Regs::ShaderType::Geometry) { + // A header without code: the mesh program's environment reads its output map. + unique_hashes[index] = 0; + continue; + } const GPUVAddr shader_addr{native ? (static_cast<u64>(shader_config.reserved[0] & 0xff) << 32) | shader_config.reserved[1] : base_addr + shader_config.offset}; @@ -89,7 +112,8 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { if (!shader_info) { const u32 start_address{native ? 0 : shader_config.offset}; try { - // Native programs are lowered while the environment is built. + // Native programs are lowered while the environment is built (once: the + // ShaderInfo keeps the result for the pipeline's environments). RethrowIfRejected(*cpu_shader_addr); GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, native ? shader_addr : base_addr, start_address}; shader_info = MakeShaderInfo(env, *cpu_shader_addr); @@ -111,6 +135,10 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { } shader_infos[index] = shader_info; unique_hashes[index] = shader_info->unique_hash; + if (mesh && (program == Tegra::Engines::Maxwell3D::Regs::ShaderType::VertexB || + program == Tegra::Engines::Maxwell3D::Regs::ShaderType::Tessellation)) { + unique_hashes[index] ^= MeshStateHash(); + } } last_shaders_valid = true; return true; @@ -202,7 +230,8 @@ void ShaderCache::GetGraphicsEnvironments(GraphicsEnvironments& result, const GPUVAddr address = native ? (static_cast<u64>(config.reserved[0] & 0xff) << 32) | config.reserved[1] : base_addr; - env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, address, start_address}; + env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, address, start_address, + shader_infos[index]->lowered_native}; env.SetCachedSize(shader_infos[index]->size_bytes); result.env_ptrs[env_index++] = &env; } @@ -342,6 +371,7 @@ const ShaderInfo* ShaderCache::MakeShaderInfo(GenericEnvironment& env, VAddr cpu info->unique_hash = env.CalculateHash(); info->size_bytes = env.ReadSizeBytes(); } + info->lowered_native = env.LoweredNative(); const size_t size_bytes{env.SourceSizeBytes() ? env.SourceSizeBytes() : info->size_bytes}; const ShaderInfo* const result{info.get()}; Register(std::move(info), cpu_addr, size_bytes); diff --git a/src/video_core/shader_cache.h b/src/video_core/shader_cache.h index f4179dd435..281a90cb6f 100644 --- a/src/video_core/shader_cache.h +++ b/src/video_core/shader_cache.h @@ -38,6 +38,8 @@ class GenericEnvironment; struct ShaderInfo { u64 unique_hash{}; size_t size_bytes{}; + /// Lowered code of a native program, reused when its pipelines are built. + std::shared_ptr<const LoweredNativeCode> lowered_native; }; class ShaderCache : public VideoCommon::ChannelSetupCaches<VideoCommon::ChannelInfo> { @@ -151,6 +153,8 @@ private: bool IsRejected(VAddr addr); /// Remembers the exception being handled as this address's rejection. void RememberRejection(VAddr addr); + /// Hash of the state task and mesh programs are translated with. + u64 MeshStateHash() const; Tegra::MaxwellDeviceMemoryManager& device_memory; diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp index 9d189ad4f0..22e661c849 100644 --- a/src/video_core/shader_environment.cpp +++ b/src/video_core/shader_environment.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <array> #include <chrono> #include <cstdlib> #include <thread> @@ -315,29 +316,107 @@ std::optional<u64> GenericEnvironment::TryFindSize() { Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw) { const auto handle{Tegra::Texture::TexturePair(raw, via_header_index)}; - ASSERT(handle.first <= tic_limit); - const GPUVAddr descriptor_addr{tic_addr + handle.first * sizeof(Tegra::Texture::TICEntry)}; + const u32 index{Tegra::Texture::TextureHeaderIndex(handle.first, tic_limit)}; + ASSERT(index <= tic_limit); + const GPUVAddr descriptor_addr{tic_addr + index * sizeof(Tegra::Texture::TICEntry)}; Tegra::Texture::TICEntry entry; gpu_memory->ReadBlock(descriptor_addr, &entry, sizeof(entry)); return entry; } -void GenericEnvironment::InitializeNativeCode(u32 source_header_size) { +namespace { + +// Reads a native program a 4 KiB page at a time instead of one 16-byte instruction at a time. +// The reads do not flush the GPU caches first (ReadBlockUnsafe): the previous per-instruction +// ReadBlock did, once per instruction. An instruction that is not inside a fully mapped page is +// left to the caller. +class NativeInstructionReader { +public: + using Instruction = Shader::Ampere::Instruction; + static constexpr GPUVAddr PageSize = 0x1000; + + explicit NativeInstructionReader(Tegra::MemoryManager& gpu_memory_) + : gpu_memory{gpu_memory_} {} + + /// Copies the instruction at `address` into `out`, loading its page first if needed. + /// Returns false when the instruction is not inside a fully mapped page. + bool TryRead(GPUVAddr address, Instruction& out) { + if (!Holds(address)) { + LoadFrom(address); + } + if (!Holds(address)) { + return false; + } + out = page[(address - page_begin) / sizeof(Instruction)]; + return true; + } + +private: + bool Holds(GPUVAddr address) const { + return address >= page_begin && address + sizeof(Instruction) <= page_end; + } + + // Loads from `address` up to the next page boundary; loads nothing if that range is not + // fully mapped. + void LoadFrom(GPUVAddr address) { + const GPUVAddr next_page = (address | (PageSize - 1)) + 1; + const GPUVAddr length = next_page - address; + page_begin = address; + page_end = address; + if (length >= sizeof(Instruction) && gpu_memory.IsFullyMappedRange(address, length)) { + gpu_memory.ReadBlockUnsafe(address, page.data(), length); + page_end = next_page; + } + } + + Tegra::MemoryManager& gpu_memory; + std::array<Instruction, PageSize / sizeof(Instruction)> page; + GPUVAddr page_begin{}; + GPUVAddr page_end{}; +}; + +} // Anonymous namespace + +// A native program is captured and lowered once, when the shader cache first meets it. The +// environments built later for its pipelines receive that result instead of lowering again. +void GenericEnvironment::InitializeNativeCode(std::shared_ptr<const LoweredNativeCode> lowered, + u32 source_header_size) { + if (!lowered) { + lowered = LowerNativeCode(source_header_size); + } + if (lowered->texture_bound != ~0U) { + // Native TEX carries its cbuf index explicitly; TEXS obtains it from the environment. + texture_bound = lowered->texture_bound; + } + native_source_size = lowered->source_size; + code = lowered->code; + cached_lowest = 0; + cached_highest = static_cast<u32>((code.size() - 1) * sizeof(u64)); + lowered_native = std::move(lowered); +} + +std::shared_ptr<const LoweredNativeCode> GenericEnvironment::LowerNativeCode(u32 source_header_size) { using Shader::Ampere::Instruction; const u32 native_header_size = source_header_size; constexpr u32 maximum_instructions = Shader::Ampere::ProgramCaptureState::MaximumInstructions; std::vector<Instruction> native; Shader::Ampere::ProgramCaptureState capture; + NativeInstructionReader reader{*gpu_memory}; for (u32 index = 0; index < maximum_instructions; ++index) { const GPUVAddr address = program_base + native_header_size + index * sizeof(Instruction); - if (!gpu_memory->IsFullyMappedRange(address, sizeof(Instruction))) { - // The program may end before a branch target past its last EXIT (the capture can't - // tell); stop at the mapping's end and let the lowering judge what was captured. - if (!native.empty()) break; - throw Shader::RuntimeError("Unmapped native shader instruction at {:#x}", address); + Instruction instruction; + if (!reader.TryRead(address, instruction)) { + // Not inside a fully mapped page: check and read this one instruction on its own. + if (!gpu_memory->IsFullyMappedRange(address, sizeof(Instruction))) { + // The program may end before a branch target past its last EXIT (the capture + // can't tell); stop at the mapping's end and let the lowering judge what was + // captured. + if (!native.empty()) break; + throw Shader::RuntimeError("Unmapped native shader instruction at {:#x}", address); + } + gpu_memory->ReadBlockUnsafe(address, &instruction, sizeof(instruction)); } - auto& instruction = native.emplace_back(); - gpu_memory->ReadBlock(address, &instruction, sizeof(instruction)); + native.push_back(instruction); const auto status = capture.Observe(instruction, index); if (status == Shader::Ampere::ProgramCaptureState::Result::InvalidTarget) { if (const char* dir = std::getenv("EDEN_NX2_SHADER_DUMP")) { @@ -439,39 +518,46 @@ void GenericEnvironment::InitializeNativeCode(u32 source_header_size) { program_base, lowered.error_offset, lowered.error, words[0] & 0xfff, words[0], words[1], words[2], words[3]); } - if (lowered.required_texture_buffer != ~0U) { - // Native TEX carries its cbuf index explicitly; TEXS obtains it from the environment. - texture_bound = lowered.required_texture_buffer; - } - native_source_size = native_header_size + lowered.source_size; - code.resize(sizeof(sph) / sizeof(u64)); - std::memcpy(code.data(), &sph, sizeof(sph)); + auto result = std::make_shared<LoweredNativeCode>(); + result->texture_bound = lowered.required_texture_buffer; + result->source_size = native_header_size + lowered.source_size; + auto& lowered_code = result->code; + lowered_code.resize(sizeof(sph) / sizeof(u64)); + std::memcpy(lowered_code.data(), &sph, sizeof(sph)); for (const u64 instruction : lowered.instructions) { - if ((code.size() * sizeof(u64)) % 32 == 0) { - code.push_back(0); // Maxwell scheduling word, absent from native instructions. + if ((lowered_code.size() * sizeof(u64)) % 32 == 0) { + lowered_code.push_back(0); // Maxwell scheduling word, absent from native instructions. } - code.push_back(instruction); + lowered_code.push_back(instruction); } // Keep the original padding ABI, then retain the native address map as // unreachable data. Existing disk-cache serialization carries it with code. - code.push_back(0); - Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets); - cached_lowest = 0; - cached_highest = static_cast<u32>((code.size() - 1) * sizeof(u64)); + lowered_code.push_back(0); + Shader::Ampere::AppendNativeAddressMap(lowered_code, lowered.source_instruction_offsets); LOG_DEBUG(Shader, "Lowered native shader at {:#x}: {} source bytes, {} instructions", - program_base, native_source_size, lowered.instructions.size()); + program_base, result->source_size, lowered.instructions.size()); + return result; } GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, Tegra::MemoryManager& gpu_memory_, Maxwell::ShaderType program, GPUVAddr program_base_, - u32 start_address_) + u32 start_address_, + std::shared_ptr<const LoweredNativeCode> lowered_native_) : GenericEnvironment{gpu_memory_, program_base_, start_address_}, maxwell3d{&maxwell3d_} { gpu_memory->ReadBlock(program_base + start_address, &sph, sizeof(sph)); initial_offset = sizeof(sph); texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot; - if (maxwell3d->GetNativeMacroEngine().native_turing) { - if (program != Maxwell::ShaderType::VertexB && program != Maxwell::ShaderType::Pixel) { + const auto& regs{maxwell3d->regs}; + const bool native{maxwell3d->GetNativeMacroEngine().native_turing}; + // Task and mesh programs: the task in the vertex slot, its mesh program in the tessellation + // slot (without a task, the mesh program is in the vertex slot). + const bool mesh_slot{native && regs.IsMeshEnabled() && + (program == Maxwell::ShaderType::VertexB || + program == Maxwell::ShaderType::Tessellation)}; + if (native) { + if (program != Maxwell::ShaderType::VertexB && program != Maxwell::ShaderType::Pixel && + !mesh_slot) { throw Shader::RuntimeError("Unsupported native graphics shader stage {}", static_cast<u32>(program)); } if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { @@ -484,7 +570,7 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, dump.write(reinterpret_cast<const char*>(bytes.data()), bytes.size()); } } - InitializeNativeCode(); + InitializeNativeCode(std::move(lowered_native_)); } gp_passthrough_mask = maxwell3d->regs.post_vtg_shader_attrib_skip_mask; switch (program) { @@ -516,6 +602,19 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, ASSERT_MSG(false, "Invalid program={}", program); break; } + if (mesh_slot) { + const bool has_task{regs.IsShaderConfigEnabled(Maxwell::ShaderType::Tessellation)}; + stage = program == Maxwell::ShaderType::VertexB && has_task ? Shader::Stage::Task + : Shader::Stage::Mesh; + mesh_registers = {regs.reg_array[Maxwell::MeshShaderA], regs.reg_array[Maxwell::MeshShaderB], + regs.reg_array[Maxwell::MeshInitShader]}; + if (stage == Shader::Stage::Mesh && regs.IsShaderConfigEnabled(Maxwell::ShaderType::Geometry)) { + const auto& geometry{regs.pipelines[static_cast<size_t>(Maxwell::ShaderType::Geometry)]}; + const GPUVAddr address{(static_cast<u64>(geometry.reserved[0] & 0xff) << 32) | + geometry.reserved[1]}; + gpu_memory->ReadBlock(address, &mesh_primitive_header, sizeof(mesh_primitive_header)); + } + } const u64 local_size{sph.LocalMemorySize()}; ASSERT(local_size <= (std::numeric_limits<u32>::max)()); local_memory_size = static_cast<u32>(local_size) + sph.common3.shader_local_memory_crs_size; @@ -594,7 +693,8 @@ u32 GraphicsEnvironment::ReadViewportTransformState() { ComputeEnvironment::ComputeEnvironment(Tegra::Engines::KeplerCompute& kepler_compute_, Tegra::MemoryManager& gpu_memory_, GPUVAddr program_base_, - u32 start_address_) + u32 start_address_, + std::shared_ptr<const LoweredNativeCode> lowered_native_) : GenericEnvironment{gpu_memory_, program_base_, start_address_}, kepler_compute{ &kepler_compute_} { const auto& qmd{kepler_compute->launch_description}; @@ -607,7 +707,7 @@ ComputeEnvironment::ComputeEnvironment(Tegra::Engines::KeplerCompute& kepler_com if (kepler_compute->IsNativeMode()) { initial_offset = sizeof(sph); is_proprietary_driver = true; - InitializeNativeCode(0); + InitializeNativeCode(std::move(lowered_native_), 0); start_address = sizeof(sph); } } @@ -712,7 +812,8 @@ void FileEnvironment::Deserialize(std::ifstream& file) { file.read(reinterpret_cast<char*>(&gp_passthrough_mask), sizeof(gp_passthrough_mask)); } } - is_proprietary_driver = texture_bound == 2; + // Native programs fold bank 1 reads as they did when they were built. + is_proprietary_driver = texture_bound == 2 || Shader::Ampere::HasNativeAddressMap(code); } void FileEnvironment::Dump(u64 pipeline_hash, u64 shader_hash) { diff --git a/src/video_core/shader_environment.h b/src/video_core/shader_environment.h index e49f01d95f..30d59955af 100644 --- a/src/video_core/shader_environment.h +++ b/src/video_core/shader_environment.h @@ -29,6 +29,19 @@ class Memorymanager; namespace VideoCommon { +/// A native (Switch 2) program after capture and lowering to Maxwell code. The shader cache keeps +/// it so the environments built for the program's pipelines don't capture and lower it again. +struct LoweredNativeCode { + /// Shader header, lowered instructions with Maxwell scheduling words, then the native + /// address map (what the environment's `code` used to be filled with). + std::vector<u64> code; + /// Bytes of the native program that were read, header included. + size_t source_size{}; + /// Constant buffer holding the texture handles if the program requires one, otherwise ~0U + /// (the engine's register value is used). + u32 texture_bound = ~0U; +}; + class GenericEnvironment : public Shader::Environment { public: explicit GenericEnvironment() = default; @@ -63,6 +76,10 @@ public: return native_source_size; } + [[nodiscard]] const std::shared_ptr<const LoweredNativeCode>& LoweredNative() const noexcept { + return lowered_native; + } + [[nodiscard]] bool CanBeSerialized() const noexcept; [[nodiscard]] u64 CalculateHash() const; @@ -78,8 +95,12 @@ public: protected: std::optional<u64> TryFindSize(); - void InitializeNativeCode(u32 source_header_size = 128); + /// Adopts an already lowered native program, or captures and lowers it when there is none. + void InitializeNativeCode(std::shared_ptr<const LoweredNativeCode> lowered, + u32 source_header_size = 128); + std::shared_ptr<const LoweredNativeCode> LowerNativeCode(u32 source_header_size); size_t native_source_size{}; + std::shared_ptr<const LoweredNativeCode> lowered_native; Tegra::Texture::TICEntry ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw); @@ -117,7 +138,8 @@ public: explicit GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, Tegra::MemoryManager& gpu_memory_, Tegra::Engines::Maxwell3D::Regs::ShaderType program, - GPUVAddr program_base_, u32 start_address_); + GPUVAddr program_base_, u32 start_address_, + std::shared_ptr<const LoweredNativeCode> lowered_native_ = {}); ~GraphicsEnvironment() override = default; @@ -143,7 +165,8 @@ public: explicit ComputeEnvironment() = default; explicit ComputeEnvironment(Tegra::Engines::KeplerCompute& kepler_compute_, Tegra::MemoryManager& gpu_memory_, GPUVAddr program_base_, - u32 start_address_); + u32 start_address_, + std::shared_ptr<const LoweredNativeCode> lowered_native_ = {}); ~ComputeEnvironment() override = default; diff --git a/src/video_core/shader_handle_program.h b/src/video_core/shader_handle_program.h new file mode 100644 index 0000000000..3d35524cc8 --- /dev/null +++ b/src/video_core/shader_handle_program.h @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "common/common_types.h" +#include "shader_recompiler/shader_info.h" +#include "video_core/memory_manager.h" + +namespace VideoCommon { + +/// The texture handle a shader reads from a storage buffer, computed from the current constant +/// buffer and storage buffer words (Shader::Info::handle_programs). cbuf_address(index) is the +/// bound constant buffer's address, 0 when none is bound. +template <typename CbufAddress> +u32 EvaluateTextureHandle(const Shader::Info& info, u32 program, Tegra::MemoryManager& gpu_memory, + CbufAddress&& cbuf_address) { + return Shader::EvaluateHandleProgram( + info.handle_programs[program], + [&](u32 index, u32 offset) -> u32 { + const GPUVAddr base{cbuf_address(index)}; + return base != 0 ? gpu_memory.Read<u32>(base + offset) : 0U; + }, + [&](u32 binding, u32 offset) -> u32 { + if (binding >= info.storage_buffers_descriptors.size()) { + return 0U; + } + const auto& descriptor{info.storage_buffers_descriptors[binding]}; + const GPUVAddr cbuf{cbuf_address(descriptor.cbuf_index)}; + if (cbuf == 0) { + return 0U; + } + const GPUVAddr base{gpu_memory.Read<u64>(cbuf + descriptor.cbuf_offset)}; + return base != 0 ? gpu_memory.Read<u32>(base + offset) : 0U; + }) + .value_or(0U); +} + +} // namespace VideoCommon diff --git a/src/video_core/texture_cache/descriptor_table.h b/src/video_core/texture_cache/descriptor_table.h index e40c128ab5..75383809c5 100644 --- a/src/video_core/texture_cache/descriptor_table.h +++ b/src/video_core/texture_cache/descriptor_table.h @@ -30,11 +30,17 @@ public: } void Invalidate() noexcept { - std::ranges::fill(read_descriptors, 0); + // Cleared on the next read: the texture cache invalidates every channel's tables for + // each image it deletes, dozens of times a frame while its collector runs. + invalidated = true; } [[nodiscard]] std::pair<T, bool> Read(Tegra::MemoryManager const& gpu_memory, u32 index) noexcept { DEBUG_ASSERT(index <= current_limit); + if (invalidated) [[unlikely]] { + std::ranges::fill(read_descriptors, 0); + invalidated = false; + } const GPUVAddr gpu_addr = current_gpu_addr + index * sizeof(T); std::pair<T, bool> result; gpu_memory.ReadBlockUnsafe(gpu_addr, std::addressof(result.first), sizeof(T)); @@ -60,6 +66,7 @@ public: read_descriptors.resize(Common::DivCeil(num_descriptors, 64U)); old_size = (std::min)(old_size, read_descriptors.size()); std::fill(read_descriptors.begin(), read_descriptors.begin() + old_size, 0); + invalidated = false; // descriptors.resize(num_descriptors); } @@ -68,6 +75,7 @@ public: std::vector<T> descriptors; GPUVAddr current_gpu_addr{}; u32 current_limit{}; + bool invalidated{}; ///< read_descriptors is to be cleared before the next read }; } // namespace VideoCommon diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 55ec4c64df..3f3ae9cafd 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -178,6 +178,57 @@ void TextureCache<P>::RunGarbageCollector() { } return false; }; + // Deleting by age alone left every device memory block partly used: the collector deletes + // thousands of images a minute and new ones land in the holes, so blocks seldom empty (title + // 1: ~0.8 GB of holes in ~190 32 MiB blocks after ten minutes, which the driver counts as + // used). Among the oldest candidates (8 per image this pass may delete), those in the + // emptiest blocks go first, so blocks drain and go back to the driver; placement already + // prefers the fullest blocks (best fit). Images with memory of their own sit in the middle. + const auto Collect = [&] { + if constexpr (requires(const Image& image) { image.MemoryBlock(); }) { + std::vector<ImageId> candidates; + const size_t limit = num_iterations * 8; + lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, [&](ImageId image_id) { + candidates.push_back(image_id); + return candidates.size() >= limit; + }); + if (candidates.empty()) { + return; + } + if (block_image_bytes_frame == ~0ULL || frame_tick - block_image_bytes_frame >= 60) { + block_image_bytes_frame = frame_tick; + block_image_bytes.clear(); + for (const auto [image_id, image] : slot_images) { + const auto block = image->MemoryBlock(); + if (block.block != 0) { + block_image_bytes[block.block] += block.size; + } + } + } + std::vector<std::pair<double, ImageId>> ordered; + ordered.reserve(candidates.size()); + for (const ImageId image_id : candidates) { + const auto block = slot_images[image_id].MemoryBlock(); + double fill = 0.5; + if (block.block != 0 && block.block_size != 0) { + const auto it = block_image_bytes.find(block.block); + fill = it != block_image_bytes.end() + ? static_cast<double>(it->second) / static_cast<double>(block.block_size) + : 0.0; + } + ordered.emplace_back(fill, image_id); + } + std::stable_sort(ordered.begin(), ordered.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + for (const auto& [fill, image_id] : ordered) { + if (Cleanup(image_id)) { + break; + } + } + } else { + lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + } + }; Configure(false); static const u64 old_age = [] { const char* value = std::getenv("EDEN_NX2_TC_OLD_AGE"); @@ -188,15 +239,23 @@ void TextureCache<P>::RunGarbageCollector() { lru_cache.ForEachItemBelow(frame_tick - old_age, Cleanup); evict_old_costly = false; } - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + Collect(); if (total_used_memory >= critical_memory) { Configure(true); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + Collect(); } } template <class P> void TextureCache<P>::ApplyThresholdOverrides(bool log) { + // On every GPU: past what a frame needs, more device memory only keeps the textures of earlier + // scenes (title 1 filled 6.7 GB of an 8 GB card and draws its races in ~5 GB), so collect from + // MaxCriticalMemory on rather than near the card's budget. + critical_memory = (std::min)(critical_memory, MaxCriticalMemory); + expected_memory = + (std::min)(expected_memory, critical_memory - (std::min)(critical_memory, u64{768_MiB})); + minimum_memory = + (std::min)(minimum_memory, expected_memory - (std::min)(expected_memory, u64{1_GiB})); // EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB override the collector's thresholds. for (const auto& [name, target] : {std::pair{"EDEN_NX2_TC_EXPECTED_MIB", &expected_memory}, std::pair{"EDEN_NX2_TC_CRITICAL_MIB", &critical_memory}}) { @@ -242,26 +301,78 @@ void TextureCache<P>::TickFrame() { // budget as the card fills (16 GB card: 13.5 GB left mid-race), hence the re-read every 30 // frames. EDEN_NX2_TC_*_MIB overrides still win. if constexpr (requires { runtime.GetDeviceMemoryBudget(); }) { + if constexpr (requires { runtime.MemoryPressureEvents(); }) { + // Images that did not fit (made in host memory or with nothing bound) mean the + // thresholds sit too high for this card and driver: leave more room until the + // pressure has been gone for a while. + if (native_thresholds_applied) { + const u32 events = runtime.MemoryPressureEvents(); + if (events != seen_pressure_events) { + seen_pressure_events = events; + pressure_margin = (std::min)(pressure_margin + 512_MiB, 3_GiB); + pressure_quiet_frames = 0; + budget_check_frames = 0; // thresholds are recomputed below + LOG_WARNING(HW_GPU, "Out of device memory: keeping {} MiB more free", + pressure_margin >> 20); + } else if (pressure_margin != 0 && ++pressure_quiet_frames % 600 == 0) { + pressure_margin -= (std::min)(pressure_margin, 128_MiB); + budget_check_frames = 0; + } + } + } if (native_thresholds_applied && budget_check_frames++ % 30 == 0) { const u64 budget = runtime.GetDeviceMemoryBudget(); if (budget > 2_GiB) { const u64 old_critical = critical_memory; const u64 margin = (std::min)(budget / 8, 1_GiB); - critical_memory = budget - margin; - expected_memory = budget - 2 * margin; + if constexpr (requires { runtime.GetDeviceMemorySlack(); }) { + // The usage counted here leaves out the unused part of the allocator's + // blocks, which the driver counts and a full heap cannot hand to a new + // image (RX 7800 XT: 1.06 GiB of it, the whole margin, with the heap at + // its budget for a minute before images stopped fitting). Reserve it + // too: at once when it grows, slowly when it shrinks, so evicting images + // (which leaves partly used blocks) does not chase the threshold down. + const u64 slack = (std::min)(runtime.GetDeviceMemorySlack(), 1536_MiB); + slack_margin = slack >= slack_margin + ? slack + : slack_margin - (std::min)(slack_margin - slack, 16_MiB); + } + critical_memory = + budget - (std::min)(margin + pressure_margin + slack_margin, budget / 2); + expected_memory = critical_memory - (std::min)(margin, critical_memory / 2); // Log the first values and real changes (the budget moves by a few MiB all // the time). const u64 change = critical_memory > old_critical ? critical_memory - old_critical : old_critical - critical_memory; - ApplyThresholdOverrides(budget_check_frames == 1 || change >= 256_MiB); + const bool log = budget_check_frames == 1 || change >= 256_MiB; + ApplyThresholdOverrides(log); + if (log) { + LOG_INFO(HW_GPU, + "Memory margins: base {} MiB, out of memory {} MiB, block slack {} MiB", + margin >> 20, pressure_margin >> 20, slack_margin >> 20); + } } } } } + if constexpr (requires { runtime.CurrentTick(); }) { + while (!gpu_sentenced_images.empty() && + runtime.IsFree(gpu_sentenced_images.front().tick)) { + gpu_sentenced_bytes -= gpu_sentenced_images.front().bytes; + gpu_sentenced_images.pop_front(); + } + } // If we can obtain the memory info, use it instead of the estimate. if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); + // Images the collector deleted keep their memory for TICKS_TO_DESTROY frames: counting + // it kept the collector in aggressive mode, deleting 40+ images a frame for frames after + // it had freed enough (title 1: frames over 100 ms for seconds at a time). + for (const u64 bytes : sentenced_image_bytes) { + total_used_memory -= (std::min)(total_used_memory, bytes); + } + total_used_memory -= (std::min)(total_used_memory, gpu_sentenced_bytes); } // Every 600 frames: device memory in use against the collector's thresholds, and what the // images themselves account for (VRAM pressure makes the collector thrash). @@ -328,10 +439,18 @@ void TextureCache<P>::TickFrame() { RunGarbageCollector(); } sentenced_images.Tick(); + sentenced_image_index = (sentenced_image_index + 1) % TICKS_TO_DESTROY; + sentenced_image_bytes[sentenced_image_index] = 0; sentenced_framebuffers.Tick(); sentenced_image_view.Tick(); TickAsyncDecode(); TickAsyncUnswizzle(); + if (frame_tick % 600 == 0 && !uploaded_ranges.empty()) { + for (const auto& [address, size] : uploaded_ranges) { + device_memory.HintCold(address, size); + } + uploaded_ranges.clear(); + } runtime.TickFrame(); ++frame_tick; @@ -690,6 +809,7 @@ typename P::Framebuffer* TextureCache<P>::GetFramebuffer() { template <class P> ImageViewId TextureCache<P>::VisitImageView(u32 index, bool compute) { auto& table = compute ? channel_state->compute_image_table : channel_state->graphics_image_table; + index = Tegra::Texture::TextureHeaderIndex(index, table.current_limit); if (index > table.current_limit) { LOG_DEBUG(HW_GPU, "Invalid image view index={}", index); return NULL_IMAGE_VIEW_ID; @@ -701,8 +821,13 @@ ImageViewId TextureCache<P>::VisitImageView(u32 index, bool compute) { if (IsValidEntry(*gpu_memory, descriptor)) { // Is new (registered view) on the texture cache side? const auto [pair, is_new_tc] = channel_state->image_views.try_emplace(descriptor); - if (is_new_tc) + if (is_new_tc) { pair->second = CreateImageView(descriptor); + if (pair->second != NULL_IMAGE_VIEW_ID) { + image_view_descriptors[pair->second].emplace_back(current_channel_id, + descriptor); + } + } PrepareImageView(pair->second, false, false); channel_state->image_view_ids.insert_or_assign(map_index, pair->second); return pair->second; @@ -1444,6 +1569,21 @@ void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) { auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image)); UploadImageContents(image, staging); runtime.InsertUploadMemoryBarrier(); + // The host GPU has its own copy now and the guest rarely reads this memory again: TickFrame + // marks it cold, so the OS pages it out before anything else when it needs RAM (title 1 + // streams several GB of textures through it). + constexpr u64 ColdHintMinBytes = 64_KiB; + if (image.guest_size_bytes >= ColdHintMinBytes && + False(image.flags & ImageFlagBits::GpuModified)) { + if (True(image.flags & ImageFlagBits::Sparse)) { + // Non-contiguous: each mapped piece (title 1 maps its own memory into these). + ForEachSparseSegment(image, [&](GPUVAddr, DAddr address, size_t size) { + uploaded_ranges.emplace_back(address, size); + }); + } else { + uploaded_ranges.emplace_back(image.cpu_addr, image.guest_size_bytes); + } + } } template <class P> @@ -1870,9 +2010,14 @@ void TextureCache<P>::InvalidateScale(Image& image) { for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; +#ifndef NDEBUG + // A walk over every entry (title 1: ~5,400 in 5 channels) for each deleted image: debug + // builds only. The tables are invalidated below, so each entry is written again before + // it is read. if constexpr (ENABLE_VALIDATION) for (auto& e : channel_info.image_view_ids) e.second = CORRUPT_ID; +#endif channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } @@ -2990,6 +3135,8 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { } // TickFrame sets the total from the driver, which can be under the sum of these estimates. total_used_memory -= (std::min)(total_used_memory, ImageMemoryBytes(slot_images[image_id])); + const u64 freed_bytes = (image.HasScaled() ? GetScaledImageSizeBytes(image) : 0) + + ImageMemoryBytes(slot_images[image_id]); const GPUVAddr gpu_addr = image.gpu_addr; const auto alloc_it = image_allocs_table.find(gpu_addr); if (alloc_it == image_allocs_table.end()) { @@ -3054,19 +3201,32 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { if (retained) { LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value()); } else if (!immediate_delete) { - sentenced_images.Push(std::move(slot_images[image_id])); + if constexpr (requires { runtime.CurrentTick(); }) { + gpu_sentenced_images.push_back( + {runtime.CurrentTick(), freed_bytes, std::move(slot_images[image_id])}); + gpu_sentenced_bytes += freed_bytes; + } else { + sentenced_images.Push(std::move(slot_images[image_id])); + sentenced_image_bytes[sentenced_image_index] += freed_bytes; + } } slot_images.erase(image_id); alloc_images.erase(alloc_image_it); if (alloc_images.empty()) { image_allocs_table.erase(alloc_it); + slot_image_allocs.erase(alloc_id); } for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; +#ifndef NDEBUG + // A walk over every entry (title 1: ~5,400 in 5 channels) for each deleted image: debug + // builds only. The tables are invalidated below, so each entry is written again before + // it is read. if constexpr (ENABLE_VALIDATION) for (auto& e : channel_info.image_view_ids) e.second = CORRUPT_ID; +#endif channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } @@ -3075,17 +3235,23 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { template <class P> void TextureCache<P>::RemoveImageViewReferences(std::span<const ImageViewId> removed_views) { - for (size_t c : active_channel_ids) { - auto& channel_info = channel_storage[c]; - auto it = channel_info.image_views.begin(); - while (it != channel_info.image_views.end()) { - const auto found = std::ranges::find(removed_views, it->second); - if (found != removed_views.end()) { - it = channel_info.image_views.erase(it); - } else { - ++it; + for (const ImageViewId removed : removed_views) { + const auto refs = image_view_descriptors.find(removed); + if (refs == image_view_descriptors.end()) { + continue; + } + for (const auto& [channel, descriptor] : refs->second) { + // The channel may be gone, or its entry point to another view since. + if (std::ranges::find(active_channel_ids, channel) == active_channel_ids.end()) { + continue; + } + auto& image_views = channel_storage[channel].image_views; + if (const auto it = image_views.find(descriptor); + it != image_views.end() && it->second == removed) { + image_views.erase(it); } } + image_view_descriptors.erase(refs); } } @@ -3102,6 +3268,7 @@ void TextureCache<P>::RemoveFramebuffers(std::span<const ImageViewId> removed_vi last_framebuffer_serial = 0; } sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id])); + slot_framebuffers.erase(framebuffer_id); it = framebuffers.erase(it); } else { ++it; diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index d4b0f597ae..09deb8da02 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -136,6 +136,10 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI u64 native_critical_memory{}; u64 native_expected_memory{}; u64 budget_check_frames{}; + u32 seen_pressure_events{}; ///< Allocator pressure events already answered + u64 pressure_margin{}; ///< Extra room under the budget after images did not fit + u64 pressure_quiet_frames{}; ///< Frames since the last such image + u64 slack_margin{}; ///< Extra room for allocator block space the driver counts as used u64 astc_opaque_images{}; u64 astc_alpha_images{}; std::vector<u8> astc_scan_buffer; @@ -144,6 +148,9 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB; static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB; + /// The most device memory the collector lets the cache fill before it collects in earnest, + /// on every GPU (ApplyThresholdOverrides). + static constexpr u64 MaxCriticalMemory = 5_GiB; static constexpr size_t GC_EMERGENCY_COUNTS = 2; using Runtime = typename P::Runtime; @@ -452,6 +459,12 @@ private: /// Remove image views references from the cache void RemoveImageViewReferences(std::span<const ImageViewId> removed_views); + /// Per image view, the (channel, descriptor) entries of the channels' image_views maps that + /// point to it. Deleting an image erased them by scanning every map (title 1: ~5,000 entries, + /// 50-130 us for each deleted image, dozens a frame while the collector runs). + std::unordered_map<ImageViewId, boost::container::small_vector<std::pair<size_t, TICEntry>, 2>> + image_view_descriptors; + /// Remove framebuffers using the given image views from the cache void RemoveFramebuffers(std::span<const ImageViewId> removed_views); @@ -571,6 +584,25 @@ private: static constexpr size_t TICKS_TO_DESTROY = 8; #endif DelayedDestructionRing<Image, TICKS_TO_DESTROY> sentenced_images; + /// Device memory of the images in each slot of sentenced_images (same index): deleted, but + /// counted in the driver's usage until the ring destroys them. + std::array<u64, TICKS_TO_DESTROY> sentenced_image_bytes{}; + size_t sentenced_image_index = 0; + /// Deleted images on runtimes that report GPU progress (CurrentTick, IsFree): destroyed, with + /// their device memory, as soon as the GPU is past them rather than TICKS_TO_DESTROY frames + /// later. While the collector deletes dozens a frame, eight frames of them held 1-1.5 GB of + /// VRAM above its threshold (title 1 on an 8 GB card: 6.6-7.0 GB in use against 5 GiB). + struct GpuSentencedImage { + u64 tick; + u64 bytes; + Image image; + }; + std::deque<GpuSentencedImage> gpu_sentenced_images; + u64 gpu_sentenced_bytes{}; + + /// Bytes of images per device memory block, for the collector (refreshed every 60 frames). + std::unordered_map<u64, u64> block_image_bytes; + u64 block_image_bytes_frame{~0ULL}; DelayedDestructionRing<ImageView, TICKS_TO_DESTROY> sentenced_image_view; DelayedDestructionRing<Framebuffer, TICKS_TO_DESTROY> sentenced_framebuffers; @@ -585,6 +617,8 @@ private: Common::ThreadWorker texture_decode_worker{1, "TextureDecoder", {}, Common::ThreadPlacement::Efficiency}; std::vector<std::unique_ptr<AsyncDecodeContext>> async_decodes; + /// Guest memory images were uploaded from since the last cold hint (device address, size). + std::vector<std::pair<DAddr, u64>> uploaded_ranges; std::deque<PendingUnswizzle> unswizzle_queue; u8 current_unswizzle_frame; diff --git a/src/video_core/textures/astc.cpp b/src/video_core/textures/astc.cpp index 950ce28d6b..a0d7e6e455 100644 --- a/src/video_core/textures/astc.cpp +++ b/src/video_core/textures/astc.cpp @@ -2038,38 +2038,74 @@ void Decompress(std::span<const uint8_t> data, uint32_t width, uint32_t height, } } +// What IsOpaque needs from a block's 11 mode bits, which are all DecodeBlockInfo reads (a void +// extent block also needs bits 10 and 11 set, checked per block). +struct BlockModeInfo { + bool error; + bool void_extent; + bool void_extent_hdr; + bool dual_plane; + u16 weight_bits; +}; + +static std::array<BlockModeInfo, 2048> MakeBlockModeInfos() { + std::array<BlockModeInfo, 2048> infos{}; + for (u32 mode = 0; mode < infos.size(); ++mode) { + // Bit 11 set, so a void extent mode decodes without its reserved-bit error. + const std::array<u8, 16> block{static_cast<u8>(mode), static_cast<u8>((mode >> 8) | 0x8)}; + InputBitStream strm(block); + const TexelWeightParams params = DecodeBlockInfo(strm); + BlockModeInfo& info = infos[mode]; + info.void_extent = params.m_bVoidExtentLDR || params.m_bVoidExtentHDR; + info.void_extent_hdr = params.m_bVoidExtentHDR; + info.error = params.m_bError && !info.void_extent; + info.dual_plane = params.m_bDualPlane; + info.weight_bits = info.error || info.void_extent + ? u16{0} + : static_cast<u16>(params.GetPackedBitSize()); + } + return infos; +} + bool IsOpaque(std::span<const uint8_t> data) { // Endpoint modes with an alpha channel: LDR luminance+alpha (4, 5), RGB scale + two alpha // (10), LDR RGBA (12, 13) and HDR RGBA (14, 15). constexpr u32 AlphaModes = (1U << 4) | (1U << 5) | (1U << 10) | (1U << 12) | (1U << 13) | (1U << 14) | (1U << 15); + // Decoding each block's mode bit by bit took ~12% of title 1's long frames, where every + // new ASTC texture is scanned (its title screen streams them in by the thousand). + static const std::array<BlockModeInfo, 2048> mode_infos = MakeBlockModeInfos(); for (size_t offset = 0; offset + 16 <= data.size(); offset += 16) { const std::span<const u8> block = data.subspan(offset, 16); - InputBitStream strm(block); - const TexelWeightParams params = DecodeBlockInfo(strm); - if (params.m_bError) { + u32 low; + std::memcpy(&low, block.data(), sizeof(low)); + const BlockModeInfo& info = mode_infos[low & 0x7FF]; + if (info.error) { continue; // decodes to the opaque error colour } - if (params.m_bVoidExtentLDR || params.m_bVoidExtentHDR) { + if (info.void_extent) { + if ((low & 0xC00) != 0xC00) { + continue; // reserved bits clear: the error colour too + } // One RGBA16 colour in bits 64-127: UNORM16, or FP16 for HDR. const u32 alpha = block[14] | (u32{block[15]} << 8); - if (alpha != (params.m_bVoidExtentHDR ? 0x3C00U : 0xFFFFU)) { + if (alpha != (info.void_extent_hdr ? 0x3C00U : 0xFFFFU)) { return false; } continue; } - const u32 partitions = strm.ReadBits<2>() + 1; - if (partitions == 4 && params.m_bDualPlane) { + const u32 partitions = ((low >> 11) & 3) + 1; + if (partitions == 4 && info.dual_plane) { continue; } if (partitions == 1) { - if ((AlphaModes >> strm.ReadBits<4>()) & 1) { + if ((AlphaModes >> ((low >> 13) & 0xF)) & 1) { return false; } continue; } - strm.ReadBits<10>(); // partition index - const u32 base_cem = strm.ReadBits<6>(); + // Bits 13-22 are the partition index. + const u32 base_cem = (low >> 23) & 0x3F; const u32 base_mode = base_cem & 3; if (base_mode == 0) { if ((AlphaModes >> (base_cem >> 2)) & 1) { @@ -2078,7 +2114,7 @@ bool IsOpaque(std::span<const uint8_t> data) { continue; } // The rest of the per-partition modes sit just below the weights (DecompressBlock). - const u32 weight_bits = params.GetPackedBitSize(); + const u32 weight_bits = info.weight_bits; if (weight_bits < 24 || weight_bits > 96) { continue; } diff --git a/src/video_core/textures/texture.h b/src/video_core/textures/texture.h index 9fc6a631bd..79396d8a68 100644 --- a/src/video_core/textures/texture.h +++ b/src/video_core/textures/texture.h @@ -165,6 +165,17 @@ static_assert(sizeof(TextureHandle) == 4, "TextureHandle has wrong size"); } } +/// Some Switch 2 bindless handles (Switch Sports' scenery, read from a uniform register) hold +/// the texture header's byte offset in the pool, index * 32, where the handle normally has the +/// index. Read it as an offset only when the plain index falls past the pool and the offset +/// lands inside it, so handles that already resolve are left alone. +[[nodiscard]] constexpr u32 TextureHeaderIndex(u32 index, u32 limit) { + if (index > limit && index % 32 == 0 && index / 32 <= limit) { + return index / 32; + } + return index; +} + struct TICEntry { union { struct { diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index b65c721bb8..01e13cb384 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -779,10 +779,14 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR .flags = flags, .physicalDevice = physical, .device = *logical, - // 64 MiB blocks on discrete GPUs too: with 256 MiB ones, Switch 2 titles' mix of + // 16 MiB blocks on discrete GPUs too: with 256 MiB ones, Switch 2 titles' mix of // texture sizes left ~38% of the image blocks empty (2 GB of an 8 GB card in title - // 1), which the texture cache counted as used and kept evicting textures for. - .preferredLargeHeapBlockSize = 64u * 1024u * 1024u, + // 1), which the texture cache counted as used and kept evicting textures for. With + // 64 MiB ones textures created and deleted by the thousand still left ~1.7 GB of holes + // after ten minutes, with 32 MiB ones ~0.5 GB: a block drains only once all of its + // images are gone, and fewer per block drain sooner. VMA gives allocations over half a + // block (8 MiB) their own memory, which goes back to the driver when they are freed. + .preferredLargeHeapBlockSize = 16u * 1024u * 1024u, .pAllocationCallbacks = nullptr, .pDeviceMemoryCallbacks = nullptr, .pHeapSizeLimit = heap_size_limit != 0 ? heap_size_limits.data() : nullptr, @@ -804,9 +808,37 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR Device::~Device() { SaveStaticPipelineCache(); ShutdownGPULogging(); + for (auto& pools : separate_pools) { + for (VmaPool pool : pools) { + if (pool != VK_NULL_HANDLE) { + vmaDestroyPool(allocator, pool); + } + } + } vmaDestroyAllocator(allocator); } +VmaPool Device::GetSeparatePool(SeparatePool kind, u32 memory_type) const { + std::scoped_lock lk{separate_pools_mutex}; + VmaPool& pool = separate_pools[static_cast<u32>(kind)][memory_type]; + if (pool == VK_NULL_HANDLE) { + const VmaPoolCreateInfo ci{ + .memoryTypeIndex = memory_type, + .flags = 0, + .blockSize = SeparatePoolBlockSize, + .minBlockCount = 0, + .maxBlockCount = 0, + .priority = 0.f, + .minAllocationAlignment = 0, + .pMemoryAllocateNext = nullptr, + }; + if (vmaCreatePool(allocator, &ci, &pool) != VK_SUCCESS) { + pool = VK_NULL_HANDLE; + } + } + return pool; +} + void Device::LoadStaticPipelineCache() { const auto create = [this](size_t size, const void* data) { static_pipeline_cache = logical.CreatePipelineCache({ @@ -1515,6 +1547,15 @@ void Device::RemoveUnsuitableExtensions() { VK_KHR_PIPELINE_EXECUTABLE_PROPERTIES_EXTENSION_NAME); } + // VK_EXT_mesh_shader: Switch 2 titles draw some geometry with task and mesh programs. Only + // the two stages are used; the multiview, shading rate and query features need more. + features.mesh_shader.multiviewMeshShader = VK_FALSE; + features.mesh_shader.primitiveFragmentShadingRateMeshShader = VK_FALSE; + features.mesh_shader.meshShaderQueries = VK_FALSE; + extensions.mesh_shader = features.mesh_shader.taskShader && features.mesh_shader.meshShader; + RemoveExtensionFeatureIfUnsuitable(extensions.mesh_shader, features.mesh_shader, + VK_EXT_MESH_SHADER_EXTENSION_NAME); + // VK_KHR_shader_quad_control extensions.shader_quad_control = features.shader_quad_control.shaderQuadControl; RemoveExtensionFeatureIfUnsuitable(extensions.shader_quad_control, features.shader_quad_control, @@ -1643,6 +1684,16 @@ u64 Device::GetDeviceMemoryUsage() const { return result; } +u64 Device::GetDeviceMemorySlack() const { + if (allocator == nullptr) { + return 0; + } + std::array<VmaBudget, VK_MAX_MEMORY_HEAPS> heap_budgets{}; + vmaGetHeapBudgets(allocator, heap_budgets.data()); + const VmaStatistics& statistics = heap_budgets[main_heap].statistics; + return statistics.blockBytes - (std::min)(statistics.blockBytes, statistics.allocationBytes); +} + u64 Device::GetDeviceMemoryBudget() const { if (!extensions.memory_budget) { return 0; diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index 8237868c6d..622d792210 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -6,8 +6,10 @@ #pragma once +#include <array> #include <atomic> #include <cstdlib> +#include <mutex> #include <optional> #include <set> #include <span> @@ -21,6 +23,7 @@ #include "video_core/vulkan_common/vulkan_wrapper.h" VK_DEFINE_HANDLE(VmaAllocator) +VK_DEFINE_HANDLE(VmaPool) // Define all features which may be used by the implementation here. // Vulkan version in the macro describes the minimum version required for feature availability. @@ -66,6 +69,7 @@ VK_DEFINE_HANDLE(VmaAllocator) FEATURE(EXT, 4444Formats, 4444_FORMATS, format_a4b4g4r4) \ FEATURE(EXT, IndexTypeUint8, INDEX_TYPE_UINT8, index_type_uint8) \ FEATURE(EXT, LineRasterization, LINE_RASTERIZATION, line_rasterization) \ + FEATURE(EXT, MeshShader, MESH_SHADER, mesh_shader) \ FEATURE(EXT, PrimitiveTopologyListRestart, PRIMITIVE_TOPOLOGY_LIST_RESTART, \ primitive_topology_list_restart) \ FEATURE(EXT, ProvokingVertex, PROVOKING_VERTEX, provoking_vertex) \ @@ -264,6 +268,17 @@ public: return allocator; } + /// Allocations kept out of the default blocks, which then hold images only: churned buffers + /// and long-lived sparse pages left holes the texture cache's collector couldn't drain. + enum class SeparatePool : u32 { + SparsePages, + DeviceBuffers, + }; + static constexpr VkDeviceSize SeparatePoolBlockSize = 16ULL << 20; + + /// The pool of that kind for a memory type, created on first use (null if that failed). + VmaPool GetSeparatePool(SeparatePool kind, u32 memory_type) const; + /// Returns the logical device. const vk::Device& GetLogical() const { return logical; @@ -633,6 +648,12 @@ FN_MAX_LIMIT_LIST return extensions.shader_quad_control && features.shader_quad_control.shaderQuadControl; } + /// Returns true if the device supports task and mesh shaders (VK_EXT_mesh_shader). + bool IsExtMeshShaderSupported() const { + return extensions.mesh_shader && features.mesh_shader.taskShader && + features.mesh_shader.meshShader; + } + /// Returns true if the device supports VK_KHR_image_format_list. bool IsKhrImageFormatListSupported() const { return extensions.image_format_list || instance_version >= VK_API_VERSION_1_2; @@ -951,6 +972,11 @@ FN_MAX_LIMIT_LIST return 0x00010300U; } + /// Returns true for an integrated GPU (its device-local memory is system memory). + bool IsIntegrated() const { + return is_integrated; + } + /// Returns true when a known debugging tool is attached. bool HasDebuggingToolAttached() const { return has_renderdoc || has_nsight_graphics || has_radeon_gpu_profiler; @@ -993,9 +1019,15 @@ FN_MAX_LIMIT_LIST /// The driver's current memory budget for this process (VK_EXT_memory_budget), in /// GetDeviceMemoryUsage() terms: the usage at which the main device-local heap reaches its - /// budget. 0 without the extension. + /// budget. 0 without the extension. The unused part of the allocator's blocks is not + /// subtracted: see GetDeviceMemorySlack(). u64 GetDeviceMemoryBudget() const; + /// Memory the allocator holds from the driver in the main device-local heap but has no + /// allocation in (the unused part of its blocks). GetDeviceMemoryUsage() does not count it, + /// yet the driver does: with the heap at its budget it is not available to new images. + u64 GetDeviceMemorySlack() const; + u32 GetSetsPerPool() const { return sets_per_pool; } @@ -1179,6 +1211,8 @@ private: private: VkInstance instance; ///< Vulkan instance. VmaAllocator allocator{}; ///< VMA allocator. + mutable std::mutex separate_pools_mutex; + mutable std::array<std::array<VmaPool, VK_MAX_MEMORY_TYPES>, 2> separate_pools{}; vk::DeviceDispatch dld; ///< Device function pointers. vk::PhysicalDevice physical; ///< Physical device. vk::Device logical; ///< Logical device. diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 1798611e26..a0a39abe39 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -209,7 +209,10 @@ MemoryAllocator::~MemoryAllocator() = default; vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicated) const { const VmaAllocationCreateInfo alloc_ci = { - .flags = static_cast<VmaAllocationCreateFlags>(VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) | + // Best fit: images freed by the thousand leave holes in the blocks, which new ones + // should fill before VMA takes another block. + .flags = static_cast<VmaAllocationCreateFlags>(VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | + VMA_ALLOCATION_CREATE_STRATEGY_MIN_MEMORY_BIT) | (dedicated ? VmaAllocationCreateFlags{VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT} : VmaAllocationCreateFlags{0}), .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, @@ -242,11 +245,32 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, ci.arrayLayers, static_cast<u32>(ci.format)); } + pressure_events.fetch_add(1, std::memory_order_relaxed); VmaAllocationCreateInfo relaxed = alloc_ci; relaxed.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; result = vmaCreateImage(allocator, &ci, &relaxed, &handle, &allocation, &alloc_info); } + if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && !huge) { + // Still out of device memory: a real image in host memory (slow, but valid) beats one + // with nothing bound, which AMD's drivers do not survive being sampled. + VmaAllocationCreateInfo host = alloc_ci; + host.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + host.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST; + host.preferredFlags = 0; + result = vmaCreateImage(allocator, &ci, &host, &handle, &allocation, &alloc_info); + static u32 host_logged{}; + if (result == VK_SUCCESS && host_logged++ < 16) { + LOG_WARNING(Render_Vulkan, + "Image {}x{}x{} ({} levels, {} layers, format {}) is out of device memory; " + "placed in host memory", + ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, + ci.arrayLayers, static_cast<u32>(ci.format)); + } + } if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && ci.imageType != VK_IMAGE_TYPE_1D) { + if (!huge) { + pressure_events.fetch_add(1, std::memory_order_relaxed); + } // Too big to back at all, e.g. title 1's sparse virtual texture arrays (8192x8192, // 68 layers, ~23 GB): on the console only resident tiles have memory. Use an image with // nothing bound, so views and copies still work, instead of aborting. @@ -313,6 +337,24 @@ VmaAllocation MemoryAllocator::AllocateSparseMemory(const VkMemoryRequirements & }; VmaAllocation allocation{}; VmaAllocationInfo info{}; + // In a pool of their own (see Device::SeparatePool); anything over half a block, or a pool + // allocation that fails, takes the default blocks. + u32 memory_type{}; + if (requirements.size <= Device::SeparatePoolBlockSize / 2 && + vmaFindMemoryTypeIndex(allocator, requirements.memoryTypeBits, &alloc_ci, &memory_type) == + VK_SUCCESS) { + if (const VmaPool pool = + device.GetSeparatePool(Device::SeparatePool::SparsePages, memory_type)) { + VmaAllocationCreateInfo pool_ci = alloc_ci; + pool_ci.pool = pool; + if (vmaAllocateMemory(allocator, &requirements, &pool_ci, &allocation, &info) == + VK_SUCCESS) { + memory = info.deviceMemory; + offset = info.offset; + return allocation; + } + } + } if (vmaAllocateMemory(allocator, &requirements, &alloc_ci, &allocation, &info) != VK_SUCCESS) { return VK_NULL_HANDLE; } @@ -327,6 +369,39 @@ void MemoryAllocator::FreeSparseMemory(VmaAllocation allocation) const { } } +bool MemoryAllocator::CreateBufferInSeparatePool(const VkBufferCreateInfo& ci, MemoryUsage usage, + const VmaAllocationCreateInfo& alloc_ci, + VkBuffer& handle, VmaAllocation& allocation, + VmaAllocationInfo& alloc_info) const { + // The buffer cache's buffers, churned by its collector: in a pool of their own (see + // Device::SeparatePool). Anything over half a block takes the default blocks (dedicated). + if (usage != MemoryUsage::DeviceLocal || ci.size > Device::SeparatePoolBlockSize / 2) { + return false; + } + u32 memory_type{}; + { + std::scoped_lock lk{buffer_type_mutex}; + const auto key = (u64{ci.usage} << 32) | ci.flags; + if (const auto it = buffer_memory_types.find(key); it != buffer_memory_types.end()) { + memory_type = it->second; + } else if (vmaFindMemoryTypeIndexForBufferInfo(allocator, &ci, &alloc_ci, &memory_type) == + VK_SUCCESS) { + buffer_memory_types.emplace(key, memory_type); + } else { + return false; + } + } + const VmaPool pool = device.GetSeparatePool(Device::SeparatePool::DeviceBuffers, memory_type); + if (pool == VK_NULL_HANDLE) { + return false; + } + VmaAllocationCreateInfo pool_ci = alloc_ci; + pool_ci.pool = pool; + pool_ci.memoryTypeBits = 0; + return vmaCreateBuffer(allocator, &ci, &pool_ci, &handle, &allocation, &alloc_info) == + VK_SUCCESS; +} + vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const { // MESA will do memcpy() if not marked as host cached, so just force mark it for most buffers auto const anv_flags = (usage == MemoryUsage::Stream @@ -348,7 +423,9 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa VmaAllocation allocation{}; VkMemoryPropertyFlags property_flags{}; - vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + if (!CreateBufferInSeparatePool(ci, usage, alloc_ci, handle, allocation, alloc_info)) { + vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + } vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags); // Log GPU memory allocation for buffers diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index e1c267ab79..8b9e72b3e3 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -6,8 +6,11 @@ #pragma once +#include <atomic> #include <memory> +#include <mutex> #include <span> +#include <unordered_map> #include <vector> #include "common/common_types.h" @@ -113,11 +116,23 @@ namespace Vulkan { vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci, bool aliased = false) const; /// Device memory to bind to part of a sparse resource. Null when there is none left. + /// Device-local buffers in their own VMA pool; false when it doesn't apply or failed. + bool CreateBufferInSeparatePool(const VkBufferCreateInfo& ci, MemoryUsage usage, + const VmaAllocationCreateInfo& alloc_ci, VkBuffer& handle, + VmaAllocation& allocation, + VmaAllocationInfo& alloc_info) const; + VmaAllocation AllocateSparseMemory(const VkMemoryRequirements &requirements, VkDeviceMemory &memory, VkDeviceSize &offset) const; void FreeSparseMemory(VmaAllocation allocation) const; + /// Counts the images that did not fit the device memory budget (made anyway, in host + /// memory or with nothing bound): the texture cache collects harder when it grows. + u32 MemoryPressureEvents() const noexcept { + return pressure_events.load(std::memory_order_relaxed); + } + vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const; vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage, @@ -153,6 +168,10 @@ namespace Vulkan { const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties. VkDeviceSize buffer_image_granularity; ///< Adjacent buffer/image granularity u32 valid_memory_types{~0u}; + mutable std::atomic<u32> pressure_events{0}; + mutable std::mutex buffer_type_mutex; + /// Memory type of device-local buffers by usage and flags (CreateBufferInSeparatePool). + mutable std::unordered_map<u64, u32> buffer_memory_types; }; /// Logs VMA's blocks and allocations per memory type, and each heap's budget. diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index dbaaf450a0..d0bca88125 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -116,6 +116,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkCmdDrawIndirectCount); X(vkCmdDrawIndexedIndirectCount); X(vkCmdDrawIndirectByteCountEXT); + X(vkCmdDrawMeshTasksEXT); X(vkCmdEndConditionalRenderingEXT); X(vkCmdEndQuery); X(vkCmdEndRenderPass); @@ -525,6 +526,19 @@ void Image::Release() const noexcept { } } +Image::MemoryBlock Image::GetMemoryBlock() const noexcept { + if (allocation == nullptr) { + return {}; + } + VmaAllocationInfo2 info{}; + vmaGetAllocationInfo2(allocator, allocation, &info); + if (info.dedicatedMemory) { + return {0, info.blockSize, info.allocationInfo.size}; + } + return {reinterpret_cast<u64>(info.allocationInfo.deviceMemory), info.blockSize, + info.allocationInfo.size}; +} + void Buffer::Flush() const { if (!is_coherent) { vmaFlushAllocation(allocator, allocation, 0, VK_WHOLE_SIZE); diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index 3010368973..f49e9518bb 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -232,6 +232,7 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkCmdDrawIndirectCount vkCmdDrawIndirectCount{}; PFN_vkCmdDrawIndexedIndirectCount vkCmdDrawIndexedIndirectCount{}; PFN_vkCmdDrawIndirectByteCountEXT vkCmdDrawIndirectByteCountEXT{}; + PFN_vkCmdDrawMeshTasksEXT vkCmdDrawMeshTasksEXT{}; PFN_vkCmdEndConditionalRenderingEXT vkCmdEndConditionalRenderingEXT{}; PFN_vkCmdEndDebugUtilsLabelEXT vkCmdEndDebugUtilsLabelEXT{}; PFN_vkCmdEndQuery vkCmdEndQuery{}; @@ -736,6 +737,15 @@ public: return usage; } + /// Where the image's memory lies: the block (0 when the image has no memory or a block of + /// its own), the block's size and the image's part of it. + struct MemoryBlock { + u64 block; + u64 block_size; + u64 size; + }; + [[nodiscard]] MemoryBlock GetMemoryBlock() const noexcept; + private: void Release() const noexcept; @@ -1319,6 +1329,10 @@ public: dld->vkCmdDrawIndexedIndirect(handle, src_buffer, src_offset, draw_count, stride); } + void DrawMeshTasksEXT(u32 group_count_x, u32 group_count_y, u32 group_count_z) const noexcept { + dld->vkCmdDrawMeshTasksEXT(handle, group_count_x, group_count_y, group_count_z); + } + void DrawIndirectCount(VkBuffer src_buffer, VkDeviceSize src_offset, VkBuffer count_buffer, VkDeviceSize count_offset, u32 draw_count, u32 stride) const noexcept { dld->vkCmdDrawIndirectCount(handle, src_buffer, src_offset, count_buffer, count_offset, diff --git a/src/yuzu/main_window.cpp b/src/yuzu/main_window.cpp index 2b43bcd2d0..2b600946bc 100644 --- a/src/yuzu/main_window.cpp +++ b/src/yuzu/main_window.cpp @@ -451,7 +451,7 @@ MainWindow::MainWindow(bool has_broken_vulkan) QtCommon::SetupContentProviders(); // Gen keys if necessary - OnCheckFirmwareDecryption(); + OnCheckFirmwareDecryption(false); #ifdef __unix__ OnCheckGraphicsBackend(); @@ -4550,8 +4550,13 @@ void MainWindow::OnMouseActivity() { } } -void MainWindow::OnCheckFirmwareDecryption() { - if (!ContentManager::AreKeysPresent()) { +void MainWindow::OnCheckFirmwareDecryption(bool prompt) { + if (!ContentManager::AreKeysPresent() && !prompt) { + // At startup only log it: loose ExeFS/RomFS dumps (all Switch 2 games so far) need no + // keys, and a modal question blocked every unattended launch. Tools > Install Decryption + // Keys still installs them. + LOG_WARNING(Frontend, "Decryption keys are missing: NCA, NSP and XCI games need them"); + } else if (!ContentManager::AreKeysPresent()) { const auto res = QtCommon::Frontend::Warning( tr("Derivation Components Missing"), tr("Decryption keys are missing. Install them now?"), diff --git a/src/yuzu/main_window.h b/src/yuzu/main_window.h index 11732ff242..766e4d7345 100644 --- a/src/yuzu/main_window.h +++ b/src/yuzu/main_window.h @@ -424,7 +424,8 @@ private slots: void OnCreateHomeMenuDesktopShortcut(); void OnCreateHomeMenuApplicationMenuShortcut(); void OnCaptureScreenshot(); - void OnCheckFirmwareDecryption(); + /// Checks for decryption keys; `prompt` asks to install them when they are missing. + void OnCheckFirmwareDecryption(bool prompt = true); #ifdef __unix__ void OnCheckGraphicsBackend(); #endif |